dsh-llm-verifier 0.1.13 → 0.1.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/LICENSE +22 -22
  2. package/README.md +328 -301
  3. package/cordis.patch.yml +4 -4
  4. package/lib/caller-BWwSVNZf.js.map +1 -0
  5. package/lib/client.js +176 -14
  6. package/lib/client.js.map +1 -0
  7. package/lib/core.js.map +1 -0
  8. package/lib/index.js +400 -51
  9. package/lib/index.js.map +1 -0
  10. package/lib/types/auto.d.ts.map +1 -0
  11. package/lib/types/auto.js +8 -2
  12. package/lib/types/auto.js.map +1 -0
  13. package/lib/types/cache.d.ts.map +1 -0
  14. package/lib/types/cache.js.map +1 -0
  15. package/lib/types/caller.d.ts.map +1 -0
  16. package/lib/types/caller.js.map +1 -0
  17. package/lib/types/client-i18n.d.ts +27 -0
  18. package/lib/types/client-i18n.d.ts.map +1 -0
  19. package/lib/types/client-i18n.js +18 -0
  20. package/lib/types/client-i18n.js.map +1 -0
  21. package/lib/types/client.d.ts +15 -2
  22. package/lib/types/client.d.ts.map +1 -0
  23. package/lib/types/client.js +137 -10
  24. package/lib/types/client.js.map +1 -0
  25. package/lib/types/config.d.ts +4 -0
  26. package/lib/types/config.d.ts.map +1 -0
  27. package/lib/types/config.js +5 -3
  28. package/lib/types/config.js.map +1 -0
  29. package/lib/types/core.d.ts.map +1 -0
  30. package/lib/types/core.js.map +1 -0
  31. package/lib/types/engine.d.ts.map +1 -0
  32. package/lib/types/engine.js.map +1 -0
  33. package/lib/types/images.d.ts.map +1 -0
  34. package/lib/types/images.js.map +1 -0
  35. package/lib/types/index.d.ts +2 -0
  36. package/lib/types/index.d.ts.map +1 -0
  37. package/lib/types/index.js +162 -9
  38. package/lib/types/index.js.map +1 -0
  39. package/lib/types/plan-gate.d.ts +27 -0
  40. package/lib/types/plan-gate.d.ts.map +1 -0
  41. package/lib/types/plan-gate.js +60 -0
  42. package/lib/types/plan-gate.js.map +1 -0
  43. package/lib/types/router.d.ts +28 -32
  44. package/lib/types/router.d.ts.map +1 -0
  45. package/lib/types/router.js +41 -9
  46. package/lib/types/router.js.map +1 -0
  47. package/lib/types/session.d.ts +11 -0
  48. package/lib/types/session.d.ts.map +1 -0
  49. package/lib/types/session.js +41 -11
  50. package/lib/types/session.js.map +1 -0
  51. package/lib/types/statistics.d.ts.map +1 -0
  52. package/lib/types/statistics.js.map +1 -0
  53. package/lib/types/team-gate.d.ts +20 -0
  54. package/lib/types/team-gate.d.ts.map +1 -0
  55. package/lib/types/team-gate.js +59 -0
  56. package/lib/types/team-gate.js.map +1 -0
  57. package/lib/types/top-logprobs.d.ts.map +1 -0
  58. package/lib/types/top-logprobs.js.map +1 -0
  59. package/lib/types/topic-storage.d.ts.map +1 -0
  60. package/lib/types/topic-storage.js.map +1 -0
  61. package/package.json +114 -112
  62. package/src/auto.test.ts +95 -86
  63. package/src/auto.ts +140 -133
  64. package/src/cache.ts +115 -115
  65. package/src/caller.test.ts +34 -34
  66. package/src/caller.ts +151 -151
  67. package/src/client-i18n.ts +18 -0
  68. package/src/client.tsx +383 -213
  69. package/src/config.ts +189 -183
  70. package/src/core.test.ts +71 -71
  71. package/src/core.ts +205 -205
  72. package/src/engine.test.ts +136 -136
  73. package/src/engine.ts +140 -140
  74. package/src/images.ts +33 -33
  75. package/src/index.ts +394 -247
  76. package/src/parity.test.ts +85 -85
  77. package/src/plan-gate.test.ts +56 -0
  78. package/src/plan-gate.ts +66 -0
  79. package/src/router.test.ts +107 -89
  80. package/src/router.ts +361 -309
  81. package/src/session.test.ts +71 -36
  82. package/src/session.ts +132 -103
  83. package/src/statistics.test.ts +80 -80
  84. package/src/statistics.ts +328 -328
  85. package/src/team-gate.test.ts +92 -0
  86. package/src/team-gate.ts +80 -0
  87. package/src/top-logprobs.test.ts +84 -84
  88. package/src/top-logprobs.ts +153 -153
  89. package/src/topic-storage.test.ts +31 -31
  90. package/src/topic-storage.ts +42 -42
package/lib/index.js CHANGED
@@ -31,8 +31,10 @@ const Config = z.object({
31
31
  autoRouteMaxInputChars: z.number().step(1).min(1e3).default(6e4),
32
32
  autoMaxModelCallsPerTask: z.number().step(1).min(1).default(48),
33
33
  autoMaxModelCallsPerSession: z.number().step(1).min(1).default(160),
34
+ autoVerifyTeamTasks: z.boolean().default(true),
35
+ autoVerifyPlanMode: z.boolean().default(true),
34
36
  provider: z.string().default("deepseek-official"),
35
- model: z.string().default("deepseek-v4-flash"),
37
+ model: z.string().default("deepseek-flash"),
36
38
  reasoningEffort: z.string(),
37
39
  maxTokens: z.number().step(1).min(1).default(32768),
38
40
  timeoutMs: z.number().step(1).min(1).default(3e5),
@@ -46,7 +48,7 @@ const Config = z.object({
46
48
  });
47
49
  function resolveConfig(config = {}) {
48
50
  const provider = (config.provider ?? "deepseek-official").trim();
49
- const model = (config.model ?? "deepseek-v4-flash").trim();
51
+ const model = (config.model ?? "deepseek-flash").trim();
50
52
  if (!provider) throw new Error("llm-verifier: provider must be non-empty");
51
53
  if (!model) throw new Error("llm-verifier: model must be non-empty");
52
54
  const autoVerifyMode = config.autoVerifyMode ?? "smart";
@@ -100,6 +102,8 @@ function resolveConfig(config = {}) {
100
102
  autoRouteSemantic: config.autoRouteSemantic ?? true,
101
103
  autoRouteMinConfidence,
102
104
  autoTrackCompletionThreshold,
105
+ autoVerifyTeamTasks: config.autoVerifyTeamTasks ?? true,
106
+ autoVerifyPlanMode: config.autoVerifyPlanMode ?? true,
103
107
  provider,
104
108
  model,
105
109
  ...reasoningEffort ? { reasoningEffort } : {},
@@ -602,12 +606,33 @@ async function loadVerifierImages(inputs, signal) {
602
606
  }
603
607
  //#endregion
604
608
  //#region src/session.ts
609
+ /**
610
+ * Read one session's event log.
611
+ *
612
+ * DSH 0.1.5 replaced the `events` array with `snapshotEvents()`; older hosts
613
+ * expose the array directly. Both shapes are accepted so the plugin keeps
614
+ * working across the versions it declares support for.
615
+ * @param session - Agent session, or any object exposing one of the two shapes.
616
+ * @returns The session's events in log order, or an empty array.
617
+ */
618
+ function sessionEvents(session) {
619
+ const candidate = session;
620
+ if (typeof candidate?.snapshotEvents === "function") return candidate.snapshotEvents();
621
+ return candidate?.events ?? [];
622
+ }
605
623
  function textOf(blocks) {
606
624
  const parts = [];
607
- for (const block of blocks) if (block.type === "text") parts.push(block.text);
608
- else if (block.type === "reasoning") parts.push("[Reasoning] " + block.text);
609
- else if (block.type === "tool-call") parts.push("[Tool Call] " + block.name + " " + block.arguments);
610
- else if (block.type === "tool-result") parts.push("[Tool Result] " + textOf(block.content));
625
+ for (const block of blocks) {
626
+ const blockType = block.type;
627
+ if (blockType === "text") parts.push(block.text);
628
+ else if (blockType === "reasoning") parts.push("[Reasoning] " + block.text);
629
+ else if (blockType === "tool-call") parts.push("[Tool Call] " + block.name + " " + block.arguments);
630
+ else if (blockType === "tool-result") parts.push("[Tool Result] " + textOf(block.content));
631
+ else if (blockType === "file") {
632
+ const fileData = block;
633
+ parts.push("[File] " + (fileData.path ?? fileData.filename ?? fileData.title ?? "attachment"));
634
+ }
635
+ }
611
636
  return parts.join("\n");
612
637
  }
613
638
  const DEFAULT_REDACT_PATTERNS = ["Bearer\\s+[A-Za-z0-9._~+\\/=-]+", "(?:api[_-]?key|token|password|secret)\\s*[=:]\\s*[\"']?[^\\s,\"';}]+"];
@@ -635,7 +660,7 @@ function sanitizeVerifierText(text, maxChars, patterns = DEFAULT_REDACT_PATTERNS
635
660
  return redacted.length <= maxChars ? redacted : redacted.slice(0, maxChars) + "\n[Truncated " + (redacted.length - maxChars) + " characters]";
636
661
  }
637
662
  async function extractSession(agent, loadImage, options = {}) {
638
- const all = agent.session.events;
663
+ const all = sessionEvents(agent.session);
639
664
  const from = options.fromSeq ?? 0;
640
665
  const to = options.toSeq ?? Number.MAX_SAFE_INTEGER;
641
666
  const events = all.filter((event) => event.seq >= from && event.seq <= to);
@@ -643,24 +668,35 @@ async function extractSession(agent, loadImage, options = {}) {
643
668
  let problem = "";
644
669
  const trace = [];
645
670
  const images = [];
646
- for (const event of events) if (event.type === "user/message") {
647
- if (event.data.source.kind !== "user") continue;
648
- const text = textOf(event.data.content);
649
- if (!problem && text.trim()) problem = text.trim();
650
- for (const block of event.data.content) if (block.type === "image") images.push(await loadImage(block.attachment));
651
- trace.push("--- User seq " + event.seq + " ---\n" + text);
652
- } else if (event.type === "assistant/message" && options.includeAssistantText !== false) trace.push("--- Assistant turn " + event.data.turn + " step " + event.data.step + " ---\n" + textOf(event.data.message.content));
653
- else if (event.type === "tool/call") trace.push("--- Tool Call turn " + event.data.turn + " step " + event.data.step + " ---\n[Command] " + event.data.name + " " + event.data.arguments);
654
- else if (event.type === "tool/result") trace.push("--- Tool Result turn " + event.data.turn + " step " + event.data.step + " ---\n[Output] " + textOf(event.data.message.content));
655
- else if (event.type === "tool/code-dispatch") {
656
- const data = event.data;
657
- const status = data.isError ? " [Error]" : "";
658
- const args = data.arguments !== void 0 ? " " + (typeof data.arguments === "string" ? data.arguments : JSON.stringify(data.arguments)) : "";
659
- const content = Array.isArray(data.content) ? textOf(data.content) : "";
660
- if (Array.isArray(data.content)) {
661
- for (const block of data.content) if (block.type === "image") images.push(await loadImage(block.attachment));
671
+ for (const rawEvent of events) {
672
+ const event = rawEvent;
673
+ if (event.type === "user/message") {
674
+ const sourceKind = event.data?.source?.kind;
675
+ if (sourceKind !== "user" && sourceKind !== "team-message") continue;
676
+ const text = textOf(event.data.content);
677
+ if (!problem && sourceKind === "user" && text.trim()) problem = text.trim();
678
+ for (const block of event.data.content) if (block.type === "image") images.push(await loadImage(block.attachment));
679
+ const tag = sourceKind === "team-message" ? "Team Message" : "User";
680
+ trace.push("--- " + tag + " seq " + event.seq + " ---\n" + text);
681
+ } else if (event.type === "assistant/message" && options.includeAssistantText !== false) trace.push("--- Assistant turn " + event.data.turn + " step " + event.data.step + " ---\n" + textOf(event.data.message.content));
682
+ else if (event.type === "tool/call") trace.push("--- Tool Call turn " + event.data.turn + " step " + event.data.step + " ---\n[Command] " + event.data.name + " " + event.data.arguments);
683
+ else if (event.type === "tool/result") trace.push("--- Tool Result turn " + event.data.turn + " step " + event.data.step + " ---\n[Output] " + textOf(event.data.message.content));
684
+ else if (event.type === "tool/ptc-dispatch" || event.type === "tool/code-dispatch") {
685
+ const data = event.data;
686
+ const status = data.isError ? " [Error]" : "";
687
+ const args = data.arguments !== void 0 ? " " + (typeof data.arguments === "string" ? data.arguments : JSON.stringify(data.arguments)) : "";
688
+ const content = Array.isArray(data.content) ? textOf(data.content) : "";
689
+ if (Array.isArray(data.content)) {
690
+ for (const block of data.content) if (block.type === "image") images.push(await loadImage(block.attachment));
691
+ }
692
+ const label = event.type === "tool/ptc-dispatch" ? "PTC Dispatch " : "Code Dispatch ";
693
+ trace.push("--- " + label + data.name + status + " seq " + event.seq + " ---\n[Command] " + data.name + args + "\n[Output] " + content);
694
+ } else if (event.type === "team/message/queued") {
695
+ const data = event.data;
696
+ const sender = data?.message?.senderName ?? "Teammate";
697
+ const content = Array.isArray(data?.message?.content) ? textOf(data.message.content) : "";
698
+ trace.push("--- Team Message Queued from " + sender + " seq " + event.seq + " ---\n" + content);
662
699
  }
663
- trace.push("--- Code Dispatch " + data.name + status + " seq " + event.seq + " ---\n[Command] " + data.name + args + "\n[Output] " + content);
664
700
  }
665
701
  const raw = redactText(trace.join("\n\n"), patterns);
666
702
  const maxChars = options.maxChars ?? 2e5;
@@ -726,6 +762,8 @@ function isConsequential(name) {
726
762
  if (PASSIVE_TOOLS.has(name) || VERIFIER_TOOLS.has(name)) return false;
727
763
  return /(?:edit|write|patch|apply|deploy|upload|delete|remove|kill|exec|shell|command|migration|database|tunnel|cluster)/iu.test(name);
728
764
  }
765
+ /** Event names that carry one settled PTC/code dispatch, across the hosts the plugin supports. */
766
+ const CODE_DISPATCH_TYPES = /* @__PURE__ */ new Set(["tool/code-dispatch", "tool/ptc-dispatch"]);
729
767
  function isSuccessfulCodeDispatch(data) {
730
768
  if (data.isError === true) return false;
731
769
  if (Array.isArray(data.content) && data.content.some((b) => b.isError === true)) return false;
@@ -746,7 +784,7 @@ function analyzeAutoTask(events, policy) {
746
784
  const calls = relevant.filter((event) => event.type === "tool/call");
747
785
  const successfulResults = new Set(relevant.filter((event) => event.type === "tool/result" && event.data.error === void 0 && event.data.message.content.every((block) => block.isError !== true)).map((event) => String(event.data.message.source.callId)));
748
786
  const pairedCalls = calls.filter((event) => successfulResults.has(String(event.data.callId)));
749
- const codeDispatches = relevant.filter((event) => event.type === "tool/code-dispatch").map((event) => event.data);
787
+ const codeDispatches = relevant.filter((event) => CODE_DISPATCH_TYPES.has(event.type)).map((event) => event.data);
750
788
  const successfulDispatches = codeDispatches.filter(isSuccessfulCodeDispatch);
751
789
  const toolCalls = calls.filter((event) => !VERIFIER_TOOLS.has(event.data.name)).length + codeDispatches.filter((d) => !VERIFIER_TOOLS.has(d.name)).length;
752
790
  const completedToolResults = pairedCalls.length + successfulDispatches.length;
@@ -823,7 +861,7 @@ var AutoVerificationBudget = class {
823
861
  state.taskAttempts = 0;
824
862
  state.lastEvaluatedSeq = -1;
825
863
  }
826
- const lastSeq = agent.session.events.at(-1)?.seq ?? -1;
864
+ const lastSeq = sessionEvents(agent.session).at(-1)?.seq ?? -1;
827
865
  if (lastSeq <= state.lastEvaluatedSeq || state.taskAttempts >= policy.maxPerTask || state.sessionAttempts >= policy.maxPerSession) {
828
866
  this.states.set(id, state);
829
867
  return false;
@@ -895,21 +933,32 @@ function buildEvidenceIndex(events) {
895
933
  const calls = /* @__PURE__ */ new Map();
896
934
  const results = /* @__PURE__ */ new Map();
897
935
  const todos = /* @__PURE__ */ new Map();
936
+ const teamTasks = /* @__PURE__ */ new Map();
898
937
  const paired = /* @__PURE__ */ new Map();
899
- for (const event of relevant) if (event.type === "tool/call") calls.set(String(event.data.callId), event);
900
- else if (event.type === "tool/result" && successful(event)) results.set(String(event.data.message.source.callId), event);
901
- else if (event.type === "todo/write") todos.set(event.seq, event.data.todos);
902
- else if (event.type === "tool/code-dispatch") {
903
- const data = event.data;
904
- if (data.isError !== true && (!Array.isArray(data.content) || data.content.every((b) => b.isError !== true))) {
905
- const subCallId = String(data.subCallId ?? "code:" + event.seq);
906
- const content = Array.isArray(data.content) ? data.content : [];
907
- paired.set(subCallId, {
908
- name: data.name,
909
- callSeq: event.seq,
910
- resultSeq: event.seq,
911
- text: blockText(content)
912
- });
938
+ const currentTeamTasks = /* @__PURE__ */ new Map();
939
+ for (const rawEvent of relevant) {
940
+ const event = rawEvent;
941
+ if (event.type === "tool/call") calls.set(String(event.data.callId), rawEvent);
942
+ else if (event.type === "tool/result" && successful(rawEvent)) results.set(String(event.data.message.source.callId), rawEvent);
943
+ else if (event.type === "todo/write") todos.set(event.seq, event.data.todos);
944
+ else if (event.type === "team/task") {
945
+ const data = event.data;
946
+ if (data?.task) {
947
+ currentTeamTasks.set(data.task.id, { ...data.task });
948
+ teamTasks.set(event.seq, [...currentTeamTasks.values()]);
949
+ }
950
+ } else if (event.type === "tool/ptc-dispatch" || event.type === "tool/code-dispatch") {
951
+ const data = event.data;
952
+ if (data.isError !== true && (!Array.isArray(data.content) || data.content.every((b) => b.isError !== true))) {
953
+ const subCallId = String(data.subCallId ?? "code:" + event.seq);
954
+ const content = Array.isArray(data.content) ? data.content : [];
955
+ paired.set(subCallId, {
956
+ name: data.name,
957
+ callSeq: event.seq,
958
+ resultSeq: event.seq,
959
+ text: blockText(content)
960
+ });
961
+ }
913
962
  }
914
963
  }
915
964
  for (const [callId, call] of calls) {
@@ -924,7 +973,8 @@ function buildEvidenceIndex(events) {
924
973
  return {
925
974
  problemSeq: taskStartSeq,
926
975
  calls: paired,
927
- todos
976
+ todos,
977
+ teamTasks
928
978
  };
929
979
  }
930
980
  function parseTrustedWorkflow(value, callId, callSeq, resultSeq, maxCandidates, maxItemChars) {
@@ -978,6 +1028,23 @@ function canonicalTodoSnapshots(index) {
978
1028
  }
979
1029
  return values;
980
1030
  }
1031
+ function canonicalTeamTaskSnapshots(index) {
1032
+ const values = [];
1033
+ let previous = "";
1034
+ for (const [seq, tasks] of index.teamTasks) {
1035
+ const canonical = JSON.stringify(tasks.map((t) => ({
1036
+ id: t.id,
1037
+ status: t.status,
1038
+ revision: t.revision
1039
+ })));
1040
+ if (canonical !== previous) values.push({
1041
+ seq,
1042
+ tasks
1043
+ });
1044
+ previous = canonical;
1045
+ }
1046
+ return values;
1047
+ }
981
1048
  function analyzeStructuredRoute(events, maxCandidates = 8, maxItemChars = 2e4) {
982
1049
  const index = buildEvidenceIndex(events);
983
1050
  if (!index) return void 0;
@@ -1031,13 +1098,32 @@ function analyzeStructuredRoute(events, maxCandidates = 8, maxItemChars = 2e4) {
1031
1098
  evidenceSeqs: snapshots.map((snapshot) => snapshot.seq)
1032
1099
  };
1033
1100
  }
1101
+ const teamSnapshots = canonicalTeamTaskSnapshots(index);
1102
+ if (teamSnapshots.length >= 2) {
1103
+ const steps = teamSnapshots.map((snapshot) => sanitizeVerifierText("Team task checkpoint seq " + snapshot.seq + ":\n" + snapshot.tasks.map((task) => "- [" + task.status + "] " + task.subject + (task.description ? " (" + task.description + ")" : "")).join("\n"), maxItemChars));
1104
+ const checkpoints = steps.map((_, i) => i + 1);
1105
+ return {
1106
+ kind: "track",
1107
+ source: "structured",
1108
+ confidence: 1,
1109
+ reason: "changed durable team tasks",
1110
+ fingerprint: stableHash({
1111
+ kind: "track",
1112
+ teamSnapshots
1113
+ }),
1114
+ steps,
1115
+ checkpoints,
1116
+ evidenceSeqs: teamSnapshots.map((snapshot) => snapshot.seq)
1117
+ };
1118
+ }
1034
1119
  }
1035
1120
  }
1036
1121
  function semanticRouteHint(events) {
1037
1122
  const index = buildEvidenceIndex(events);
1038
1123
  if (!index) return false;
1039
- if ([...index.calls.values()].some((pair) => pair.name === "subagent" || pair.name === "subagent_fork" || pair.name === "workflow")) return true;
1124
+ if ([...index.calls.values()].some((pair) => pair.name === "subagent" || pair.name === "subagent_fork" || pair.name === "workflow" || pair.name === "exit_plan_mode")) return true;
1040
1125
  if (canonicalTodoSnapshots(index).length >= 2) return true;
1126
+ if (canonicalTeamTaskSnapshots(index).length >= 2) return true;
1041
1127
  return false;
1042
1128
  }
1043
1129
  function buildSemanticRoutePrompt(problem, events, maxCandidates, maxItemChars = 2e4) {
@@ -1161,7 +1247,7 @@ var AutoVerifierRouter = class {
1161
1247
  states = /* @__PURE__ */ new Map();
1162
1248
  serial = 0;
1163
1249
  state(agent) {
1164
- const taskStartSeq = latestDirectUserSeq(agent.session.events);
1250
+ const taskStartSeq = latestDirectUserSeq(sessionEvents(agent.session));
1165
1251
  if (taskStartSeq === void 0) return void 0;
1166
1252
  const id = String(agent.id);
1167
1253
  const state = this.states.get(id) ?? {
@@ -1221,6 +1307,10 @@ var AutoVerifierRouter = class {
1221
1307
  state.failed.add(reservation.fingerprint);
1222
1308
  if (strict) state.strictBlocked = true;
1223
1309
  }
1310
+ /** Whether this exact fingerprint already passed within the current task. */
1311
+ completedFingerprint(agent, fingerprint) {
1312
+ return this.state(agent)?.completed.has(fingerprint) ?? false;
1313
+ }
1224
1314
  finalRequired(agent) {
1225
1315
  return this.state(agent)?.finalRequiredFromSeq;
1226
1316
  }
@@ -1232,6 +1322,120 @@ var AutoVerifierRouter = class {
1232
1322
  }
1233
1323
  };
1234
1324
  //#endregion
1325
+ //#region src/plan-gate.ts
1326
+ /**
1327
+ * Read the plan markdown out of one `exit_plan_mode` argument object.
1328
+ * @param args - Parsed tool arguments; anything but `{ plan: string }` yields ''.
1329
+ * @returns Trimmed plan markdown, or '' when the call carries no plan.
1330
+ */
1331
+ function planFromArguments(args) {
1332
+ if (typeof args !== "object" || args === null) return "";
1333
+ const plan = args.plan;
1334
+ return typeof plan === "string" ? plan.trim() : "";
1335
+ }
1336
+ function buildPlanPreReviewPrompt(problem, planText, maxChars = 2e4) {
1337
+ return [
1338
+ "You are an expert independent technical plan verifier. A candidate implementation plan is about to be submitted to the human for approval.",
1339
+ "Evaluate whether the plan is sound, executable, and comprehensive.",
1340
+ "Scoring criteria:",
1341
+ "A: Flawless, thorough, with clear steps, rigorous verification, edge cases handled.",
1342
+ "B-J: Mostly sound, minor gaps or nuances missing.",
1343
+ "K-S: Noticeable architectural gaps, missing regression checks, or ambiguities.",
1344
+ "T: Fundamentally flawed, dangerous, or missing core requirements.",
1345
+ "",
1346
+ "Task requirement:",
1347
+ sanitizeVerifierText(problem, 4e3),
1348
+ "",
1349
+ "Proposed Plan:",
1350
+ sanitizeVerifierText(planText, maxChars),
1351
+ "",
1352
+ "Output format:",
1353
+ "The first line must be exactly \"Verdict: <single uppercase letter A-T>\" — one letter, nothing else on the line.",
1354
+ "Line 2: Summary: <One sentence assessment>",
1355
+ "Line 3+: Key strengths, blind spots, and verification guidance."
1356
+ ].join("\n\n");
1357
+ }
1358
+ /**
1359
+ * Parse the `Verdict: <A-T>` line the judge prompts require.
1360
+ *
1361
+ * Markdown decoration around the required line is stripped first, so "**Verdict:
1362
+ * A**", "- Verdict: B." or a numbered list marker still produce a grade — a
1363
+ * formatting habit must never silently switch the gate off. Prose that is not a
1364
+ * grade is still rejected: "Verdict: Failed" and "Verdict: Approved" carry no
1365
+ * verdict letter, and an answer without a verdict line has no score at all —
1366
+ * callers must skip such a review, never treat it as a pass. A is 1.0 and T is
1367
+ * 0.0, matching the top-logprob A-T scale used by the judge prompts.
1368
+ * @param text - Raw judge model output.
1369
+ * @returns The parsed verdict, or undefined when no verdict line is present.
1370
+ */
1371
+ function parseVerdictLetter(text) {
1372
+ for (const rawLine of text.split(/\r?\n/)) {
1373
+ const line = rawLine.replace(/[*_`~#>]/g, " ").replace(/^\s*(?:[-+]|\d+[.)])\s+/, " ").trim();
1374
+ const match = /^verdict\s*:\s*([A-Ta-t])\s*[.)]?$/i.exec(line);
1375
+ if (match === null) continue;
1376
+ const verdict = match[1].toUpperCase();
1377
+ return {
1378
+ verdict,
1379
+ score: Math.max(0, Math.min(1, 1 - (verdict.charCodeAt(0) - 65) / 19)),
1380
+ feedback: text.trim()
1381
+ };
1382
+ }
1383
+ }
1384
+ //#endregion
1385
+ //#region src/team-gate.ts
1386
+ function inspectTeamTasks(events, fromSeq = 0) {
1387
+ const taskMap = /* @__PURE__ */ new Map();
1388
+ const completionSeq = /* @__PURE__ */ new Map();
1389
+ for (const rawEvent of events) {
1390
+ const event = rawEvent;
1391
+ if (event.type === "team/task") {
1392
+ const data = event.data;
1393
+ if (data?.task) {
1394
+ taskMap.set(data.task.id, { ...data.task });
1395
+ if (data.task.status === "completed") completionSeq.set(data.task.id, event.seq);
1396
+ else completionSeq.delete(data.task.id);
1397
+ }
1398
+ }
1399
+ }
1400
+ const completedTasks = [...completionSeq.entries()].filter(([id, seq]) => seq >= fromSeq && taskMap.get(id)?.status === "completed").map(([id, seq]) => ({
1401
+ task: { ...taskMap.get(id) },
1402
+ seq
1403
+ })).sort((left, right) => left.seq - right.seq);
1404
+ const latest = completedTasks.at(-1);
1405
+ return {
1406
+ latestCompletedTask: latest?.task,
1407
+ completedSeq: latest?.seq,
1408
+ completedTasks,
1409
+ activeTasks: [...taskMap.values()].filter((t) => t.status !== "deleted"),
1410
+ hasRecentCompletedTask: latest !== void 0
1411
+ };
1412
+ }
1413
+ function buildTeamTaskVerificationPrompt(task, executionTrace, maxChars = 2e4) {
1414
+ return [
1415
+ "You are an expert independent technical verifier reviewing a completed Agent Teams task.",
1416
+ "Verify if the task goal has been satisfied by concrete evidence (code edits, test runs, successful commands).",
1417
+ "Task Details:",
1418
+ "ID: " + task.id,
1419
+ "Subject: " + task.subject,
1420
+ task.description ? "Description: " + task.description : "",
1421
+ "",
1422
+ "Execution Evidence & Trace:",
1423
+ sanitizeVerifierText(executionTrace, maxChars),
1424
+ "",
1425
+ "Evaluate whether the task is genuinely completed with verified proof.",
1426
+ "Scoring criteria:",
1427
+ "A: Completely fulfilled with solid execution/test evidence.",
1428
+ "B-J: Mostly complete, minor gaps or edge cases not fully asserted.",
1429
+ "K-S: Incomplete, missing key requirements or untested.",
1430
+ "T: Unfulfilled, failed, or contradictory evidence.",
1431
+ "",
1432
+ "Output format:",
1433
+ "The first line must be exactly \"Verdict: <single uppercase letter A-T>\" — one letter, nothing else on the line.",
1434
+ "Line 2: Summary: <One sentence evaluation of task fulfillment>",
1435
+ "Line 3+: Remaining gaps or next steps for the team."
1436
+ ].filter(Boolean).join("\n\n");
1437
+ }
1438
+ //#endregion
1235
1439
  //#region src/statistics.ts
1236
1440
  const VERIFIER_TOOL_NAMES = [
1237
1441
  "verifier_route_classify",
@@ -2010,8 +2214,7 @@ function apply(ctx, config = {}) {
2010
2214
  summary: "Automatic verifier routed " + decision.kind
2011
2215
  }
2012
2216
  });
2013
- services.connection.rpc.handle("/llm-verifier", async (endpoint, payload) => {
2014
- if (endpoint !== "statistics") return rpcFailure("unknown llm-verifier endpoint");
2217
+ const handleStatisticsQuery = async (payload) => {
2015
2218
  if (typeof payload !== "object" || payload === null) return rpcFailure("statistics payload must be an object");
2016
2219
  const row = payload;
2017
2220
  const sessionId = typeof row.sessionId === "string" && row.sessionId.length > 0 ? row.sessionId : void 0;
@@ -2028,25 +2231,171 @@ function apply(ctx, config = {}) {
2028
2231
  } catch (error) {
2029
2232
  return rpcFailure(error instanceof Error ? error.message : String(error));
2030
2233
  }
2031
- }, { authority: "loopback" });
2234
+ };
2235
+ const anyConn = services.connection;
2236
+ if (anyConn && anyConn.fetch && typeof anyConn.fetch.register === "function") try {
2237
+ anyConn.fetch.register({
2238
+ path: "/api/llm-verifier/statistics",
2239
+ methods: ["POST"],
2240
+ requestBody: "buffered",
2241
+ fetch: async (request) => {
2242
+ let body;
2243
+ try {
2244
+ body = await request.json();
2245
+ } catch {
2246
+ return new Response("body is not JSON", { status: 400 });
2247
+ }
2248
+ const isRpcEnvelope = typeof body === "object" && body !== null && body.type === "client-request" && typeof body.rpcId === "string";
2249
+ const rpcId = isRpcEnvelope ? body.rpcId : "direct";
2250
+ const payload = isRpcEnvelope ? body.payload : body;
2251
+ const outcome = await handleStatisticsQuery(payload);
2252
+ if (isRpcEnvelope) return Response.json({
2253
+ type: "server-response",
2254
+ rpcId,
2255
+ result: outcome
2256
+ });
2257
+ return Response.json(outcome);
2258
+ }
2259
+ });
2260
+ } catch (e) {
2261
+ ctx.logger.warn("failed to register /api/llm-verifier/statistics fetch route: " + String(e));
2262
+ }
2263
+ const legacyRpc = services.connection;
2264
+ if (typeof legacyRpc.rpc?.handle === "function") try {
2265
+ legacyRpc.rpc.handle("/llm-verifier", async (endpoint, payload) => {
2266
+ if (endpoint !== "statistics") return rpcFailure("unknown llm-verifier endpoint");
2267
+ return handleStatisticsQuery(payload);
2268
+ });
2269
+ } catch (e) {
2270
+ ctx.logger.warn("failed to register /llm-verifier rpc fallback: " + String(e));
2271
+ }
2032
2272
  ctx.on("agent/disposed", ({ agent }) => {
2033
2273
  autoBudget.release(agent);
2034
2274
  autoRouter.release(agent);
2035
2275
  topics.delete(String(agent.id));
2036
2276
  });
2277
+ ctx.on("tools/pre-execute", async (exec, next) => {
2278
+ const selected = current();
2279
+ if (!selected.enabled || selected.autoVerifyMode === "manual" || !selected.autoVerifyPlanMode) return next();
2280
+ if (exec.name !== "exit_plan_mode" || exec.agent === void 0 || exec.signal.aborted) return next();
2281
+ const plan = planFromArguments(exec.arguments);
2282
+ if (!plan) return next();
2283
+ const agent = exec.agent;
2284
+ const policy = routePolicy(selected);
2285
+ const evidence = analyzeAutoTask(sessionEvents(agent.session), {
2286
+ mode: selected.autoVerifyMode,
2287
+ minToolCalls: selected.autoVerifyMinToolCalls,
2288
+ maxPerTask: selected.autoVerifyMaxPerTask,
2289
+ maxPerSession: selected.autoVerifyMaxPerSession
2290
+ });
2291
+ const planFingerprint = stableHash({
2292
+ phase: "plan_review",
2293
+ plan
2294
+ });
2295
+ const reservation = autoRouter.reserve(agent, "plan_review", planFingerprint, 1, policy);
2296
+ if (reservation === void 0) {
2297
+ if (!autoRouter.completedFingerprint(agent, planFingerprint)) ctx.logger.warn("llm-verifier plan pre-review skipped (verification in flight or auto-verification budget exhausted)");
2298
+ return next();
2299
+ }
2300
+ try {
2301
+ const toSeq = sessionEvents(agent.session).at(-1)?.seq ?? -1;
2302
+ const extracted = await extractTask(agent, evidence.taskStartSeq, toSeq, selected.autoVerifyMaxChars, exec.signal);
2303
+ const verdict = parseVerdictLetter((await classifyRoute(agent, buildPlanPreReviewPrompt(extracted.problem, plan, selected.autoRouteMaxInputChars), exec.signal)).text);
2304
+ if (verdict === void 0) {
2305
+ autoRouter.fail(agent, reservation, false);
2306
+ ctx.logger.warn("llm-verifier plan pre-review produced no verdict line; the plan was allowed through");
2307
+ return next();
2308
+ }
2309
+ if (verdict.score >= selected.autoVerifyThreshold) {
2310
+ autoRouter.commit(agent, reservation);
2311
+ return next();
2312
+ }
2313
+ autoRouter.fail(agent, reservation, selected.autoVerifyMode === "strict");
2314
+ return {
2315
+ kind: "deny",
2316
+ reason: "[Automatic Verifier Plan Pre-review] scored " + (verdict.score * 100).toFixed(1) + "% against a " + (selected.autoVerifyThreshold * 100).toFixed(0) + "% threshold.\n" + verdict.feedback.slice(0, 4e3) + "\nRevise the plan to address these findings, then call exit_plan_mode again."
2317
+ };
2318
+ } catch (error) {
2319
+ autoRouter.fail(agent, reservation, false);
2320
+ ctx.logger.warn("llm-verifier plan pre-review failed: " + (error instanceof Error ? error.message : String(error)));
2321
+ return next();
2322
+ }
2323
+ });
2037
2324
  ctx.on("agent/turn-stopping", async ({ agent, signal }) => {
2038
2325
  const selected = current();
2039
2326
  if (!selected.enabled || selected.autoVerifyMode === "manual" || signal.aborted) return;
2040
2327
  const policy = routePolicy(selected);
2041
- const evidence = analyzeAutoTask(agent.session.events, {
2328
+ const evidence = analyzeAutoTask(sessionEvents(agent.session), {
2042
2329
  mode: selected.autoVerifyMode,
2043
2330
  minToolCalls: selected.autoVerifyMinToolCalls,
2044
2331
  maxPerTask: selected.autoVerifyMaxPerTask,
2045
2332
  maxPerSession: selected.autoVerifyMaxPerSession
2046
2333
  });
2047
- const admittedLastSeq = agent.session.events.at(-1)?.seq ?? -1;
2048
- const snapshot = agent.session.events.filter((event) => event.seq <= admittedLastSeq);
2049
- const stillCurrent = () => !signal.aborted && (agent.session.events.at(-1)?.seq ?? -1) === admittedLastSeq;
2334
+ const admittedLastSeq = sessionEvents(agent.session).at(-1)?.seq ?? -1;
2335
+ const snapshot = sessionEvents(agent.session).filter((event) => event.seq <= admittedLastSeq);
2336
+ const stillCurrent = () => !signal.aborted && (sessionEvents(agent.session).at(-1)?.seq ?? -1) === admittedLastSeq;
2337
+ if (selected.autoVerifyTeamTasks) {
2338
+ const teamInspection = inspectTeamTasks(snapshot, evidence.taskStartSeq);
2339
+ for (const completed of teamInspection.completedTasks) {
2340
+ const task = completed.task;
2341
+ const taskReservation = autoRouter.reserve(agent, "team_task", stableHash({
2342
+ phase: "team_task",
2343
+ taskId: task.id,
2344
+ status: task.status,
2345
+ seq: completed.seq
2346
+ }), 1, policy);
2347
+ if (taskReservation === void 0) continue;
2348
+ try {
2349
+ const prompt = buildTeamTaskVerificationPrompt(task, (await extractTask(agent, evidence.taskStartSeq, admittedLastSeq, selected.autoVerifyMaxChars, signal)).trace, selected.autoRouteMaxInputChars);
2350
+ const classified = await classifyRoute(agent, prompt, signal);
2351
+ if (!stillCurrent()) {
2352
+ autoRouter.fail(agent, taskReservation, false);
2353
+ return;
2354
+ }
2355
+ const verdict = parseVerdictLetter(classified.text);
2356
+ if (verdict === void 0) {
2357
+ autoRouter.fail(agent, taskReservation, false);
2358
+ ctx.logger.warn("llm-verifier team task verification produced no verdict line; task " + task.id + " was not gated");
2359
+ continue;
2360
+ }
2361
+ if (verdict.score >= selected.autoVerifyThreshold) {
2362
+ autoRouter.commit(agent, taskReservation, admittedLastSeq);
2363
+ continue;
2364
+ }
2365
+ autoRouter.fail(agent, taskReservation, selected.autoVerifyMode === "strict");
2366
+ agent.steer(createUserMessage({
2367
+ content: [{
2368
+ type: "text",
2369
+ text: "[Automatic Verifier Team Task Gate]\nTask \"" + task.subject + "\" (#" + task.id + ") verification scored " + (verdict.score * 100).toFixed(1) + "% (Threshold: " + (selected.autoVerifyThreshold * 100).toFixed(0) + "%).\n" + verdict.feedback + "\nProvide verified execution evidence or resolve remaining issues before completing the task."
2370
+ }],
2371
+ source: {
2372
+ kind: "plugin",
2373
+ plugin: "dsh-llm-verifier",
2374
+ form: "notice",
2375
+ summary: "Team Task Gate Feedback"
2376
+ }
2377
+ }));
2378
+ return;
2379
+ } catch (error) {
2380
+ autoRouter.fail(agent, taskReservation, selected.autoVerifyMode === "strict");
2381
+ ctx.logger.warn("llm-verifier team task verification failed: " + (error instanceof Error ? error.message : String(error)));
2382
+ if (selected.autoVerifyMode === "strict" && !signal.aborted) {
2383
+ agent.steer(createUserMessage({
2384
+ content: [{
2385
+ type: "text",
2386
+ text: "[Automatic Verifier Team Task Gate]\nTask verification failed: " + (error instanceof Error ? error.message : String(error))
2387
+ }],
2388
+ source: {
2389
+ kind: "plugin",
2390
+ plugin: "dsh-llm-verifier"
2391
+ }
2392
+ }));
2393
+ return;
2394
+ }
2395
+ continue;
2396
+ }
2397
+ }
2398
+ }
2050
2399
  let decision = boundDecision(analyzeStructuredRoute(snapshot, selected.autoRouteMaxCandidates, selected.autoRouteMaxItemChars), policy);
2051
2400
  if (decision === void 0 && selected.autoRouteSemantic && (selected.autoVerifyMode === "strict" || semanticRouteHint(snapshot))) {
2052
2401
  const fingerprint = stableHash({
@@ -2551,6 +2900,6 @@ function apply(ctx, config = {}) {
2551
2900
  }));
2552
2901
  }
2553
2902
  //#endregion
2554
- export { AutoVerificationBudget, AutoVerifierRouter, Config, DEFAULT_CRITERIA, DEFAULT_GROUND_TRUTH_NOTE, GRANULARITY, LETTERS, RequestLimiter, SCALE_DESCRIPTION, ScoreCache, SingleFlight, StatisticsStore, VERIFIER_TOOL_NAMES, VerifierEngine, accumulatePairs, analyzeAutoTask, analyzeStructuredRoute, apply, automaticFeedback, boundDecision, bradleyTerry, buildEvidenceIndex, buildPairwisePrompt, buildProgressPrompt, buildSemanticRoutePrompt, callVerifier, emptyRunStats, errorDetails, extractProgressScore, extractScore, inject, latestDirectUserSeq, mergeStatisticsOverviews, name, normalizeCriteria, normalizeScoreLetter, parseSemanticRoute, pivotRoundPairs, rankScores, resolveCacheFile, resolveStatisticsFile, resolveTopicDataDir, ringCycle, seededRandom, semanticDecision, semanticRouteHint, stableHash, topPivots };
2903
+ export { AutoVerificationBudget, AutoVerifierRouter, Config, DEFAULT_CRITERIA, DEFAULT_GROUND_TRUTH_NOTE, GRANULARITY, LETTERS, RequestLimiter, SCALE_DESCRIPTION, ScoreCache, SingleFlight, StatisticsStore, VERIFIER_TOOL_NAMES, VerifierEngine, accumulatePairs, analyzeAutoTask, analyzeStructuredRoute, apply, automaticFeedback, boundDecision, bradleyTerry, buildEvidenceIndex, buildPairwisePrompt, buildPlanPreReviewPrompt, buildProgressPrompt, buildSemanticRoutePrompt, buildTeamTaskVerificationPrompt, callVerifier, emptyRunStats, errorDetails, extractProgressScore, extractScore, inject, inspectTeamTasks, latestDirectUserSeq, mergeStatisticsOverviews, name, normalizeCriteria, normalizeScoreLetter, parseSemanticRoute, parseVerdictLetter, pivotRoundPairs, planFromArguments, rankScores, resolveCacheFile, resolveStatisticsFile, resolveTopicDataDir, ringCycle, seededRandom, semanticDecision, semanticRouteHint, stableHash, topPivots };
2555
2904
 
2556
2905
  //# sourceMappingURL=index.js.map