jeopi-cli 16.2.29 → 16.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/dist/cli.js +2618 -2553
- package/dist/types/config/settings-schema.d.ts +12 -2
- package/dist/types/goals/evidence-gate.d.ts +62 -0
- package/dist/types/goals/index.d.ts +1 -0
- package/dist/types/goals/runtime.d.ts +29 -0
- package/dist/types/goals/verifier.d.ts +9 -0
- package/dist/types/session/agent-session.d.ts +22 -7
- package/package.json +12 -12
- package/src/autoresearch/index.ts +60 -6
- package/src/config/settings-schema.ts +14 -2
- package/src/goals/evidence-gate.ts +137 -0
- package/src/goals/index.ts +1 -0
- package/src/goals/runtime.ts +86 -0
- package/src/goals/verifier.ts +100 -0
- package/src/prompts/agents/goal-verifier.md +51 -0
- package/src/prompts/goals/goal-continuation.md +1 -0
- package/src/prompts/tools/goal.md +1 -1
- package/src/sdk.ts +2 -0
- package/src/session/agent-session.ts +84 -8
- package/src/task/agents.ts +2 -0
package/src/goals/runtime.ts
CHANGED
|
@@ -3,8 +3,16 @@ import goalBudgetLimitPrompt from "../prompts/goals/goal-budget-limit.md" with {
|
|
|
3
3
|
import goalContinuationPrompt from "../prompts/goals/goal-continuation.md" with { type: "text" };
|
|
4
4
|
import goalModeActivePrompt from "../prompts/goals/goal-mode-active.md" with { type: "text" };
|
|
5
5
|
import { assertSubstantiveCompletionEvidence } from "./completion-evidence";
|
|
6
|
+
import { GoalEvidenceTracker, isVerificationSignal, MUTATION_TOOL_NAMES } from "./evidence-gate";
|
|
6
7
|
import type { Goal, GoalBudgetSteering, GoalModeState, GoalRuntimeEvent, GoalTokenUsage } from "./state";
|
|
7
8
|
|
|
9
|
+
export interface GoalCompletionVerdict {
|
|
10
|
+
verdict: "okay" | "iterate" | "reject";
|
|
11
|
+
justification: string;
|
|
12
|
+
summary?: string;
|
|
13
|
+
requiredFixes: string[];
|
|
14
|
+
}
|
|
15
|
+
|
|
8
16
|
export interface GoalRuntimeHost {
|
|
9
17
|
getState(): GoalModeState | undefined;
|
|
10
18
|
setState(state: GoalModeState | undefined): void;
|
|
@@ -16,6 +24,16 @@ export interface GoalRuntimeHost {
|
|
|
16
24
|
content: string;
|
|
17
25
|
deliverAs?: "steer" | "followUp" | "nextTurn";
|
|
18
26
|
}): Promise<void>;
|
|
27
|
+
/**
|
|
28
|
+
* Spawn an independent grader sub-agent to verdict a completion claim against
|
|
29
|
+
* the objective and the actual repo diff (article gap: "the agent that wrote
|
|
30
|
+
* the code is not the agent that grades it"). The sub-agent sees only the
|
|
31
|
+
* objective + evidence text passed here plus its own repo inspection — never
|
|
32
|
+
* this session's transcript/reasoning. Optional: hosts that don't wire this
|
|
33
|
+
* (e.g. test harnesses, `goal.verifyCompletion` disabled) skip independent
|
|
34
|
+
* grading and fall back to the deterministic evidence gate alone.
|
|
35
|
+
*/
|
|
36
|
+
verifyCompletion?(objective: string, evidence: string): Promise<GoalCompletionVerdict>;
|
|
19
37
|
now?(): number;
|
|
20
38
|
}
|
|
21
39
|
|
|
@@ -117,9 +135,12 @@ function isAccountingStatus(goal: Goal): boolean {
|
|
|
117
135
|
|
|
118
136
|
export class GoalRuntime {
|
|
119
137
|
readonly #host: GoalRuntimeHost;
|
|
138
|
+
readonly #evidence = new GoalEvidenceTracker();
|
|
120
139
|
#turnSnapshot: GoalTurnSnapshot | undefined;
|
|
121
140
|
#wallClock: GoalWallClockSnapshot;
|
|
122
141
|
#budgetReportedFor: string | undefined;
|
|
142
|
+
/** Set when an independent-verifier non-okay verdict was bounced once; cleared by any new mutation/verification or a fresh goal. Mirrors GoalEvidenceTracker's bypass latch, kept separate because it gates a different (probabilistic) signal. */
|
|
143
|
+
#verifierBypassArmed = false;
|
|
123
144
|
#accountingTail: Promise<void> = Promise.resolve();
|
|
124
145
|
|
|
125
146
|
constructor(host: GoalRuntimeHost) {
|
|
@@ -202,6 +223,30 @@ export class GoalRuntime {
|
|
|
202
223
|
this.#budgetReportedFor = undefined;
|
|
203
224
|
}
|
|
204
225
|
|
|
226
|
+
/** Deterministic completion-evidence tracker for the active goal (see evidence-gate.ts). */
|
|
227
|
+
get evidence(): GoalEvidenceTracker {
|
|
228
|
+
return this.#evidence;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/**
|
|
232
|
+
* Record a successful tool execution as completion evidence for the active goal.
|
|
233
|
+
* Mutation tools (edit/write/ast_edit) arm the completion gate; bash/eval runs
|
|
234
|
+
* whose command or output head carries a verification signal (test/build/
|
|
235
|
+
* typecheck/lint) satisfy it. No-op when no goal is accounting.
|
|
236
|
+
*/
|
|
237
|
+
recordToolEvidence(toolName: string, detail?: { command?: string; outputHead?: string }): void {
|
|
238
|
+
if (!this.#hasAccountingState()) return;
|
|
239
|
+
if (MUTATION_TOOL_NAMES[toolName]) {
|
|
240
|
+
this.#evidence.recordMutation();
|
|
241
|
+
this.#verifierBypassArmed = false;
|
|
242
|
+
return;
|
|
243
|
+
}
|
|
244
|
+
if (detail && isVerificationSignal(detail.command ?? "", detail.outputHead ?? "")) {
|
|
245
|
+
this.#evidence.recordVerification();
|
|
246
|
+
this.#verifierBypassArmed = false;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
|
|
205
250
|
onTurnStart(turnId: string, baselineUsage: GoalTokenUsage): void {
|
|
206
251
|
this.#turnSnapshot = { turnId, baselineUsage: { ...baselineUsage } };
|
|
207
252
|
const state = this.#host.getState();
|
|
@@ -393,6 +438,8 @@ export class GoalRuntime {
|
|
|
393
438
|
}
|
|
394
439
|
const state = this.#createGoalState(objective, input.tokenBudget);
|
|
395
440
|
this.#budgetReportedFor = undefined;
|
|
441
|
+
this.#evidence.reset();
|
|
442
|
+
this.#verifierBypassArmed = false;
|
|
396
443
|
this.#markActiveAccounting(state.goal);
|
|
397
444
|
await this.#commitState(state, { persist: "goal" });
|
|
398
445
|
return state;
|
|
@@ -411,6 +458,8 @@ export class GoalRuntime {
|
|
|
411
458
|
await this.#flushUsageLocked("suppressed");
|
|
412
459
|
const state = this.#createGoalState(objective, input.tokenBudget);
|
|
413
460
|
this.#budgetReportedFor = undefined;
|
|
461
|
+
this.#evidence.reset();
|
|
462
|
+
this.#verifierBypassArmed = false;
|
|
414
463
|
this.#markActiveAccounting(state.goal);
|
|
415
464
|
await this.#commitState(state, { persist: "goal" });
|
|
416
465
|
return state;
|
|
@@ -487,6 +536,43 @@ export class GoalRuntime {
|
|
|
487
536
|
if (state.goal.status === "dropped") {
|
|
488
537
|
throw new Error("cannot complete a dropped goal");
|
|
489
538
|
}
|
|
539
|
+
// Deterministic evidence gate (evidence-gate.ts): a goal session that
|
|
540
|
+
// mutated project files cannot self-report complete unless a verification
|
|
541
|
+
// signal was observed after the last mutation. First violation bounces
|
|
542
|
+
// once with a corrective message; a repeat complete with no new mutation
|
|
543
|
+
// passes (escape hatch for docs/config-only work).
|
|
544
|
+
const gate = this.#evidence.classifyCompletion();
|
|
545
|
+
if (gate.block) {
|
|
546
|
+
throw new Error(gate.message);
|
|
547
|
+
}
|
|
548
|
+
// Independent grader (goal-verifier sub-agent, see GoalRuntimeHost.verifyCompletion):
|
|
549
|
+
// maker != grader, per the compounding-loop primitive — the same agent that did
|
|
550
|
+
// the work does not get to be the sole judge of whether it's done. Same bounce-once
|
|
551
|
+
// shape as the deterministic gate above, kept as a separate latch since this signal
|
|
552
|
+
// is probabilistic (an LLM verdict), not mechanical.
|
|
553
|
+
if (this.#host.verifyCompletion) {
|
|
554
|
+
const verdict = await this.#host.verifyCompletion(state.goal.objective, verifiedEvidence);
|
|
555
|
+
if (verdict.verdict !== "okay") {
|
|
556
|
+
if (this.#verifierBypassArmed) {
|
|
557
|
+
// Second attempt with no new mutation/verification since the bounce:
|
|
558
|
+
// escape hatch for a verifier false-positive-reject/iterate.
|
|
559
|
+
this.#verifierBypassArmed = false;
|
|
560
|
+
} else {
|
|
561
|
+
this.#verifierBypassArmed = true;
|
|
562
|
+
const fixes =
|
|
563
|
+
verdict.requiredFixes.length > 0
|
|
564
|
+
? `\nRequired fixes:\n${verdict.requiredFixes.map(fix => `- ${fix}`).join("\n")}`
|
|
565
|
+
: "";
|
|
566
|
+
throw new Error(
|
|
567
|
+
`Completion blocked: independent verifier verdict was "${verdict.verdict}", not "okay". ` +
|
|
568
|
+
`${verdict.justification}${fixes}\nAddress the gaps above, then call complete again. ` +
|
|
569
|
+
"If the verifier is wrong, call complete again unchanged to override.",
|
|
570
|
+
);
|
|
571
|
+
}
|
|
572
|
+
} else {
|
|
573
|
+
this.#verifierBypassArmed = false;
|
|
574
|
+
}
|
|
575
|
+
}
|
|
490
576
|
state.enabled = false;
|
|
491
577
|
state.goal.status = "complete";
|
|
492
578
|
state.goal.completionEvidence = verifiedEvidence;
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Independent-verifier bridge for goal-mode completion (see GoalRuntimeHost.verifyCompletion
|
|
3
|
+
* in runtime.ts). Spawns the bundled `goal-verifier` sub-agent — a fresh session with its own
|
|
4
|
+
* kernel, its own read-only tool set, and no exposure to the maker's transcript/reasoning —
|
|
5
|
+
* to grade a completion claim against the objective and the actual repo diff. Reuses the eval
|
|
6
|
+
* tool's `agent()` bridge as a plain async call (no eval kernel required outside eval cells;
|
|
7
|
+
* `withBridgeTimeoutPause` in agent-bridge.ts is a no-op when no `emitStatus` sink is wired).
|
|
8
|
+
*/
|
|
9
|
+
import { runEvalAgent } from "../eval/agent-bridge";
|
|
10
|
+
import { getBundledAgent } from "../task/agents";
|
|
11
|
+
import type { ToolSession } from "../tools";
|
|
12
|
+
import type { GoalCompletionVerdict } from "./runtime";
|
|
13
|
+
|
|
14
|
+
const VERIFIER_AGENT_NAME = "goal-verifier";
|
|
15
|
+
|
|
16
|
+
/** Cap the evidence claim embedded in the verifier prompt; the verifier re-derives ground truth from the repo, not from a long claim. */
|
|
17
|
+
const MAX_EVIDENCE_CHARS = 4000;
|
|
18
|
+
|
|
19
|
+
function parseVerdict(output: string): GoalCompletionVerdict {
|
|
20
|
+
let data: unknown;
|
|
21
|
+
try {
|
|
22
|
+
data = JSON.parse(output);
|
|
23
|
+
} catch {
|
|
24
|
+
// Verifier didn't return a structured payload (refused, crashed, plain-text
|
|
25
|
+
// reply). Fail open to "iterate" rather than silently trusting completion —
|
|
26
|
+
// a broken verifier bounces once instead of being bypassed unnoticed.
|
|
27
|
+
return {
|
|
28
|
+
verdict: "iterate",
|
|
29
|
+
justification: `goal-verifier did not return a structured verdict. Raw output: ${output.slice(0, 500)}`,
|
|
30
|
+
requiredFixes: [],
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
if (data === null || typeof data !== "object") {
|
|
34
|
+
return {
|
|
35
|
+
verdict: "iterate",
|
|
36
|
+
justification: "goal-verifier returned a non-object payload.",
|
|
37
|
+
requiredFixes: [],
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
const record = data as Record<string, unknown>;
|
|
41
|
+
const verdict = record.verdict;
|
|
42
|
+
if (verdict !== "okay" && verdict !== "iterate" && verdict !== "reject") {
|
|
43
|
+
return {
|
|
44
|
+
verdict: "iterate",
|
|
45
|
+
justification: "goal-verifier returned an unrecognized verdict value.",
|
|
46
|
+
requiredFixes: [],
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
const rawFixes = record.required_fixes;
|
|
50
|
+
const requiredFixes = Array.isArray(rawFixes)
|
|
51
|
+
? rawFixes.filter((fix): fix is string => typeof fix === "string")
|
|
52
|
+
: [];
|
|
53
|
+
const justification = typeof record.justification === "string" ? record.justification : "";
|
|
54
|
+
const summary = typeof record.summary === "string" ? record.summary : undefined;
|
|
55
|
+
return { verdict, justification, summary, requiredFixes };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Grade a goal-completion claim with the independent `goal-verifier` sub-agent.
|
|
60
|
+
* Never throws: a spawn failure (depth limit, disabled agent, no model) fails
|
|
61
|
+
* open to `iterate` so it costs one corrective bounce, the same as a genuine
|
|
62
|
+
* gap — never a silent skip of independent grading, never a permanent block.
|
|
63
|
+
*/
|
|
64
|
+
export async function verifyGoalCompletion(
|
|
65
|
+
session: ToolSession,
|
|
66
|
+
objective: string,
|
|
67
|
+
evidence: string,
|
|
68
|
+
): Promise<GoalCompletionVerdict> {
|
|
69
|
+
const truncatedEvidence =
|
|
70
|
+
evidence.length > MAX_EVIDENCE_CHARS ? `${evidence.slice(0, MAX_EVIDENCE_CHARS)}\n[truncated]` : evidence;
|
|
71
|
+
const verifierPrompt =
|
|
72
|
+
`<objective>\n${objective}\n</objective>\n\n` +
|
|
73
|
+
`<evidence>\n${truncatedEvidence}\n</evidence>\n\n` +
|
|
74
|
+
"Verdict whether the objective above is genuinely satisfied by the repo's current state. " +
|
|
75
|
+
"The evidence is the maker's claim, not a fact — verify it independently.";
|
|
76
|
+
try {
|
|
77
|
+
// Explicit schema, not the agent's own frontmatter default: runEvalAgent (the
|
|
78
|
+
// eval `agent()` bridge) only applies `outputSchema` when the caller supplies
|
|
79
|
+
// one — unlike the `task` tool, it never falls back to `effectiveAgent.output`
|
|
80
|
+
// (task/index.ts). Read the bundled definition's own `output` so this can't
|
|
81
|
+
// drift from goal-verifier.md's declared shape. Omit the key entirely when
|
|
82
|
+
// undefined: `Object.hasOwn` (agent-bridge.ts's `structured` check) is true
|
|
83
|
+
// for an explicit `schema: undefined`, which would wrongly disable the
|
|
84
|
+
// agent's frontmatter-schema fallback in the bridge.
|
|
85
|
+
const schema = getBundledAgent(VERIFIER_AGENT_NAME)?.output;
|
|
86
|
+
const args =
|
|
87
|
+
schema === undefined
|
|
88
|
+
? { prompt: verifierPrompt, agent: VERIFIER_AGENT_NAME }
|
|
89
|
+
: { prompt: verifierPrompt, agent: VERIFIER_AGENT_NAME, schema };
|
|
90
|
+
const result = await runEvalAgent(args, { session });
|
|
91
|
+
return parseVerdict(result.text);
|
|
92
|
+
} catch (err) {
|
|
93
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
94
|
+
return {
|
|
95
|
+
verdict: "iterate",
|
|
96
|
+
justification: `Independent verifier could not run: ${message}`,
|
|
97
|
+
requiredFixes: [],
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: goal-verifier
|
|
3
|
+
description: "Read-only independent grader for goal-mode completion. Verdicts whether the actual diff satisfies the stated objective and evidence claim — sees only the objective, evidence, and repo state, never the maker's reasoning transcript."
|
|
4
|
+
tools: read, grep, glob, bash, ast_grep
|
|
5
|
+
spawns: explore
|
|
6
|
+
model: pi/slow
|
|
7
|
+
thinking-level: high
|
|
8
|
+
output:
|
|
9
|
+
properties:
|
|
10
|
+
verdict:
|
|
11
|
+
metadata:
|
|
12
|
+
description: Completion verdict for the goal under review
|
|
13
|
+
enum: [okay, iterate, reject]
|
|
14
|
+
justification:
|
|
15
|
+
metadata:
|
|
16
|
+
description: Evidence-backed reasoning for the verdict, citing inspected files/diff hunks/command output
|
|
17
|
+
type: string
|
|
18
|
+
summary:
|
|
19
|
+
metadata:
|
|
20
|
+
description: 1-3 sentence plain-text summary of what was actually verified
|
|
21
|
+
type: string
|
|
22
|
+
optionalProperties:
|
|
23
|
+
required_fixes:
|
|
24
|
+
metadata:
|
|
25
|
+
description: Concrete gaps that must be fixed before the goal can be marked complete (empty for okay)
|
|
26
|
+
elements:
|
|
27
|
+
type: string
|
|
28
|
+
---
|
|
29
|
+
|
|
30
|
+
Decide whether a goal-mode objective is genuinely satisfied by the repo's current state — you are the independent grader, not the agent that did the work.
|
|
31
|
+
|
|
32
|
+
<procedure>
|
|
33
|
+
1. Read the `<objective>` and `<evidence>` given to you below. Treat both as claims to verify, not facts.
|
|
34
|
+
2. Run `git status` and `git diff` (add `git diff --stat` first on a large diff) to see every file actually changed this session. Read the changed files for context beyond the diff hunks when the diff alone doesn't establish correctness.
|
|
35
|
+
3. If the evidence claims a verification command ran (test/build/lint/typecheck), re-run the narrowest form of that command yourself against the current tree — do not trust the claim without confirming it. Do not run a full/slow suite when a scoped one covers the changed files; do not run anything destructive.
|
|
36
|
+
4. Cross-check the diff against the objective: does every part of the objective have corresponding evidence in the diff, or a clear reason no code change was needed for that part? Flag objective clauses with no corresponding change.
|
|
37
|
+
5. Decide the verdict and record it with the structured output.
|
|
38
|
+
</procedure>
|
|
39
|
+
|
|
40
|
+
<constraints>
|
|
41
|
+
- Read-only: you NEVER edit, write, or revert files. Verification commands you run must not mutate the tree (no `--fix`, no auto-format writes, no `git commit`/`git checkout`).
|
|
42
|
+
- Do not invent problems; reject or iterate only with concrete, cited gaps between the objective and the diff.
|
|
43
|
+
- A verdict grounded in nothing is worthless: your justification MUST name the files/diff hunks/commands you actually inspected or ran, and their actual output.
|
|
44
|
+
- You have no access to the maker's reasoning, chat transcript, or thinking blocks — judge the objective against the artifact (diff + verification output) alone. If the evidence text asserts something the diff doesn't support, that is a gap, not a detail to take on faith.
|
|
45
|
+
</constraints>
|
|
46
|
+
|
|
47
|
+
<verdicts>
|
|
48
|
+
- `okay` — every part of the objective is satisfied by the diff, and claimed verification actually holds when re-checked.
|
|
49
|
+
- `iterate` — real progress exists but named gaps remain (missing piece of the objective, evidence claim that doesn't hold up, or verification that fails when re-run); list each in `required_fixes`.
|
|
50
|
+
- `reject` — the diff does not address the objective at all, or the evidence is fabricated/contradicted by re-running it.
|
|
51
|
+
</verdicts>
|
|
@@ -22,6 +22,7 @@ Before calling `goal({op:"complete"})`, you MUST perform a completion audit agai
|
|
|
22
22
|
4. **Match verification scope to claim scope.** A narrow check (one file passes its unit test) does not prove a broad claim (the feature works end-to-end).
|
|
23
23
|
5. **Treat uncertainty as not-yet-achieved.** Indirect evidence, partial coverage, missing artifacts, or "looks right" without inspection mean continue working. Gather stronger evidence or do more work.
|
|
24
24
|
6. **Budget exhaustion is not completion.** NEVER call complete merely because tokens are nearly out. If the budget is tight and the work is unfinished, leave the goal active and stop the turn — the user or runtime decides next steps.
|
|
25
|
+
7. **Reinterpretation is a signal, not a shortcut.** If you catch yourself re-reading the objective to make what you already built count as done, that reinterpretation itself is evidence the objective is NOT met as written — keep working on the gap instead of completing.
|
|
25
26
|
|
|
26
27
|
Call `goal({op:"complete"})` only when every deliverable has direct, current-state evidence proving it is satisfied. The completion call is a load-bearing claim; it ends the autonomous loop and surfaces a "done" report to the user.
|
|
27
28
|
|
|
@@ -4,7 +4,7 @@ Use a single `op` field:
|
|
|
4
4
|
- `create` starts a goal. Requires `objective`; optional `token_budget` must be positive. Use only when no goal exists and no goal is paused.
|
|
5
5
|
- `get` returns the current goal (active or paused) and remaining token budget.
|
|
6
6
|
- `resume` re-activates a paused goal so work can continue.
|
|
7
|
-
- `complete` marks the goal complete after you have verified every deliverable against current evidence. Requires `evidence`: a specific, non-generic description of what you checked and how (tests run, output inspected, files diffed, etc.) — status words like "done" or "verified" are rejected.
|
|
7
|
+
- `complete` marks the goal complete after you have verified every deliverable against current evidence. Requires `evidence`: a specific, non-generic description of what you checked and how (tests run, output inspected, files diffed, etc.) — status words like "done" or "verified" are rejected. Completion is also evidence-gated at runtime: if this goal session modified project files, a verification run (test/build/typecheck/lint) must have succeeded AFTER the last modification, or the first `complete` is rejected with instructions. For docs/config-only work where no verification applies, call `complete` again and explain why in `evidence`.
|
|
8
8
|
- `drop` discards the current goal without completing it.
|
|
9
9
|
|
|
10
10
|
NEVER call `complete` because a budget is low or a turn is ending. Call it only when the goal is actually done and verified.
|
package/src/sdk.ts
CHANGED
|
@@ -76,6 +76,7 @@ import {
|
|
|
76
76
|
setActiveSkills,
|
|
77
77
|
} from "./extensibility/skills";
|
|
78
78
|
import { type FileSlashCommand, loadSlashCommands as loadSlashCommandsInternal } from "./extensibility/slash-commands";
|
|
79
|
+
import { verifyGoalCompletion } from "./goals/verifier";
|
|
79
80
|
import type { HindsightSessionState } from "./hindsight/state";
|
|
80
81
|
import { LocalProtocolHandler, type LocalProtocolOptions } from "./internal-urls";
|
|
81
82
|
import { LSP_STARTUP_EVENT_CHANNEL, type LspStartupEvent } from "./lsp/startup-events";
|
|
@@ -2756,6 +2757,7 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {}
|
|
|
2756
2757
|
providerSessionId: options.providerSessionId,
|
|
2757
2758
|
parentEvalSessionId: options.parentEvalSessionId,
|
|
2758
2759
|
advisorTools,
|
|
2760
|
+
verifyGoalCompletion: (objective, evidence) => verifyGoalCompletion(toolSession, objective, evidence),
|
|
2759
2761
|
titleSystemPrompt: options.titleSystemPrompt,
|
|
2760
2762
|
});
|
|
2761
2763
|
hasSession = true;
|
|
@@ -222,7 +222,7 @@ import { ExtensionToolWrapper } from "../extensibility/extensions/wrapper";
|
|
|
222
222
|
import type { HookCommandContext } from "../extensibility/hooks/types";
|
|
223
223
|
import type { Skill, SkillWarning } from "../extensibility/skills";
|
|
224
224
|
import { expandSlashCommand, type FileSlashCommand } from "../extensibility/slash-commands";
|
|
225
|
-
import { GoalRuntime } from "../goals/runtime";
|
|
225
|
+
import { type GoalCompletionVerdict, GoalRuntime } from "../goals/runtime";
|
|
226
226
|
import type { Goal, GoalModeState } from "../goals/state";
|
|
227
227
|
import type { HindsightSessionState } from "../hindsight/state";
|
|
228
228
|
import { type LocalProtocolOptions, resolveLocalUrlToPath } from "../internal-urls";
|
|
@@ -539,13 +539,19 @@ function refusalBackoffBudgetMs(): number {
|
|
|
539
539
|
}
|
|
540
540
|
/**
|
|
541
541
|
* Budget for callers on the user-visible `/quit` / `/exit` shutdown path that
|
|
542
|
-
* want to cap how long they wait for `MnemopiSessionState.dispose()`
|
|
543
|
-
*
|
|
544
|
-
*
|
|
545
|
-
*
|
|
546
|
-
*
|
|
542
|
+
* want to cap how long they wait for `MnemopiSessionState.dispose()` (and the
|
|
543
|
+
* Hindsight retain-queue flush, see the dispose() call site) to finish writing
|
|
544
|
+
* this session's memory before the process exits. Consolidate fires fresh LLM
|
|
545
|
+
* fact extractions, each a 1-3s round-trip; the retain-queue flush is a single
|
|
546
|
+
* HTTP POST. The prior 1500ms budget was tighter than a single extraction
|
|
547
|
+
* round-trip, so it lost the race on effectively every session with any
|
|
548
|
+
* memory-worthy content — "write before walking away" in name only. 8s still
|
|
549
|
+
* bounds worst-case UI unresponsiveness on /exit while giving a small
|
|
550
|
+
* consolidation batch (1-3 facts) a realistic chance to land before the
|
|
551
|
+
* process exits. Callers that keep the process/session host alive must omit
|
|
552
|
+
* it so dispose still awaits the full write pipeline unbounded.
|
|
547
553
|
*/
|
|
548
|
-
export const SHUTDOWN_CONSOLIDATE_BUDGET_MS =
|
|
554
|
+
export const SHUTDOWN_CONSOLIDATE_BUDGET_MS = 8_000;
|
|
549
555
|
|
|
550
556
|
export interface AgentSessionDisposeOptions {
|
|
551
557
|
mnemopiConsolidateTimeoutMs?: number;
|
|
@@ -728,6 +734,15 @@ export interface AgentSessionConfig {
|
|
|
728
734
|
rebuildSystemPrompt?: (toolNames: string[], tools: Map<string, AgentTool>) => Promise<{ systemPrompt: string[] }>;
|
|
729
735
|
/** Rebuild the SSH tool from current capability discovery results. */
|
|
730
736
|
reloadSshTool?: () => Promise<AgentTool | null>;
|
|
737
|
+
/**
|
|
738
|
+
* Independent-verifier bridge for goal-mode completion (see goals/verifier.ts).
|
|
739
|
+
* Spawns the bundled `goal-verifier` sub-agent to grade a completion claim against
|
|
740
|
+
* the objective and the actual repo diff — a fresh session/context, no exposure to
|
|
741
|
+
* this session's transcript. Wired from sdk.ts (needs the constructed `toolSession`).
|
|
742
|
+
* Undefined when `goal.verifyCompletion` is off or the caller has no tool session
|
|
743
|
+
* (e.g. minimal SDK embedders) — GoalRuntime skips independent grading in that case.
|
|
744
|
+
*/
|
|
745
|
+
verifyGoalCompletion?: (objective: string, evidence: string) => Promise<GoalCompletionVerdict>;
|
|
731
746
|
requestedToolNames?: ReadonlySet<string>;
|
|
732
747
|
/**
|
|
733
748
|
* Optional accessor for live MCP server instructions. Read by the session's
|
|
@@ -1566,6 +1581,9 @@ export class AgentSession {
|
|
|
1566
1581
|
* used as the open barrier for the next build so two writers never share a file. */
|
|
1567
1582
|
#advisorRecorderClosed: Promise<void> = Promise.resolve();
|
|
1568
1583
|
#goalTurnCounter = 0;
|
|
1584
|
+
/** Command text of in-flight bash/eval calls, keyed by toolCallId — consumed by the
|
|
1585
|
+
* goal evidence gate at tool_execution_end to classify verification runs. */
|
|
1586
|
+
#goalEvidenceCommands = new Map<string, string>();
|
|
1569
1587
|
#planReferenceSent = false;
|
|
1570
1588
|
#planReferencePath = "local://PLAN.md";
|
|
1571
1589
|
#clientBridge: ClientBridge | undefined;
|
|
@@ -2207,6 +2225,14 @@ export class AgentSession {
|
|
|
2207
2225
|
{ deliverAs: message.deliverAs },
|
|
2208
2226
|
);
|
|
2209
2227
|
},
|
|
2228
|
+
verifyCompletion: config.verifyGoalCompletion
|
|
2229
|
+
? async (objective: string, evidence: string): Promise<GoalCompletionVerdict> => {
|
|
2230
|
+
if (!this.settings.get("goal.verifyCompletion")) {
|
|
2231
|
+
return { verdict: "okay", justification: "goal.verifyCompletion disabled", requiredFixes: [] };
|
|
2232
|
+
}
|
|
2233
|
+
return config.verifyGoalCompletion!(objective, evidence);
|
|
2234
|
+
}
|
|
2235
|
+
: undefined,
|
|
2210
2236
|
});
|
|
2211
2237
|
this.#cancelExitRecorder = postmortem.register(`agent-session:${this.sessionManager.getSessionId()}`, reason => {
|
|
2212
2238
|
this.#recordSessionExit(reason);
|
|
@@ -3049,6 +3075,24 @@ export class AgentSession {
|
|
|
3049
3075
|
this.sessionManager.appendCustomEntry(TOOL_EXECUTION_START_CUSTOM_TYPE, data);
|
|
3050
3076
|
}
|
|
3051
3077
|
|
|
3078
|
+
/** Feed successful tool executions to the goal completion-evidence gate.
|
|
3079
|
+
* Mutations (edit/write/ast_edit) arm it; verification-signal bash/eval runs
|
|
3080
|
+
* satisfy it. Failed executions prove nothing and are skipped. */
|
|
3081
|
+
#recordGoalToolEvidence(event: Extract<AgentEvent, { type: "tool_execution_end" }>): void {
|
|
3082
|
+
const command = this.#goalEvidenceCommands.get(event.toolCallId);
|
|
3083
|
+
this.#goalEvidenceCommands.delete(event.toolCallId);
|
|
3084
|
+
if (event.isError || event.toolName === "goal") return;
|
|
3085
|
+
let outputHead: string | undefined;
|
|
3086
|
+
if (command !== undefined) {
|
|
3087
|
+
const content = (event.result as { content?: Array<{ type?: string; text?: string }> } | undefined)?.content;
|
|
3088
|
+
if (Array.isArray(content)) {
|
|
3089
|
+
const firstText = content.find(block => block?.type === "text" && typeof block.text === "string");
|
|
3090
|
+
outputHead = firstText?.text;
|
|
3091
|
+
}
|
|
3092
|
+
}
|
|
3093
|
+
this.#goalRuntime.recordToolEvidence(event.toolName, command !== undefined ? { command, outputHead } : undefined);
|
|
3094
|
+
}
|
|
3095
|
+
|
|
3052
3096
|
#recordSessionExit(reason: postmortem.Reason | "dispose"): void {
|
|
3053
3097
|
if (this.#exitRecorded) return;
|
|
3054
3098
|
this.#exitRecorded = true;
|
|
@@ -3434,6 +3478,16 @@ export class AgentSession {
|
|
|
3434
3478
|
|
|
3435
3479
|
if (event.type === "tool_execution_start") {
|
|
3436
3480
|
this.#recordToolExecutionStart(event);
|
|
3481
|
+
// Stash bash/eval command text for the goal evidence gate; consumed at
|
|
3482
|
+
// tool_execution_end. Bounded: entries are deleted on end, and the map
|
|
3483
|
+
// is cleared at turn start.
|
|
3484
|
+
if (event.toolName === "bash" || event.toolName === "eval") {
|
|
3485
|
+
const command = (event.args as { command?: unknown; code?: unknown } | undefined) ?? {};
|
|
3486
|
+
const text = typeof command.command === "string" ? command.command : command.code;
|
|
3487
|
+
if (typeof text === "string" && text.length > 0) {
|
|
3488
|
+
this.#goalEvidenceCommands.set(event.toolCallId, text);
|
|
3489
|
+
}
|
|
3490
|
+
}
|
|
3437
3491
|
}
|
|
3438
3492
|
|
|
3439
3493
|
try {
|
|
@@ -3445,6 +3499,7 @@ export class AgentSession {
|
|
|
3445
3499
|
|
|
3446
3500
|
if (event.type === "turn_start") {
|
|
3447
3501
|
this.#resetStreamingEditState();
|
|
3502
|
+
this.#goalEvidenceCommands.clear();
|
|
3448
3503
|
// TTSR: Reset buffer on turn start
|
|
3449
3504
|
this.#ttsrManager?.resetBuffer();
|
|
3450
3505
|
}
|
|
@@ -3465,6 +3520,7 @@ export class AgentSession {
|
|
|
3465
3520
|
}
|
|
3466
3521
|
}
|
|
3467
3522
|
if (event.type === "tool_execution_end") {
|
|
3523
|
+
this.#recordGoalToolEvidence(event);
|
|
3468
3524
|
if (event.toolName === "goal") {
|
|
3469
3525
|
await this.#goalRuntime.onGoalToolCompleted();
|
|
3470
3526
|
} else {
|
|
@@ -5585,7 +5641,27 @@ export class AgentSession {
|
|
|
5585
5641
|
// Reversed, the spliced batch survives just long enough to fail the
|
|
5586
5642
|
// identity check and get dropped with a `session vanished` warning.
|
|
5587
5643
|
const hindsightState = this.getHindsightSessionState();
|
|
5588
|
-
|
|
5644
|
+
// Bounded when a shutdown budget is set (interactive /quit /exit): the
|
|
5645
|
+
// underlying HindsightApi client makes plain `fetch()` calls with no
|
|
5646
|
+
// timeout/signal, so an unreachable or hung Hindsight server would
|
|
5647
|
+
// otherwise block process exit indefinitely. Callers that keep the
|
|
5648
|
+
// process/session host alive (no timeout passed) still await the flush
|
|
5649
|
+
// unbounded, matching the mnemopi dispose contract below.
|
|
5650
|
+
if (hindsightState) {
|
|
5651
|
+
const { mnemopiConsolidateTimeoutMs: timeoutMs } = options;
|
|
5652
|
+
if (timeoutMs !== undefined && timeoutMs > 0) {
|
|
5653
|
+
try {
|
|
5654
|
+
await withTimeout(hindsightState.flushRetainQueue(), timeoutMs, "Hindsight retain-queue flush");
|
|
5655
|
+
} catch (error) {
|
|
5656
|
+
logger.warn("Hindsight retain-queue flush exceeded shutdown budget; remaining items dropped", {
|
|
5657
|
+
timeoutMs,
|
|
5658
|
+
error: String(error),
|
|
5659
|
+
});
|
|
5660
|
+
}
|
|
5661
|
+
} else {
|
|
5662
|
+
await hindsightState.flushRetainQueue();
|
|
5663
|
+
}
|
|
5664
|
+
}
|
|
5589
5665
|
this.setHindsightSessionState(undefined);
|
|
5590
5666
|
hindsightState?.dispose();
|
|
5591
5667
|
const mnemopiState = setMnemopiSessionState(this, undefined);
|
package/src/task/agents.ts
CHANGED
|
@@ -12,6 +12,7 @@ import designerMd from "../prompts/agents/designer.md" with { type: "text" };
|
|
|
12
12
|
import exploreMd from "../prompts/agents/explore.md" with { type: "text" };
|
|
13
13
|
// Embed agent markdown files at build time
|
|
14
14
|
import agentFrontmatterTemplate from "../prompts/agents/frontmatter.md" with { type: "text" };
|
|
15
|
+
import goalVerifierMd from "../prompts/agents/goal-verifier.md" with { type: "text" };
|
|
15
16
|
import librarianMd from "../prompts/agents/librarian.md" with { type: "text" };
|
|
16
17
|
|
|
17
18
|
import planMd from "../prompts/agents/plan.md" with { type: "text" };
|
|
@@ -47,6 +48,7 @@ const EMBEDDED_AGENT_DEFS: EmbeddedAgentDef[] = [
|
|
|
47
48
|
{ fileName: "explore.md", template: exploreMd },
|
|
48
49
|
{ fileName: "plan.md", template: planMd },
|
|
49
50
|
{ fileName: "critic.md", template: criticMd },
|
|
51
|
+
{ fileName: "goal-verifier.md", template: goalVerifierMd },
|
|
50
52
|
{ fileName: "architect.md", template: architectMd },
|
|
51
53
|
{ fileName: "designer.md", template: designerMd },
|
|
52
54
|
{ fileName: "reviewer.md", template: reviewerMd },
|