jeopi-cli 16.2.28 → 16.2.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/cli.js +2638 -2637
- package/dist/types/goals/evidence-gate.d.ts +62 -0
- package/dist/types/goals/index.d.ts +1 -0
- package/dist/types/goals/runtime.d.ts +13 -0
- package/package.json +12 -12
- package/src/config/model-registry.ts +39 -6
- package/src/goals/evidence-gate.ts +137 -0
- package/src/goals/index.ts +1 -0
- package/src/goals/runtime.ts +35 -0
- package/src/prompts/goals/goal-continuation.md +1 -0
- package/src/prompts/tools/goal.md +1 -1
- package/src/session/agent-session.ts +33 -0
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic evidence gate for goal completion — the code-level counterpart to
|
|
3
|
+
* the prompt-side completion audit (`prompts/goals/goal-continuation.md`).
|
|
4
|
+
*
|
|
5
|
+
* `assertSubstantiveCompletionEvidence` (completion-evidence.ts) only checks that
|
|
6
|
+
* the completion *claim text* is non-generic; it cannot see whether the work was
|
|
7
|
+
* actually verified. This gate closes that "gate theater" failure mode with
|
|
8
|
+
* runtime signals the session already observes: a goal turn that MUTATED project
|
|
9
|
+
* files can self-report complete only when a verification signal (test / build /
|
|
10
|
+
* typecheck / lint run) was observed AFTER the last mutation.
|
|
11
|
+
*
|
|
12
|
+
* Design mirrors gajae-code's `classifyDoneGate` + jeo-code's `applyEvidenceGate`:
|
|
13
|
+
* - Pure classification over recorded signals — no re-scan of the transcript.
|
|
14
|
+
* - Single-pushback latch: the first blocked `complete` bounces with a corrective
|
|
15
|
+
* message; a second `complete` with no new evidence passes (the escape hatch for
|
|
16
|
+
* docs/config-only changes where no verification command applies). Any new
|
|
17
|
+
* mutation recorded after a bounce re-arms the gate.
|
|
18
|
+
*/
|
|
19
|
+
/** Tool names whose successful execution counts as a project-file mutation. */
|
|
20
|
+
export declare const MUTATION_TOOL_NAMES: Record<string, true>;
|
|
21
|
+
/**
|
|
22
|
+
* Commands (or their output head) that count as a verification signal: a test,
|
|
23
|
+
* build, typecheck, or lint invocation. Matches command text and the first
|
|
24
|
+
* `VERIFY_OUTPUT_SCAN_LIMIT` chars of output — enough to catch a runner's banner
|
|
25
|
+
* without rescanning megabytes of logs.
|
|
26
|
+
*/
|
|
27
|
+
export declare const VERIFY_SIGNAL_RE: RegExp;
|
|
28
|
+
/** Output prefix length scanned for verification signals. */
|
|
29
|
+
export declare const VERIFY_OUTPUT_SCAN_LIMIT = 2000;
|
|
30
|
+
/** True when a bash/eval command (or the head of its output) proves a verification ran. */
|
|
31
|
+
export declare function isVerificationSignal(command: string, output?: string): boolean;
|
|
32
|
+
export type EvidenceGateState = "pass" | "unverified" | "stale-verification";
|
|
33
|
+
export interface EvidenceGateVerdict {
|
|
34
|
+
state: EvidenceGateState;
|
|
35
|
+
/** When true, `complete` should be bounced once with `message`. */
|
|
36
|
+
block: boolean;
|
|
37
|
+
/** Corrective message for the bounce; empty when `state === "pass"`. */
|
|
38
|
+
message: string;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Tracks mutation/verification evidence for the active goal. Sequence-ordered so
|
|
42
|
+
* "verified, then mutated again" is detected as stale evidence regardless of how
|
|
43
|
+
* many turns separate the two.
|
|
44
|
+
*/
|
|
45
|
+
export declare class GoalEvidenceTracker {
|
|
46
|
+
#private;
|
|
47
|
+
/** Reset all evidence (new goal, or goal replaced). */
|
|
48
|
+
reset(): void;
|
|
49
|
+
/** Record a successful project-file mutation (edit/write/ast_edit). */
|
|
50
|
+
recordMutation(): void;
|
|
51
|
+
/** Record a successful verification run (test/build/typecheck/lint). */
|
|
52
|
+
recordVerification(): void;
|
|
53
|
+
get sawMutation(): boolean;
|
|
54
|
+
get sawVerification(): boolean;
|
|
55
|
+
/** True when the last mutation happened after the last verification. */
|
|
56
|
+
get verificationStale(): boolean;
|
|
57
|
+
/**
|
|
58
|
+
* Classify a `complete` attempt. First blocked attempt arms the bypass latch;
|
|
59
|
+
* the next attempt (with no new mutation in between) passes.
|
|
60
|
+
*/
|
|
61
|
+
classifyCompletion(): EvidenceGateVerdict;
|
|
62
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { GoalEvidenceTracker } from "./evidence-gate";
|
|
1
2
|
import type { Goal, GoalBudgetSteering, GoalModeState, GoalRuntimeEvent, GoalTokenUsage } from "./state";
|
|
2
3
|
export interface GoalRuntimeHost {
|
|
3
4
|
getState(): GoalModeState | undefined;
|
|
@@ -37,6 +38,18 @@ export declare class GoalRuntime {
|
|
|
37
38
|
constructor(host: GoalRuntimeHost);
|
|
38
39
|
get snapshot(): GoalRuntimeSnapshot;
|
|
39
40
|
clearAccounting(): void;
|
|
41
|
+
/** Deterministic completion-evidence tracker for the active goal (see evidence-gate.ts). */
|
|
42
|
+
get evidence(): GoalEvidenceTracker;
|
|
43
|
+
/**
|
|
44
|
+
* Record a successful tool execution as completion evidence for the active goal.
|
|
45
|
+
* Mutation tools (edit/write/ast_edit) arm the completion gate; bash/eval runs
|
|
46
|
+
* whose command or output head carries a verification signal (test/build/
|
|
47
|
+
* typecheck/lint) satisfy it. No-op when no goal is accounting.
|
|
48
|
+
*/
|
|
49
|
+
recordToolEvidence(toolName: string, detail?: {
|
|
50
|
+
command?: string;
|
|
51
|
+
outputHead?: string;
|
|
52
|
+
}): void;
|
|
40
53
|
onTurnStart(turnId: string, baselineUsage: GoalTokenUsage): void;
|
|
41
54
|
onToolCompleted(toolName: string): Promise<void>;
|
|
42
55
|
onGoalToolCompleted(): Promise<void>;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"type": "module",
|
|
3
3
|
"name": "jeopi-cli",
|
|
4
|
-
"version": "16.2.
|
|
4
|
+
"version": "16.2.30",
|
|
5
5
|
"description": "Coding agent CLI with read, bash, edit, write tools and session management",
|
|
6
6
|
"homepage": "https://github.com/akillness/jeopi",
|
|
7
7
|
"author": "Can Boluk",
|
|
@@ -57,17 +57,17 @@
|
|
|
57
57
|
"@agentclientprotocol/sdk": "0.25.0",
|
|
58
58
|
"@babel/parser": "^7.29.7",
|
|
59
59
|
"@mozilla/readability": "^0.6.0",
|
|
60
|
-
"jeopi-hashline": "16.2.
|
|
61
|
-
"jeopi-stats": "16.2.
|
|
62
|
-
"jeopi-agent-core": "16.2.
|
|
63
|
-
"jeopi-ai": "16.2.
|
|
64
|
-
"jeopi-catalog": "16.2.
|
|
65
|
-
"jeopi-mnemopi": "16.2.
|
|
66
|
-
"jeopi-natives": "16.2.
|
|
67
|
-
"jeopi-tui": "16.2.
|
|
68
|
-
"jeopi-utils": "16.2.
|
|
69
|
-
"jeopi-wire": "16.2.
|
|
70
|
-
"jeopi-snapcompact": "16.2.
|
|
60
|
+
"jeopi-hashline": "16.2.30",
|
|
61
|
+
"jeopi-stats": "16.2.30",
|
|
62
|
+
"jeopi-agent-core": "16.2.30",
|
|
63
|
+
"jeopi-ai": "16.2.30",
|
|
64
|
+
"jeopi-catalog": "16.2.30",
|
|
65
|
+
"jeopi-mnemopi": "16.2.30",
|
|
66
|
+
"jeopi-natives": "16.2.30",
|
|
67
|
+
"jeopi-tui": "16.2.30",
|
|
68
|
+
"jeopi-utils": "16.2.30",
|
|
69
|
+
"jeopi-wire": "16.2.30",
|
|
70
|
+
"jeopi-snapcompact": "16.2.30",
|
|
71
71
|
"@opentelemetry/api": "^1.9.1",
|
|
72
72
|
"@opentelemetry/context-async-hooks": "^2.7.1",
|
|
73
73
|
"@opentelemetry/exporter-trace-otlp-proto": "^0.218.0",
|
|
@@ -156,6 +156,17 @@ const AUTHORITATIVE_RUNTIME_CATALOG_PROVIDERS = new Set<string>(
|
|
|
156
156
|
),
|
|
157
157
|
);
|
|
158
158
|
|
|
159
|
+
function isAuthoritativeLocalDiscovery(providerConfig: DiscoveryProviderConfig): boolean {
|
|
160
|
+
switch (providerConfig.discovery.type) {
|
|
161
|
+
case "ollama":
|
|
162
|
+
case "llama.cpp":
|
|
163
|
+
case "lm-studio":
|
|
164
|
+
return true;
|
|
165
|
+
default:
|
|
166
|
+
return false;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
159
170
|
function isAuthoritativeProjectCatalogModel(model: Model<Api>): boolean {
|
|
160
171
|
return (
|
|
161
172
|
model.provider === "google-vertex" &&
|
|
@@ -1334,7 +1345,21 @@ export class ModelRegistry {
|
|
|
1334
1345
|
this.#discoverBuiltInProviderModels(strategy, providerFilter),
|
|
1335
1346
|
]);
|
|
1336
1347
|
const discovered = [...configuredDiscovered, ...builtInDiscovery.models];
|
|
1337
|
-
|
|
1348
|
+
const successfulAuthoritativeDiscoverableProviders = new Set<string>();
|
|
1349
|
+
for (const providerConfig of selectedDiscoverableProviders) {
|
|
1350
|
+
if (!isAuthoritativeLocalDiscovery(providerConfig)) {
|
|
1351
|
+
continue;
|
|
1352
|
+
}
|
|
1353
|
+
const state = this.#providerDiscoveryStates.get(providerConfig.provider);
|
|
1354
|
+
if (state && (state.status === "ok" || state.status === "empty") && !state.stale) {
|
|
1355
|
+
successfulAuthoritativeDiscoverableProviders.add(providerConfig.provider);
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1358
|
+
if (
|
|
1359
|
+
discovered.length === 0 &&
|
|
1360
|
+
builtInDiscovery.authoritativeProviders.size === 0 &&
|
|
1361
|
+
successfulAuthoritativeDiscoverableProviders.size === 0
|
|
1362
|
+
) {
|
|
1338
1363
|
return;
|
|
1339
1364
|
}
|
|
1340
1365
|
const discoveredModels = this.#applyHardcodedModelPolicies(
|
|
@@ -1350,6 +1375,9 @@ export class ModelRegistry {
|
|
|
1350
1375
|
for (const provider of builtInDiscovery.authoritativeProviders) {
|
|
1351
1376
|
authoritativeProviders.add(provider);
|
|
1352
1377
|
}
|
|
1378
|
+
for (const provider of successfulAuthoritativeDiscoverableProviders) {
|
|
1379
|
+
authoritativeProviders.add(provider);
|
|
1380
|
+
}
|
|
1353
1381
|
const baseModels =
|
|
1354
1382
|
authoritativeProviders.size > 0 ? dropProviderModels(this.#models, authoritativeProviders) : this.#models;
|
|
1355
1383
|
const resolved = this.#mergeResolvedModels(baseModels, discoveredModels);
|
|
@@ -1385,9 +1413,9 @@ export class ModelRegistry {
|
|
|
1385
1413
|
const cacheProviderId = this.#configuredDiscoveryCacheProviderId(providerConfig);
|
|
1386
1414
|
const cached = readModelCache<Api>(cacheProviderId, 24 * 60 * 60 * 1000, Date.now, this.#cacheDbPath);
|
|
1387
1415
|
const cacheOlderThanConfig = cached !== null && this.#isDiscoveryCacheOlderThanModelsConfig(cached.updatedAt);
|
|
1388
|
-
const
|
|
1389
|
-
|
|
1390
|
-
|
|
1416
|
+
const shouldForceOnlineDiscovery =
|
|
1417
|
+
strategy === "online-if-uncached" && (cacheOlderThanConfig || isAuthoritativeLocalDiscovery(providerConfig));
|
|
1418
|
+
const effectiveStrategy = shouldForceOnlineDiscovery ? "online" : strategy;
|
|
1391
1419
|
const requiresAuth = !this.#keylessProviders.has(providerConfig.provider);
|
|
1392
1420
|
if (requiresAuth) {
|
|
1393
1421
|
const apiKey = await this.#peekApiKeyForProvider(providerConfig.provider);
|
|
@@ -1450,7 +1478,10 @@ export class ModelRegistry {
|
|
|
1450
1478
|
provider: providerId,
|
|
1451
1479
|
status,
|
|
1452
1480
|
optional: providerConfig.optional ?? false,
|
|
1453
|
-
stale:
|
|
1481
|
+
stale:
|
|
1482
|
+
result.stale ||
|
|
1483
|
+
status === "cached" ||
|
|
1484
|
+
(shouldForceOnlineDiscovery && status !== "ok" && status !== "empty"),
|
|
1454
1485
|
fetchedAt: discoveryError ? cached?.updatedAt : Date.now(),
|
|
1455
1486
|
models: result.models.map(model => model.id),
|
|
1456
1487
|
error: discoveryError,
|
|
@@ -1627,7 +1658,9 @@ export class ModelRegistry {
|
|
|
1627
1658
|
): Promise<BuiltInDiscoveryResult> {
|
|
1628
1659
|
try {
|
|
1629
1660
|
const manager = createModelManager({ ...options, cacheDbPath: this.#cacheDbPath });
|
|
1630
|
-
const
|
|
1661
|
+
const effectiveStrategy =
|
|
1662
|
+
strategy === "online-if-uncached" && options.providerId === "lm-studio" ? "online" : strategy;
|
|
1663
|
+
const result = await manager.refresh(effectiveStrategy);
|
|
1631
1664
|
const models = result.models.map(model =>
|
|
1632
1665
|
model.provider === options.providerId ? model : { ...model, provider: options.providerId },
|
|
1633
1666
|
);
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic evidence gate for goal completion — the code-level counterpart to
|
|
3
|
+
* the prompt-side completion audit (`prompts/goals/goal-continuation.md`).
|
|
4
|
+
*
|
|
5
|
+
* `assertSubstantiveCompletionEvidence` (completion-evidence.ts) only checks that
|
|
6
|
+
* the completion *claim text* is non-generic; it cannot see whether the work was
|
|
7
|
+
* actually verified. This gate closes that "gate theater" failure mode with
|
|
8
|
+
* runtime signals the session already observes: a goal turn that MUTATED project
|
|
9
|
+
* files can self-report complete only when a verification signal (test / build /
|
|
10
|
+
* typecheck / lint run) was observed AFTER the last mutation.
|
|
11
|
+
*
|
|
12
|
+
* Design mirrors gajae-code's `classifyDoneGate` + jeo-code's `applyEvidenceGate`:
|
|
13
|
+
* - Pure classification over recorded signals — no re-scan of the transcript.
|
|
14
|
+
* - Single-pushback latch: the first blocked `complete` bounces with a corrective
|
|
15
|
+
* message; a second `complete` with no new evidence passes (the escape hatch for
|
|
16
|
+
* docs/config-only changes where no verification command applies). Any new
|
|
17
|
+
* mutation recorded after a bounce re-arms the gate.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/** Tool names whose successful execution counts as a project-file mutation. */
|
|
21
|
+
export const MUTATION_TOOL_NAMES: Record<string, true> = {
|
|
22
|
+
edit: true,
|
|
23
|
+
write: true,
|
|
24
|
+
ast_edit: true,
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Commands (or their output head) that count as a verification signal: a test,
|
|
29
|
+
* build, typecheck, or lint invocation. Matches command text and the first
|
|
30
|
+
* `VERIFY_OUTPUT_SCAN_LIMIT` chars of output — enough to catch a runner's banner
|
|
31
|
+
* without rescanning megabytes of logs.
|
|
32
|
+
*/
|
|
33
|
+
export const VERIFY_SIGNAL_RE = /\b(test|tests|tsc|tsgo|typecheck|lint|build|check|spec|pytest|vitest|jest|biome)\b/i;
|
|
34
|
+
|
|
35
|
+
/** Output prefix length scanned for verification signals. */
|
|
36
|
+
export const VERIFY_OUTPUT_SCAN_LIMIT = 2000;
|
|
37
|
+
|
|
38
|
+
/** True when a bash/eval command (or the head of its output) proves a verification ran. */
|
|
39
|
+
export function isVerificationSignal(command: string, output = ""): boolean {
|
|
40
|
+
return VERIFY_SIGNAL_RE.test(command) || VERIFY_SIGNAL_RE.test(output.slice(0, VERIFY_OUTPUT_SCAN_LIMIT));
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export type EvidenceGateState = "pass" | "unverified" | "stale-verification";
|
|
44
|
+
|
|
45
|
+
export interface EvidenceGateVerdict {
|
|
46
|
+
state: EvidenceGateState;
|
|
47
|
+
/** When true, `complete` should be bounced once with `message`. */
|
|
48
|
+
block: boolean;
|
|
49
|
+
/** Corrective message for the bounce; empty when `state === "pass"`. */
|
|
50
|
+
message: string;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Tracks mutation/verification evidence for the active goal. Sequence-ordered so
|
|
55
|
+
* "verified, then mutated again" is detected as stale evidence regardless of how
|
|
56
|
+
* many turns separate the two.
|
|
57
|
+
*/
|
|
58
|
+
export class GoalEvidenceTracker {
|
|
59
|
+
#seq = 0;
|
|
60
|
+
#lastMutationSeq = 0;
|
|
61
|
+
#lastVerificationSeq = 0;
|
|
62
|
+
/** Set when a `complete` was bounced; cleared by any new mutation. */
|
|
63
|
+
#bypassArmed = false;
|
|
64
|
+
|
|
65
|
+
/** Reset all evidence (new goal, or goal replaced). */
|
|
66
|
+
reset(): void {
|
|
67
|
+
this.#seq = 0;
|
|
68
|
+
this.#lastMutationSeq = 0;
|
|
69
|
+
this.#lastVerificationSeq = 0;
|
|
70
|
+
this.#bypassArmed = false;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Record a successful project-file mutation (edit/write/ast_edit). */
|
|
74
|
+
recordMutation(): void {
|
|
75
|
+
this.#lastMutationSeq = ++this.#seq;
|
|
76
|
+
// New mutation invalidates a previously granted escape hatch: the model
|
|
77
|
+
// must re-justify (or re-verify) the new change.
|
|
78
|
+
this.#bypassArmed = false;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Record a successful verification run (test/build/typecheck/lint). */
|
|
82
|
+
recordVerification(): void {
|
|
83
|
+
this.#lastVerificationSeq = ++this.#seq;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
get sawMutation(): boolean {
|
|
87
|
+
return this.#lastMutationSeq > 0;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
get sawVerification(): boolean {
|
|
91
|
+
return this.#lastVerificationSeq > 0;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** True when the last mutation happened after the last verification. */
|
|
95
|
+
get verificationStale(): boolean {
|
|
96
|
+
return this.#lastMutationSeq > this.#lastVerificationSeq;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Classify a `complete` attempt. First blocked attempt arms the bypass latch;
|
|
101
|
+
* the next attempt (with no new mutation in between) passes.
|
|
102
|
+
*/
|
|
103
|
+
classifyCompletion(): EvidenceGateVerdict {
|
|
104
|
+
if (!this.sawMutation) {
|
|
105
|
+
return { state: "pass", block: false, message: "" };
|
|
106
|
+
}
|
|
107
|
+
if (this.sawVerification && !this.verificationStale) {
|
|
108
|
+
return { state: "pass", block: false, message: "" };
|
|
109
|
+
}
|
|
110
|
+
if (this.#bypassArmed) {
|
|
111
|
+
// Second attempt without new evidence: the escape hatch for changes
|
|
112
|
+
// where no verification command applies (docs/config-only).
|
|
113
|
+
return { state: "pass", block: false, message: "" };
|
|
114
|
+
}
|
|
115
|
+
this.#bypassArmed = true;
|
|
116
|
+
if (!this.sawVerification) {
|
|
117
|
+
return {
|
|
118
|
+
state: "unverified",
|
|
119
|
+
block: true,
|
|
120
|
+
message:
|
|
121
|
+
"Completion blocked: this goal session modified project files but no verification signal " +
|
|
122
|
+
"(test/build/typecheck/lint run) was observed. Run the narrowest command that proves the " +
|
|
123
|
+
"change works, then call complete again. If verification is genuinely not applicable " +
|
|
124
|
+
"(docs/config-only change), call complete again and say why in the evidence.",
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
return {
|
|
128
|
+
state: "stale-verification",
|
|
129
|
+
block: true,
|
|
130
|
+
message:
|
|
131
|
+
"Completion blocked: files were modified AFTER the last successful verification run, so that " +
|
|
132
|
+
"evidence no longer reflects the current tree. Re-run the narrowest verification command " +
|
|
133
|
+
"against the latest changes, then call complete again. If the later edits are " +
|
|
134
|
+
"verification-irrelevant (docs/config-only), call complete again and say why in the evidence.",
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
}
|
package/src/goals/index.ts
CHANGED
package/src/goals/runtime.ts
CHANGED
|
@@ -3,6 +3,7 @@ import goalBudgetLimitPrompt from "../prompts/goals/goal-budget-limit.md" with {
|
|
|
3
3
|
import goalContinuationPrompt from "../prompts/goals/goal-continuation.md" with { type: "text" };
|
|
4
4
|
import goalModeActivePrompt from "../prompts/goals/goal-mode-active.md" with { type: "text" };
|
|
5
5
|
import { assertSubstantiveCompletionEvidence } from "./completion-evidence";
|
|
6
|
+
import { GoalEvidenceTracker, isVerificationSignal, MUTATION_TOOL_NAMES } from "./evidence-gate";
|
|
6
7
|
import type { Goal, GoalBudgetSteering, GoalModeState, GoalRuntimeEvent, GoalTokenUsage } from "./state";
|
|
7
8
|
|
|
8
9
|
export interface GoalRuntimeHost {
|
|
@@ -117,6 +118,7 @@ function isAccountingStatus(goal: Goal): boolean {
|
|
|
117
118
|
|
|
118
119
|
export class GoalRuntime {
|
|
119
120
|
readonly #host: GoalRuntimeHost;
|
|
121
|
+
readonly #evidence = new GoalEvidenceTracker();
|
|
120
122
|
#turnSnapshot: GoalTurnSnapshot | undefined;
|
|
121
123
|
#wallClock: GoalWallClockSnapshot;
|
|
122
124
|
#budgetReportedFor: string | undefined;
|
|
@@ -202,6 +204,28 @@ export class GoalRuntime {
|
|
|
202
204
|
this.#budgetReportedFor = undefined;
|
|
203
205
|
}
|
|
204
206
|
|
|
207
|
+
/** Deterministic completion-evidence tracker for the active goal (see evidence-gate.ts). */
|
|
208
|
+
get evidence(): GoalEvidenceTracker {
|
|
209
|
+
return this.#evidence;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* Record a successful tool execution as completion evidence for the active goal.
|
|
214
|
+
* Mutation tools (edit/write/ast_edit) arm the completion gate; bash/eval runs
|
|
215
|
+
* whose command or output head carries a verification signal (test/build/
|
|
216
|
+
* typecheck/lint) satisfy it. No-op when no goal is accounting.
|
|
217
|
+
*/
|
|
218
|
+
recordToolEvidence(toolName: string, detail?: { command?: string; outputHead?: string }): void {
|
|
219
|
+
if (!this.#hasAccountingState()) return;
|
|
220
|
+
if (MUTATION_TOOL_NAMES[toolName]) {
|
|
221
|
+
this.#evidence.recordMutation();
|
|
222
|
+
return;
|
|
223
|
+
}
|
|
224
|
+
if (detail && isVerificationSignal(detail.command ?? "", detail.outputHead ?? "")) {
|
|
225
|
+
this.#evidence.recordVerification();
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
205
229
|
onTurnStart(turnId: string, baselineUsage: GoalTokenUsage): void {
|
|
206
230
|
this.#turnSnapshot = { turnId, baselineUsage: { ...baselineUsage } };
|
|
207
231
|
const state = this.#host.getState();
|
|
@@ -393,6 +417,7 @@ export class GoalRuntime {
|
|
|
393
417
|
}
|
|
394
418
|
const state = this.#createGoalState(objective, input.tokenBudget);
|
|
395
419
|
this.#budgetReportedFor = undefined;
|
|
420
|
+
this.#evidence.reset();
|
|
396
421
|
this.#markActiveAccounting(state.goal);
|
|
397
422
|
await this.#commitState(state, { persist: "goal" });
|
|
398
423
|
return state;
|
|
@@ -411,6 +436,7 @@ export class GoalRuntime {
|
|
|
411
436
|
await this.#flushUsageLocked("suppressed");
|
|
412
437
|
const state = this.#createGoalState(objective, input.tokenBudget);
|
|
413
438
|
this.#budgetReportedFor = undefined;
|
|
439
|
+
this.#evidence.reset();
|
|
414
440
|
this.#markActiveAccounting(state.goal);
|
|
415
441
|
await this.#commitState(state, { persist: "goal" });
|
|
416
442
|
return state;
|
|
@@ -487,6 +513,15 @@ export class GoalRuntime {
|
|
|
487
513
|
if (state.goal.status === "dropped") {
|
|
488
514
|
throw new Error("cannot complete a dropped goal");
|
|
489
515
|
}
|
|
516
|
+
// Deterministic evidence gate (evidence-gate.ts): a goal session that
|
|
517
|
+
// mutated project files cannot self-report complete unless a verification
|
|
518
|
+
// signal was observed after the last mutation. First violation bounces
|
|
519
|
+
// once with a corrective message; a repeat complete with no new mutation
|
|
520
|
+
// passes (escape hatch for docs/config-only work).
|
|
521
|
+
const gate = this.#evidence.classifyCompletion();
|
|
522
|
+
if (gate.block) {
|
|
523
|
+
throw new Error(gate.message);
|
|
524
|
+
}
|
|
490
525
|
state.enabled = false;
|
|
491
526
|
state.goal.status = "complete";
|
|
492
527
|
state.goal.completionEvidence = verifiedEvidence;
|
|
@@ -22,6 +22,7 @@ Before calling `goal({op:"complete"})`, you MUST perform a completion audit agai
|
|
|
22
22
|
4. **Match verification scope to claim scope.** A narrow check (one file passes its unit test) does not prove a broad claim (the feature works end-to-end).
|
|
23
23
|
5. **Treat uncertainty as not-yet-achieved.** Indirect evidence, partial coverage, missing artifacts, or "looks right" without inspection mean continue working. Gather stronger evidence or do more work.
|
|
24
24
|
6. **Budget exhaustion is not completion.** NEVER call complete merely because tokens are nearly out. If the budget is tight and the work is unfinished, leave the goal active and stop the turn — the user or runtime decides next steps.
|
|
25
|
+
7. **Reinterpretation is a signal, not a shortcut.** If you catch yourself re-reading the objective to make what you already built count as done, that reinterpretation itself is evidence the objective is NOT met as written — keep working on the gap instead of completing.
|
|
25
26
|
|
|
26
27
|
Call `goal({op:"complete"})` only when every deliverable has direct, current-state evidence proving it is satisfied. The completion call is a load-bearing claim; it ends the autonomous loop and surfaces a "done" report to the user.
|
|
27
28
|
|
|
@@ -4,7 +4,7 @@ Use a single `op` field:
|
|
|
4
4
|
- `create` starts a goal. Requires `objective`; optional `token_budget` must be positive. Use only when no goal exists and no goal is paused.
|
|
5
5
|
- `get` returns the current goal (active or paused) and remaining token budget.
|
|
6
6
|
- `resume` re-activates a paused goal so work can continue.
|
|
7
|
-
- `complete` marks the goal complete after you have verified every deliverable against current evidence. Requires `evidence`: a specific, non-generic description of what you checked and how (tests run, output inspected, files diffed, etc.) — status words like "done" or "verified" are rejected.
|
|
7
|
+
- `complete` marks the goal complete after you have verified every deliverable against current evidence. Requires `evidence`: a specific, non-generic description of what you checked and how (tests run, output inspected, files diffed, etc.) — status words like "done" or "verified" are rejected. Completion is also evidence-gated at runtime: if this goal session modified project files, a verification run (test/build/typecheck/lint) must have succeeded AFTER the last modification, or the first `complete` is rejected with instructions. For docs/config-only work where no verification applies, call `complete` again and explain why in `evidence`.
|
|
8
8
|
- `drop` discards the current goal without completing it.
|
|
9
9
|
|
|
10
10
|
NEVER call `complete` because a budget is low or a turn is ending. Call it only when the goal is actually done and verified.
|
|
@@ -1566,6 +1566,9 @@ export class AgentSession {
|
|
|
1566
1566
|
* used as the open barrier for the next build so two writers never share a file. */
|
|
1567
1567
|
#advisorRecorderClosed: Promise<void> = Promise.resolve();
|
|
1568
1568
|
#goalTurnCounter = 0;
|
|
1569
|
+
/** Command text of in-flight bash/eval calls, keyed by toolCallId — consumed by the
|
|
1570
|
+
* goal evidence gate at tool_execution_end to classify verification runs. */
|
|
1571
|
+
#goalEvidenceCommands = new Map<string, string>();
|
|
1569
1572
|
#planReferenceSent = false;
|
|
1570
1573
|
#planReferencePath = "local://PLAN.md";
|
|
1571
1574
|
#clientBridge: ClientBridge | undefined;
|
|
@@ -3049,6 +3052,24 @@ export class AgentSession {
|
|
|
3049
3052
|
this.sessionManager.appendCustomEntry(TOOL_EXECUTION_START_CUSTOM_TYPE, data);
|
|
3050
3053
|
}
|
|
3051
3054
|
|
|
3055
|
+
/** Feed successful tool executions to the goal completion-evidence gate.
|
|
3056
|
+
* Mutations (edit/write/ast_edit) arm it; verification-signal bash/eval runs
|
|
3057
|
+
* satisfy it. Failed executions prove nothing and are skipped. */
|
|
3058
|
+
#recordGoalToolEvidence(event: Extract<AgentEvent, { type: "tool_execution_end" }>): void {
|
|
3059
|
+
const command = this.#goalEvidenceCommands.get(event.toolCallId);
|
|
3060
|
+
this.#goalEvidenceCommands.delete(event.toolCallId);
|
|
3061
|
+
if (event.isError || event.toolName === "goal") return;
|
|
3062
|
+
let outputHead: string | undefined;
|
|
3063
|
+
if (command !== undefined) {
|
|
3064
|
+
const content = (event.result as { content?: Array<{ type?: string; text?: string }> } | undefined)?.content;
|
|
3065
|
+
if (Array.isArray(content)) {
|
|
3066
|
+
const firstText = content.find(block => block?.type === "text" && typeof block.text === "string");
|
|
3067
|
+
outputHead = firstText?.text;
|
|
3068
|
+
}
|
|
3069
|
+
}
|
|
3070
|
+
this.#goalRuntime.recordToolEvidence(event.toolName, command !== undefined ? { command, outputHead } : undefined);
|
|
3071
|
+
}
|
|
3072
|
+
|
|
3052
3073
|
#recordSessionExit(reason: postmortem.Reason | "dispose"): void {
|
|
3053
3074
|
if (this.#exitRecorded) return;
|
|
3054
3075
|
this.#exitRecorded = true;
|
|
@@ -3434,6 +3455,16 @@ export class AgentSession {
|
|
|
3434
3455
|
|
|
3435
3456
|
if (event.type === "tool_execution_start") {
|
|
3436
3457
|
this.#recordToolExecutionStart(event);
|
|
3458
|
+
// Stash bash/eval command text for the goal evidence gate; consumed at
|
|
3459
|
+
// tool_execution_end. Bounded: entries are deleted on end, and the map
|
|
3460
|
+
// is cleared at turn start.
|
|
3461
|
+
if (event.toolName === "bash" || event.toolName === "eval") {
|
|
3462
|
+
const command = (event.args as { command?: unknown; code?: unknown } | undefined) ?? {};
|
|
3463
|
+
const text = typeof command.command === "string" ? command.command : command.code;
|
|
3464
|
+
if (typeof text === "string" && text.length > 0) {
|
|
3465
|
+
this.#goalEvidenceCommands.set(event.toolCallId, text);
|
|
3466
|
+
}
|
|
3467
|
+
}
|
|
3437
3468
|
}
|
|
3438
3469
|
|
|
3439
3470
|
try {
|
|
@@ -3445,6 +3476,7 @@ export class AgentSession {
|
|
|
3445
3476
|
|
|
3446
3477
|
if (event.type === "turn_start") {
|
|
3447
3478
|
this.#resetStreamingEditState();
|
|
3479
|
+
this.#goalEvidenceCommands.clear();
|
|
3448
3480
|
// TTSR: Reset buffer on turn start
|
|
3449
3481
|
this.#ttsrManager?.resetBuffer();
|
|
3450
3482
|
}
|
|
@@ -3465,6 +3497,7 @@ export class AgentSession {
|
|
|
3465
3497
|
}
|
|
3466
3498
|
}
|
|
3467
3499
|
if (event.type === "tool_execution_end") {
|
|
3500
|
+
this.#recordGoalToolEvidence(event);
|
|
3468
3501
|
if (event.toolName === "goal") {
|
|
3469
3502
|
await this.#goalRuntime.onGoalToolCompleted();
|
|
3470
3503
|
} else {
|