@jwilger/pi-development-system 0.76.0 → 0.77.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/context/turn-verifier.ts +17 -1
- package/src/jev/questions/turn.ts +34 -10
package/package.json
CHANGED
|
@@ -5,6 +5,7 @@ import type {
|
|
|
5
5
|
ToolResultEvent,
|
|
6
6
|
} from "@earendil-works/pi-coding-agent";
|
|
7
7
|
import { redactSecrets } from "../core/redact.ts";
|
|
8
|
+
import { reviewOf } from "../core/review-flow.ts";
|
|
8
9
|
import { exitCodeOf, summarizeOutput } from "../core/test-runner.ts";
|
|
9
10
|
import type { DevsysState, Phase } from "../core/types.ts";
|
|
10
11
|
import type { Jev } from "../jev/client.ts";
|
|
@@ -58,6 +59,9 @@ export function asksUser(text: string): boolean {
|
|
|
58
59
|
return QUESTION_END.test(lead) || (LIST_ITEM.test(last) && CHOICE_LEAD_IN.test(lead));
|
|
59
60
|
}
|
|
60
61
|
|
|
62
|
+
const hasReviewRound = (state: DevsysState, slice: DevsysState["activeSlice"] & string): boolean =>
|
|
63
|
+
(reviewOf(state, slice)?.rounds.length ?? 0) > 0;
|
|
64
|
+
|
|
61
65
|
/** True when a scope.expansion departure already covers the slice: the remedy the note asks for exists. */
|
|
62
66
|
const expansionRecorded = (state: DevsysState, slice: string): boolean =>
|
|
63
67
|
state.openDepartures.some(
|
|
@@ -72,6 +76,9 @@ const claimMessage = (): string =>
|
|
|
72
76
|
const driftMessage = (slice: string): string =>
|
|
73
77
|
`Development system: your last message describes work beyond the active slice "${slice}". Either return to the slice, or call devsys_record_departure (gate scope.expansion) to record why the scope is growing.`;
|
|
74
78
|
|
|
79
|
+
const reviewMessage = (slice: string): string =>
|
|
80
|
+
`Development system: your last message calls the slice "${slice}" finished, but no review round has been recorded for it. Call devsys_review_start for a fresh-context review before committing.`;
|
|
81
|
+
|
|
75
82
|
const TARGET_MAX = 120;
|
|
76
83
|
|
|
77
84
|
const note = (content: string) => ({
|
|
@@ -107,7 +114,7 @@ function evidenceOf(event: ToolResultEvent): ToolEvidence {
|
|
|
107
114
|
|
|
108
115
|
/** The corrective notes a judged turn earns: an unverified claim, and drift not yet covered by a departure. */
|
|
109
116
|
function correctionsFor(
|
|
110
|
-
judged: { unverifiedClaim: number; driftFromSlice: number },
|
|
117
|
+
judged: { unverifiedClaim: number; driftFromSlice: number; sliceDone: number },
|
|
111
118
|
state: DevsysState,
|
|
112
119
|
) {
|
|
113
120
|
const slice = state.activeSlice;
|
|
@@ -118,6 +125,11 @@ function correctionsFor(
|
|
|
118
125
|
return [
|
|
119
126
|
...(judged.unverifiedClaim >= VERIFIER_THRESHOLD ? [note(claimMessage())] : []),
|
|
120
127
|
...(drifted && slice !== undefined ? [note(driftMessage(slice))] : []),
|
|
128
|
+
...(slice !== undefined &&
|
|
129
|
+
judged.sliceDone >= VERIFIER_THRESHOLD &&
|
|
130
|
+
!hasReviewRound(state, slice)
|
|
131
|
+
? [note(reviewMessage(slice))]
|
|
132
|
+
: []),
|
|
121
133
|
];
|
|
122
134
|
}
|
|
123
135
|
|
|
@@ -163,6 +175,10 @@ export function registerTurnVerifier(deps: TurnVerifierDeps): void {
|
|
|
163
175
|
assistantText: parts.text,
|
|
164
176
|
toolEvidence: evidence,
|
|
165
177
|
...(state.activeSlice === undefined ? {} : { activeSlice: state.activeSlice }),
|
|
178
|
+
checkDone:
|
|
179
|
+
state.phase === "implementing" &&
|
|
180
|
+
state.activeSlice !== undefined &&
|
|
181
|
+
!hasReviewRound(state, state.activeSlice),
|
|
166
182
|
});
|
|
167
183
|
if (!judged.ok) return undefined;
|
|
168
184
|
const entries = correctionsFor(judged.value, state);
|
|
@@ -24,13 +24,31 @@ export const DRIFT_QUESTION: ClassifierBoolQuestion = {
|
|
|
24
24
|
},
|
|
25
25
|
};
|
|
26
26
|
|
|
27
|
+
export const DONE_QUESTION: ClassifierBoolQuestion = {
|
|
28
|
+
type: "bool",
|
|
29
|
+
instructions:
|
|
30
|
+
"`activeSlice` names the one slice of work the agent is doing. Does `assistantText` say that the slice's work is finished and ready to commit or hand over, rather than reporting progress, planning more work or asking something?",
|
|
31
|
+
criteria: {
|
|
32
|
+
true: "The message presents the slice as complete: implemented, tested and ready to commit, push or review",
|
|
33
|
+
false:
|
|
34
|
+
"The message reports partial progress, plans next steps, asks a question or is unrelated to finishing the slice",
|
|
35
|
+
},
|
|
36
|
+
};
|
|
37
|
+
|
|
27
38
|
export type ToolEvidence = { tool: string; summary: string; exitCode?: number };
|
|
28
39
|
export type TurnInput = {
|
|
29
40
|
assistantText: string;
|
|
30
41
|
toolEvidence: readonly ToolEvidence[];
|
|
31
42
|
activeSlice?: string;
|
|
43
|
+
/** Also ask whether the message calls the slice finished (only meaningful with an active slice). */
|
|
44
|
+
checkDone?: boolean;
|
|
45
|
+
};
|
|
46
|
+
export type TurnJudgement = {
|
|
47
|
+
unverifiedClaim: number;
|
|
48
|
+
driftFromSlice: number;
|
|
49
|
+
/** 0 unless `checkDone` was set and a slice is active. */
|
|
50
|
+
sliceDone: number;
|
|
32
51
|
};
|
|
33
|
-
export type TurnJudgement = { unverifiedClaim: number; driftFromSlice: number };
|
|
34
52
|
|
|
35
53
|
const TEXT_MAX = 4000;
|
|
36
54
|
const SUMMARY_MAX = 240;
|
|
@@ -54,7 +72,7 @@ const probability = (
|
|
|
54
72
|
: err({ kind: "provider", message: `malformed ${name} answer` });
|
|
55
73
|
};
|
|
56
74
|
|
|
57
|
-
/**
|
|
75
|
+
/** Narrow questions asked together; drift and done only when a slice is active (done only on request). */
|
|
58
76
|
export async function judgeTurn(
|
|
59
77
|
jev: Jev,
|
|
60
78
|
input: TurnInput,
|
|
@@ -65,17 +83,23 @@ export async function judgeTurn(
|
|
|
65
83
|
toolEvidence: input.toolEvidence.slice(-EVIDENCE_MAX).map(evidenceLine),
|
|
66
84
|
...(slice === undefined ? {} : { activeSlice: slice }),
|
|
67
85
|
};
|
|
68
|
-
const
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
);
|
|
86
|
+
const wantDone = slice !== undefined && input.checkDone === true;
|
|
87
|
+
const asked = await jev.ask(state, {
|
|
88
|
+
claim: CLAIM_QUESTION,
|
|
89
|
+
...(slice === undefined ? {} : { drift: DRIFT_QUESTION }),
|
|
90
|
+
...(wantDone ? { done: DONE_QUESTION } : {}),
|
|
91
|
+
});
|
|
74
92
|
if (!asked.ok) return asked;
|
|
75
93
|
const claim = probability(asked.value.claim, "claim");
|
|
76
94
|
if (!claim.ok) return claim;
|
|
77
|
-
if (slice === undefined)
|
|
95
|
+
if (slice === undefined)
|
|
96
|
+
return ok({ unverifiedClaim: claim.value, driftFromSlice: 0, sliceDone: 0 });
|
|
78
97
|
const drift = probability(asked.value.drift, "drift");
|
|
79
98
|
if (!drift.ok) return drift;
|
|
80
|
-
|
|
99
|
+
if (!wantDone) {
|
|
100
|
+
return ok({ unverifiedClaim: claim.value, driftFromSlice: drift.value, sliceDone: 0 });
|
|
101
|
+
}
|
|
102
|
+
const done = probability(asked.value.done, "done");
|
|
103
|
+
if (!done.ok) return done;
|
|
104
|
+
return ok({ unverifiedClaim: claim.value, driftFromSlice: drift.value, sliceDone: done.value });
|
|
81
105
|
}
|