@jwilger/pi-development-system 0.34.0 → 0.35.0

package/extensions/development-system.ts

@@ -3,6 +3,7 @@ import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-a
 import { appendContextTail, renderContextTail } from "../src/context/context-tail.ts";
 import { renderStatus, renderStatusLine, STATUS_KEY } from "../src/context/status.ts";
 import { applyPromptSection } from "../src/context/system-prompt.ts";
+import { DEFAULT_VERIFIER_MAX, registerTurnVerifier } from "../src/context/turn-verifier.ts";
 import { type Exec, timeoutAsFailure } from "../src/core/exec.ts";
 import { detectProfiles } from "../src/core/profile.ts";
 import { createApprovalStore } from "../src/gates/approvals.ts";
@@ -101,6 +102,15 @@ export function createDevelopmentSystem(pi: ExtensionAPI) {
   });
   registerTestGuard({ pi, state, approvals, jev: (ctx) => jevHolder.forContext(ctx) });
   registerTestEvidence({ pi, state });
+  registerTurnVerifier({
+    pi,
+    state,
+    jev: (ctx) => jevHolder.forContext(ctx),
+    maxPerSession: async (ctx) => {
+      const config = await loadConfig(ctx.cwd);
+      return config.ok ? config.value.verifier.maxPerSession : DEFAULT_VERIFIER_MAX;
+    },
+  });
   registerRedFirstGuard({ pi, state });
   registerLintSuppressionGuard({ pi, state });
   pi.registerTool(createRecordDepartureTool({ pi, state }));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jwilger/pi-development-system",
-  "version": "0.34.0",
+  "version": "0.35.0",
   "description": "A pi extension package representing a seasoned approach to software development using a full AI SDLC.",
   "keywords": [
     "pi-package"

package/src/context/turn-verifier.ts

@@ -0,0 +1,114 @@
+import type { AgentMessage } from "@earendil-works/pi-agent-core";
+import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import { redactSecrets } from "../core/redact.ts";
+import { summarizeOutput } from "../core/test-runner.ts";
+import type { Phase } from "../core/types.ts";
+import type { Jev } from "../jev/client.ts";
+import { judgeTurn, type ToolEvidence } from "../jev/questions/turn.ts";
+import type { SessionState } from "../state/session-state.ts";
+
+export const VERIFIER_THRESHOLD = 0.75;
+export const DEFAULT_VERIFIER_MAX = 6;
+export const VERIFIER_ENTRY_TYPE = "devsys-verifier";
+const VERIFIED_PHASES: ReadonlySet<Phase> = new Set(["implementing", "reviewing", "delivering"]);
+const MAX_EVIDENCE = 60;
+
+export type TurnVerifierDeps = {
+  pi: ExtensionAPI;
+  state: SessionState;
+  jev(ctx: ExtensionContext): Jev;
+  maxPerSession(ctx: ExtensionContext): number | Promise<number>;
+};
+
+type Parts = { text: string; callsTools: boolean };
+
+/** Text and tool-call presence of an assistant message; undefined for any other role. */
+export function assistantParts(message: AgentMessage): Parts | undefined {
+  if (message.role !== "assistant") return undefined;
+  const text = message.content.flatMap((c) => (c.type === "text" ? [c.text] : [])).join("\n");
+  return { text, callsTools: message.content.some((c) => c.type === "toolCall") };
+}
+
+/** True when the last non-empty line of the message ends with a question mark. */
+export function asksUser(text: string): boolean {
+  const last = text
+    .split("\n")
+    .map((l) => l.trim())
+    .filter((l) => l !== "")
+    .at(-1);
+  return last !== undefined && /\?[\s)"'*_`]*$/.test(last);
+}
+
+const claimMessage = (): string =>
+  "Development system: your last message claims something was run, passed or done, but no tool evidence in this run shows it. Run the verification now, or restate without the claim.";
+
+const driftMessage = (slice: string): string =>
+  `Development system: your last message describes work beyond the active slice "${slice}". Either return to the slice, or call devsys_record_departure (gate scope.expansion) to record why the scope is growing.`;
+
+const note = (content: string) => ({
+  type: "custom_message" as const,
+  customType: VERIFIER_ENTRY_TYPE,
+  content,
+  display: true,
+});
+
+/**
+ * At turn end, asks Jev whether the final message claims results no tool call supports, or wanders
+ * from the active slice, and forces ONE corrective continuation. Bounded: never on a turn that
+ * still calls tools, never on a question to the user, never twice in a row, never more than
+ * `maxPerSession` times. Jev trouble means silence, not a block.
+ */
+export function registerTurnVerifier(deps: TurnVerifierDeps): void {
+  let evidence: ToolEvidence[] = [];
+  let corrected = 0;
+  let justCorrected = false;
+
+  deps.pi.on("session_start", () => {
+    corrected = 0;
+    justCorrected = false;
+    evidence = [];
+  });
+  deps.pi.on("agent_start", () => {
+    evidence = [];
+    justCorrected = false;
+  });
+  deps.pi.on("tool_result", (event) => {
+    const text = event.content.flatMap((c) => (c.type === "text" ? [c.text] : [])).join("\n");
+    const summary = redactSecrets(summarizeOutput(text));
+    const item: ToolEvidence =
+      event.toolName === "bash"
+        ? { tool: "bash", summary, exitCode: event.isError ? 1 : 0 }
+        : { tool: event.toolName, summary };
+    evidence = [...evidence, item].slice(-MAX_EVIDENCE);
+  });
+
+  deps.pi.on("turn_end", async (event, ctx) => {
+    const state = deps.state.get();
+    if (!VERIFIED_PHASES.has(state.phase)) return undefined;
+    const parts = assistantParts(event.message);
+    if (parts === undefined || parts.callsTools || parts.text.trim() === "") return undefined;
+    if (justCorrected) {
+      justCorrected = false;
+      return undefined;
+    }
+    if (asksUser(parts.text) || corrected >= (await deps.maxPerSession(ctx))) return undefined;
+    const jev = deps.jev(ctx);
+    if (jev.availability() === "offline") return undefined;
+    const judged = await judgeTurn(jev, {
+      assistantText: parts.text,
+      toolEvidence: evidence,
+      ...(state.activeSlice === undefined ? {} : { activeSlice: state.activeSlice }),
+    });
+    if (!judged.ok) return undefined;
+    const entries = [
+      ...(judged.value.unverifiedClaim >= VERIFIER_THRESHOLD ? [note(claimMessage())] : []),
+      ...(judged.value.driftFromSlice >= VERIFIER_THRESHOLD && state.activeSlice !== undefined
+        ? [note(driftMessage(state.activeSlice))]
+        : []),
+    ];
+    if (entries.length === 0) return undefined;
+    corrected += 1;
+    justCorrected = true;
+    return { continue: true, entries };
+  });
+}

package/src/jev/questions/turn.ts

@@ -0,0 +1,81 @@
+import type { ClassifierBoolQuestion } from "@earendil-works/pi-ai";
+import { redactSecrets } from "../../core/redact.ts";
+import { err, ok, type Result } from "../../core/result.ts";
+import type { Jev, JevError } from "../client.ts";
+
+export const CLAIM_QUESTION: ClassifierBoolQuestion = {
+  type: "bool",
+  instructions:
+    '`assistantText` is the last message an AI coding agent wrote. `toolEvidence` lists the tool calls it made this turn (tool, summary, exit code). Does the message state as fact that something was run, passed, fixed, built, committed, pushed or verified, where `toolEvidence` does not show it? A claim that is plainly about a past turn, a plan, an intention, a question or a hedge ("I will run", "should pass") is not a claim of fact.',
+  criteria: {
+    true: "It asserts a completed action or a passing result that no tool call this turn supports (or that a failing tool call contradicts)",
+    false:
+      "Every stated result is backed by the tool evidence, or the message only plans, asks or hedges",
+  },
+};
+
+export const DRIFT_QUESTION: ClassifierBoolQuestion = {
+  type: "bool",
+  instructions:
+    "`activeSlice` names the one slice of work the agent agreed to do. Given `assistantText` and `toolEvidence`, is the agent doing substantial work that the slice does not cover: new features, extra refactors, unrelated files, or changing scope? Housekeeping needed to do the slice (tests, docs for it, commits) is not drift.",
+  criteria: {
+    true: "The agent is doing or proposing work beyond the slice's stated scope",
+    false: "The work stays within the slice, or is the housekeeping it needs",
+  },
+};
+
+export type ToolEvidence = { tool: string; summary: string; exitCode?: number };
+export type TurnInput = {
+  assistantText: string;
+  toolEvidence: readonly ToolEvidence[];
+  activeSlice?: string;
+};
+export type TurnJudgement = { unverifiedClaim: number; driftFromSlice: number };
+
+const TEXT_MAX = 4000;
+const SUMMARY_MAX = 240;
+const EVIDENCE_MAX = 30;
+
+const clip = (text: string, max: number): string =>
+  redactSecrets(text.slice(0, max * 2)).slice(0, max);
+
+const evidenceLine = (e: ToolEvidence): string =>
+  `${clip(e.tool, 40)}${e.exitCode === undefined ? "" : ` exit ${e.exitCode}`}: ${clip(e.summary, SUMMARY_MAX)}`;
+
+const probability = (
+  answer: { type: string; probability?: number } | undefined,
+  name: string,
+): Result<number, JevError> => {
+  if (answer?.type !== "bool" || typeof answer.probability !== "number") {
+    return err({ kind: "provider", message: `missing ${name} answer` });
+  }
+  return Number.isFinite(answer.probability)
+    ? ok(answer.probability)
+    : err({ kind: "provider", message: `malformed ${name} answer` });
+};
+
+/** Two independent narrow questions; drift is only asked when a slice is active. */
+export async function judgeTurn(
+  jev: Jev,
+  input: TurnInput,
+): Promise<Result<TurnJudgement, JevError>> {
+  const slice = input.activeSlice === undefined ? undefined : clip(input.activeSlice, 200);
+  const state = {
+    assistantText: clip(input.assistantText, TEXT_MAX),
+    toolEvidence: input.toolEvidence.slice(-EVIDENCE_MAX).map(evidenceLine),
+    ...(slice === undefined ? {} : { activeSlice: slice }),
+  };
+  const asked = await jev.ask(
+    state,
+    slice === undefined
+      ? { claim: CLAIM_QUESTION }
+      : { claim: CLAIM_QUESTION, drift: DRIFT_QUESTION },
+  );
+  if (!asked.ok) return asked;
+  const claim = probability(asked.value.claim, "claim");
+  if (!claim.ok) return claim;
+  if (slice === undefined) return ok({ unverifiedClaim: claim.value, driftFromSlice: 0 });
+  const drift = probability(asked.value.drift, "drift");
+  if (!drift.ok) return drift;
+  return ok({ unverifiedClaim: claim.value, driftFromSlice: drift.value });
+}