@jwilger/pi-development-system 0.34.0 → 0.36.0

package/extensions/development-system.ts

@@ -1,9 +1,14 @@
 import { readFileSync } from "node:fs";
 import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import { cadenceLine, DEFAULT_PUSH_MINUTES } from "../src/context/cadence.ts";
+import { registerCompactionResync } from "../src/context/compaction.ts";
 import { appendContextTail, renderContextTail } from "../src/context/context-tail.ts";
+import { registerModelAdvice } from "../src/context/model-advice.ts";
 import { renderStatus, renderStatusLine, STATUS_KEY } from "../src/context/status.ts";
 import { applyPromptSection } from "../src/context/system-prompt.ts";
+import { DEFAULT_VERIFIER_MAX, registerTurnVerifier } from "../src/context/turn-verifier.ts";
 import { type Exec, timeoutAsFailure } from "../src/core/exec.ts";
+import { defaultMatrix } from "../src/core/models.ts";
 import { detectProfiles } from "../src/core/profile.ts";
 import { createApprovalStore } from "../src/gates/approvals.ts";
 import { registerCommitGuard } from "../src/gates/commit-guard.ts";
@@ -37,6 +42,7 @@ export function createDevelopmentSystem(pi: ExtensionAPI) {
   const jevHolder = createJevHolder();
   let jevModel: string | undefined;
   let lastCtx: ExtensionContext | undefined;
+  let pushMinutes = DEFAULT_PUSH_MINUTES;
 
   const refreshStatus = () => {
     lastCtx?.ui.setStatus(STATUS_KEY, renderStatusLine(state.get(), jevModel));
@@ -44,6 +50,7 @@ export function createDevelopmentSystem(pi: ExtensionAPI) {
 
   const refreshProfiles = async (ctx: ExtensionContext) => {
     const config = await loadConfig(ctx.cwd);
+    pushMinutes = config.ok ? config.value.cadence.pushMinutes : DEFAULT_PUSH_MINUTES;
     const override = config.ok ? config.value.profiles.override : [];
     const profiles = await detectProfiles(ctx.cwd, override);
     const current = state.get().profiles ?? [];
@@ -74,7 +81,13 @@ export function createDevelopmentSystem(pi: ExtensionAPI) {
   });
 
   pi.on("context", (event) => {
-    const messages = appendContextTail(event.messages, renderContextTail(state.get()), Date.now());
+    const now = Date.now();
+    const cadence = cadenceLine(state.get(), now, pushMinutes);
+    const messages = appendContextTail(
+      event.messages,
+      renderContextTail(state.get(), cadence),
+      now,
+    );
     return messages === undefined ? undefined : { messages };
   });
 
@@ -101,6 +114,24 @@ export function createDevelopmentSystem(pi: ExtensionAPI) {
   });
   registerTestGuard({ pi, state, approvals, jev: (ctx) => jevHolder.forContext(ctx) });
   registerTestEvidence({ pi, state });
+  registerCompactionResync({ pi, state });
+  registerModelAdvice({
+    pi,
+    state,
+    matrix: async (ctx) => {
+      const config = await loadConfig(ctx.cwd);
+      return config.ok ? config.value.models : defaultMatrix();
+    },
+  });
+  registerTurnVerifier({
+    pi,
+    state,
+    jev: (ctx) => jevHolder.forContext(ctx),
+    maxPerSession: async (ctx) => {
+      const config = await loadConfig(ctx.cwd);
+      return config.ok ? config.value.verifier.maxPerSession : DEFAULT_VERIFIER_MAX;
+    },
+  });
   registerRedFirstGuard({ pi, state });
   registerLintSuppressionGuard({ pi, state });
   pi.registerTool(createRecordDepartureTool({ pi, state }));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jwilger/pi-development-system",
-  "version": "0.34.0",
+  "version": "0.36.0",
   "description": "A pi extension package representing a seasoned approach to software development using a full AI SDLC.",
   "keywords": [
     "pi-package"

package/src/context/cadence.ts

@@ -0,0 +1,18 @@
+export const DEFAULT_PUSH_MINUTES = 60;
+
+import type { DevsysState } from "../core/types.ts";
+
+/** Advisory only: a long gap since the last push while implementing hints the increment is too big. */
+export function cadenceLine(
+  state: DevsysState,
+  now: number,
+  pushMinutes: number,
+): string | undefined {
+  if (state.phase !== "implementing" || state.lastPushAt === undefined) return undefined;
+  const pushed = Date.parse(state.lastPushAt);
+  if (!Number.isFinite(pushed)) return undefined;
+  const minutes = Math.floor((now - pushed) / 60_000);
+  return minutes > pushMinutes
+    ? `⚠ ${minutes} min since last push — is this increment too big?`
+    : undefined;
+}

package/src/context/compaction.ts

@@ -0,0 +1,24 @@
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import type { SessionState } from "../state/session-state.ts";
+import { renderStateBlock } from "./state-block.ts";
+
+export const RESYNC_ENTRY_TYPE = "devsys-resync";
+
+/**
+ * pi lets `session_before_compact` only REPLACE the whole summary (summary, firstKeptEntryId,
+ * tokensBefore), which would mean writing our own summariser and paying another model call. So the
+ * summary is left to pi, and once it lands we re-state the working state as one message the next
+ * request sees. State itself lives in session entries and the per-turn context tail, so nothing is
+ * lost; this just makes it visible to the model again.
+ */
+export function registerCompactionResync(deps: { pi: ExtensionAPI; state: SessionState }): void {
+  deps.pi.on("session_compact", () => {
+    const state = deps.state.get();
+    if (state.phase === "idle" && state.openDepartures.length === 0) return;
+    deps.pi.sendMessage({
+      customType: RESYNC_ENTRY_TYPE,
+      content: `[devsys resync] Context was compacted. Current working state:\n\n${renderStateBlock(state)}`,
+      display: true,
+    });
+  });
+}

package/src/context/context-tail.ts

@@ -9,7 +9,7 @@ const scopeLabel = (d: Departure): string =>
   d.scope.kind === "slice" ? `slice ${d.scope.slice}` : d.scope.kind;
 
 /** Small, cache-friendly reminder appended at the end of the context; undefined when nothing to say. */
-export function renderContextTail(state: DevsysState): string | undefined {
+export function renderContextTail(state: DevsysState, cadence?: string): string | undefined {
   if (state.phase === "idle" && state.openDepartures.length === 0) return undefined;
   const head = [
     "[development-system]",
@@ -17,7 +17,7 @@ export function renderContextTail(state: DevsysState): string | undefined {
     `Jev: ${state.jev}`,
     ...((state.profiles?.length ?? 0) > 0 ? [`profiles: ${state.profiles?.join(", ")}`] : []),
   ];
-  const tailLines = [REMINDER];
+  const tailLines = [...(cadence === undefined ? [] : [cadence]), REMINDER];
   const room = MAX_LINES - head.length - tailLines.length - 1;
   const lines = state.openDepartures.map(
     (d) => `- ${d.gate} — ${d.chosen.replace(/\s+/g, " ")} (${scopeLabel(d)})`,

package/src/context/model-advice.ts

@@ -0,0 +1,58 @@
+import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import { adviseModel } from "../core/model-advice.ts";
+import type { ModelMatrix } from "../core/models.ts";
+import type { Phase } from "../core/types.ts";
+import { availableModels } from "../state/models-command.ts";
+import type { SessionState } from "../state/session-state.ts";
+
+const ADVICE_ENTRY_TYPE = "devsys-model-advice";
+
+export type ModelAdviceDeps = {
+  pi: ExtensionAPI;
+  state: SessionState;
+  matrix(ctx: ExtensionContext): Promise<ModelMatrix>;
+};
+
+type Seen = { ctx: ExtensionContext; model: { provider: string; id: string } | undefined };
+
+/**
+ * Recommends — never switches — when the coordinator's model tier is below what the phase's slot
+ * resolves to. Runs on model_select and on phase change, at most once per (phase, model).
+ */
+export function registerModelAdvice(deps: ModelAdviceDeps): void {
+  const told = new Set<string>();
+  let last: Seen | undefined;
+  let phase: Phase = deps.state.get().phase;
+
+  const check = async (seen: Seen): Promise<void> => {
+    const { model } = seen;
+    const current = deps.state.get().phase;
+    if (model === undefined) return;
+    const ref = `${model.provider}/${model.id}`;
+    const key = `${current}|${ref}`;
+    if (told.has(key)) return;
+    const advice = adviseModel({
+      matrix: await deps.matrix(seen.ctx),
+      available: availableModels(seen.ctx.modelRegistry),
+      phase: current,
+      model: ref,
+    });
+    if (advice === undefined) return;
+    told.add(key);
+    deps.pi.sendMessage({ customType: ADVICE_ENTRY_TYPE, content: advice, display: true });
+  };
+
+  deps.pi.on("session_start", (_event, ctx) => {
+    last = { ctx, model: ctx.model };
+    phase = deps.state.get().phase;
+  });
+  deps.pi.on("model_select", async (event, ctx) => {
+    last = { ctx, model: event.model };
+    await check(last);
+  });
+  deps.state.onChange((state) => {
+    if (state.phase === phase) return;
+    phase = state.phase;
+    if (last !== undefined) void check(last).catch(() => undefined);
+  });
+}

package/src/context/state-block.ts

@@ -0,0 +1,30 @@
+import type { DevsysState } from "../core/types.ts";
+import { activeReview } from "./status.ts";
+
+const MAX_DEPARTURES = 8;
+
+/** The durable working state as a short markdown block, for compaction resync and the like. */
+export function renderStateBlock(state: DevsysState): string {
+  const departures = state.openDepartures.map(
+    (d) => `  - ${d.gate} — ${d.chosen.replace(/\s+/g, " ")}`,
+  );
+  const shown =
+    departures.length <= MAX_DEPARTURES
+      ? departures
+      : [
+          ...departures.slice(0, MAX_DEPARTURES - 1),
+          `  - …and ${departures.length - MAX_DEPARTURES + 1} more`,
+        ];
+  const run = state.lastTestRun;
+  return [
+    "## Development System State",
+    `- phase: ${state.phase}`,
+    `- sizing: ${state.sizing ?? "unset"}`,
+    `- slice: ${state.activeSlice ?? "none"}`,
+    `- open departures: ${state.openDepartures.length === 0 ? "none" : state.openDepartures.length}`,
+    ...shown,
+    `- ${activeReview(state) ?? "review: none"}`,
+    `- last test run: ${run === undefined ? "none" : `exit ${run.exitCode} — ${run.summary}`}`,
+    `- last push: ${state.lastPushAt ?? "none"}`,
+  ].join("\n");
+}

package/src/context/status.ts

@@ -13,7 +13,7 @@ const ciLabel = (state: DevsysState): string | undefined => {
   return `ci ${ci.status}${sha}`;
 };
 
-const activeReview = (state: DevsysState): string | undefined => {
+export const activeReview = (state: DevsysState): string | undefined => {
   // Without an active slice (before /devsys-start exists) show the most recent review instead of nothing.
   const review =
     state.activeSlice === undefined ? state.reviews?.at(-1) : reviewOf(state, state.activeSlice);

package/src/context/turn-verifier.ts

@@ -0,0 +1,114 @@
+import type { AgentMessage } from "@earendil-works/pi-agent-core";
+import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import { redactSecrets } from "../core/redact.ts";
+import { summarizeOutput } from "../core/test-runner.ts";
+import type { Phase } from "../core/types.ts";
+import type { Jev } from "../jev/client.ts";
+import { judgeTurn, type ToolEvidence } from "../jev/questions/turn.ts";
+import type { SessionState } from "../state/session-state.ts";
+
+export const VERIFIER_THRESHOLD = 0.75;
+export const DEFAULT_VERIFIER_MAX = 6;
+export const VERIFIER_ENTRY_TYPE = "devsys-verifier";
+const VERIFIED_PHASES: ReadonlySet<Phase> = new Set(["implementing", "reviewing", "delivering"]);
+const MAX_EVIDENCE = 60;
+
+export type TurnVerifierDeps = {
+  pi: ExtensionAPI;
+  state: SessionState;
+  jev(ctx: ExtensionContext): Jev;
+  maxPerSession(ctx: ExtensionContext): number | Promise<number>;
+};
+
+type Parts = { text: string; callsTools: boolean };
+
+/** Text and tool-call presence of an assistant message; undefined for any other role. */
+export function assistantParts(message: AgentMessage): Parts | undefined {
+  if (message.role !== "assistant") return undefined;
+  const text = message.content.flatMap((c) => (c.type === "text" ? [c.text] : [])).join("\n");
+  return { text, callsTools: message.content.some((c) => c.type === "toolCall") };
+}
+
+/** True when the last non-empty line of the message ends with a question mark. */
+export function asksUser(text: string): boolean {
+  const last = text
+    .split("\n")
+    .map((l) => l.trim())
+    .filter((l) => l !== "")
+    .at(-1);
+  return last !== undefined && /\?[\s)"'*_`]*$/.test(last);
+}
+
+const claimMessage = (): string =>
+  "Development system: your last message claims something was run, passed or done, but no tool evidence in this run shows it. Run the verification now, or restate without the claim.";
+
+const driftMessage = (slice: string): string =>
+  `Development system: your last message describes work beyond the active slice "${slice}". Either return to the slice, or call devsys_record_departure (gate scope.expansion) to record why the scope is growing.`;
+
+const note = (content: string) => ({
+  type: "custom_message" as const,
+  customType: VERIFIER_ENTRY_TYPE,
+  content,
+  display: true,
+});
+
+/**
+ * At turn end, asks Jev whether the final message claims results no tool call supports, or wanders
+ * from the active slice, and forces ONE corrective continuation. Bounded: never on a turn that
+ * still calls tools, never on a question to the user, never twice in a row, never more than
+ * `maxPerSession` times. Jev trouble means silence, not a block.
+ */
+export function registerTurnVerifier(deps: TurnVerifierDeps): void {
+  let evidence: ToolEvidence[] = [];
+  let corrected = 0;
+  let justCorrected = false;
+
+  deps.pi.on("session_start", () => {
+    corrected = 0;
+    justCorrected = false;
+    evidence = [];
+  });
+  deps.pi.on("agent_start", () => {
+    evidence = [];
+    justCorrected = false;
+  });
+  deps.pi.on("tool_result", (event) => {
+    const text = event.content.flatMap((c) => (c.type === "text" ? [c.text] : [])).join("\n");
+    const summary = redactSecrets(summarizeOutput(text));
+    const item: ToolEvidence =
+      event.toolName === "bash"
+        ? { tool: "bash", summary, exitCode: event.isError ? 1 : 0 }
+        : { tool: event.toolName, summary };
+    evidence = [...evidence, item].slice(-MAX_EVIDENCE);
+  });
+
+  deps.pi.on("turn_end", async (event, ctx) => {
+    const state = deps.state.get();
+    if (!VERIFIED_PHASES.has(state.phase)) return undefined;
+    const parts = assistantParts(event.message);
+    if (parts === undefined || parts.callsTools || parts.text.trim() === "") return undefined;
+    if (justCorrected) {
+      justCorrected = false;
+      return undefined;
+    }
+    if (asksUser(parts.text) || corrected >= (await deps.maxPerSession(ctx))) return undefined;
+    const jev = deps.jev(ctx);
+    if (jev.availability() === "offline") return undefined;
+    const judged = await judgeTurn(jev, {
+      assistantText: parts.text,
+      toolEvidence: evidence,
+      ...(state.activeSlice === undefined ? {} : { activeSlice: state.activeSlice }),
+    });
+    if (!judged.ok) return undefined;
+    const entries = [
+      ...(judged.value.unverifiedClaim >= VERIFIER_THRESHOLD ? [note(claimMessage())] : []),
+      ...(judged.value.driftFromSlice >= VERIFIER_THRESHOLD && state.activeSlice !== undefined
+        ? [note(driftMessage(state.activeSlice))]
+        : []),
+    ];
+    if (entries.length === 0) return undefined;
+    corrected += 1;
+    justCorrected = true;
+    return { continue: true, entries };
+  });
+}

package/src/core/model-advice.ts

@@ -0,0 +1,81 @@
+import {
+  type Available,
+  flattenSlot,
+  type ModelMatrix,
+  matchesPattern,
+  resolveSlot,
+  type Slot,
+} from "./models.ts";
+import type { Phase } from "./types.ts";
+
+/** Highest first. */
+export const TIERS = ["frontier", "strong", "fast"] as const;
+export type Tier = (typeof TIERS)[number];
+
+const PHASE_SLOT: Partial<Record<Phase, Slot>> = {
+  planning: "planning",
+  implementing: "implementer",
+  reviewing: "reviewer",
+};
+
+export const requiredSlot = (phase: Phase): Slot | undefined => PHASE_SLOT[phase];
+
+const split = (ref: string): { provider: string; id: string } | undefined => {
+  const slash = ref.indexOf("/");
+  return slash <= 0 || slash === ref.length - 1
+    ? undefined
+    : { provider: ref.slice(0, slash), id: ref.slice(slash + 1) };
+};
+
+const matches = (candidates: readonly string[], model: string): boolean => {
+  const ref = split(model);
+  return (
+    ref !== undefined &&
+    candidates.some((candidate) => {
+      const c = split(candidate);
+      return c !== undefined && c.provider === ref.provider && matchesPattern(c.id, ref.id);
+    })
+  );
+};
+
+/**
+ * The lowest tier whose candidates include `provider/id`: a model that appears as a fallback in a
+ * higher tier (sol in frontier) still belongs to the tier it is natively listed in. Undefined when
+ * the model is in none of them.
+ */
+export function tierOf(matrix: ModelMatrix, model: string): Tier | undefined {
+  // Direct candidates only: `fast` falls back to `@strong`, which must not make strong models fast.
+  return [...TIERS].reverse().find((tier) =>
+    matches(
+      matrix[tier].filter((c) => !c.startsWith("@")),
+      model,
+    ),
+  );
+}
+
+export type AdviceInput = {
+  matrix: ModelMatrix;
+  available: readonly Available[];
+  phase: Phase;
+  /** The coordinator's current model as `provider/id`. */
+  model: string;
+};
+
+/**
+ * One recommendation when the coordinator's tier is below what the phase's slot resolves to; never
+ * a model switch. Silent when the phase has no slot, the slot cannot be resolved here, or either
+ * model is outside the three tiers (nothing to compare).
+ */
+export function adviseModel(input: AdviceInput): string | undefined {
+  const slot = requiredSlot(input.phase);
+  if (slot === undefined) return undefined;
+  const wanted = resolveSlot(input.matrix, slot, input.available);
+  if (!wanted.ok) return undefined;
+  // Anything the slot itself lists is acceptable for the phase, whatever tier it natively sits in.
+  if (matches(flattenSlot(input.matrix, slot), input.model)) return undefined;
+  const have = tierOf(input.matrix, input.model);
+  const need = tierOf(input.matrix, wanted.value.model);
+  if (have === undefined || need === undefined) return undefined;
+  if (TIERS.indexOf(have) <= TIERS.indexOf(need)) return undefined;
+  return `${input.phase} on ${input.model} (${have} tier) — the matrix prefers ${wanted.value.model} (${need} tier) for the ${slot} slot. Consider asking an advisor subagent for the hard decisions, or continue and record a departure (gate models.phase-mismatch). The system never switches your model for you.`;
+}

package/src/core/models.ts

@@ -80,7 +80,7 @@ export function compareModelVersions(a: string, b: string): number {
 }
 
 /** `*`-only glob over a model id (no regular expressions built from configuration). */
-function matchesPattern(pattern: string, id: string): boolean {
+export function matchesPattern(pattern: string, id: string): boolean {
   const parts = pattern.split("*");
   if (parts.length === 1) return pattern === id;
   const first = parts[0] ?? "";

package/src/jev/questions/turn.ts

@@ -0,0 +1,81 @@
+import type { ClassifierBoolQuestion } from "@earendil-works/pi-ai";
+import { redactSecrets } from "../../core/redact.ts";
+import { err, ok, type Result } from "../../core/result.ts";
+import type { Jev, JevError } from "../client.ts";
+
+export const CLAIM_QUESTION: ClassifierBoolQuestion = {
+  type: "bool",
+  instructions:
+    '`assistantText` is the last message an AI coding agent wrote. `toolEvidence` lists the tool calls it made this turn (tool, summary, exit code). Does the message state as fact that something was run, passed, fixed, built, committed, pushed or verified, where `toolEvidence` does not show it? A claim that is plainly about a past turn, a plan, an intention, a question or a hedge ("I will run", "should pass") is not a claim of fact.',
+  criteria: {
+    true: "It asserts a completed action or a passing result that no tool call this turn supports (or that a failing tool call contradicts)",
+    false:
+      "Every stated result is backed by the tool evidence, or the message only plans, asks or hedges",
+  },
+};
+
+export const DRIFT_QUESTION: ClassifierBoolQuestion = {
+  type: "bool",
+  instructions:
+    "`activeSlice` names the one slice of work the agent agreed to do. Given `assistantText` and `toolEvidence`, is the agent doing substantial work that the slice does not cover: new features, extra refactors, unrelated files, or changing scope? Housekeeping needed to do the slice (tests, docs for it, commits) is not drift.",
+  criteria: {
+    true: "The agent is doing or proposing work beyond the slice's stated scope",
+    false: "The work stays within the slice, or is the housekeeping it needs",
+  },
+};
+
+export type ToolEvidence = { tool: string; summary: string; exitCode?: number };
+export type TurnInput = {
+  assistantText: string;
+  toolEvidence: readonly ToolEvidence[];
+  activeSlice?: string;
+};
+export type TurnJudgement = { unverifiedClaim: number; driftFromSlice: number };
+
+const TEXT_MAX = 4000;
+const SUMMARY_MAX = 240;
+const EVIDENCE_MAX = 30;
+
+const clip = (text: string, max: number): string =>
+  redactSecrets(text.slice(0, max * 2)).slice(0, max);
+
+const evidenceLine = (e: ToolEvidence): string =>
+  `${clip(e.tool, 40)}${e.exitCode === undefined ? "" : ` exit ${e.exitCode}`}: ${clip(e.summary, SUMMARY_MAX)}`;
+
+const probability = (
+  answer: { type: string; probability?: number } | undefined,
+  name: string,
+): Result<number, JevError> => {
+  if (answer?.type !== "bool" || typeof answer.probability !== "number") {
+    return err({ kind: "provider", message: `missing ${name} answer` });
+  }
+  return Number.isFinite(answer.probability)
+    ? ok(answer.probability)
+    : err({ kind: "provider", message: `malformed ${name} answer` });
+};
+
+/** Two independent narrow questions; drift is only asked when a slice is active. */
+export async function judgeTurn(
+  jev: Jev,
+  input: TurnInput,
+): Promise<Result<TurnJudgement, JevError>> {
+  const slice = input.activeSlice === undefined ? undefined : clip(input.activeSlice, 200);
+  const state = {
+    assistantText: clip(input.assistantText, TEXT_MAX),
+    toolEvidence: input.toolEvidence.slice(-EVIDENCE_MAX).map(evidenceLine),
+    ...(slice === undefined ? {} : { activeSlice: slice }),
+  };
+  const asked = await jev.ask(
+    state,
+    slice === undefined
+      ? { claim: CLAIM_QUESTION }
+      : { claim: CLAIM_QUESTION, drift: DRIFT_QUESTION },
+  );
+  if (!asked.ok) return asked;
+  const claim = probability(asked.value.claim, "claim");
+  if (!claim.ok) return claim;
+  if (slice === undefined) return ok({ unverifiedClaim: claim.value, driftFromSlice: 0 });
+  const drift = probability(asked.value.drift, "drift");
+  if (!drift.ok) return drift;
+  return ok({ unverifiedClaim: claim.value, driftFromSlice: drift.value });
+}