@jwilger/pi-development-system 0.81.0 → 0.83.0

package/README.md

@@ -22,8 +22,9 @@ what the current phase expects. When you say a slice is finished with no review
 it tells you to start one. The slash commands (`/devsys-start`, `/devsys-plan`, `/devsys-review`)
 are shortcuts to the same tools.
 
-With `codemode` enabled (`"defaultTools": ["+codemode"]` in pi settings), rarely used tools and
-the `judge_*` Jev wrappers are reached through scripts; without it they are declared directly.
+With `codemode` enabled (`"defaultTools": ["+codemode"]` in pi settings), rarely used tools are
+reached through scripts and the `judge_*` Jev wrappers exist for scripts only; without codemode
+the rarely used tools are declared directly and the wrappers are absent.
 
 ## Development
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jwilger/pi-development-system",
-  "version": "0.81.0",
+  "version": "0.83.0",
   "description": "A pi extension package representing a seasoned approach to software development using a full AI SDLC.",
   "keywords": [
     "pi-package"

package/src/context/devsys-tool.ts

@@ -9,7 +9,7 @@ import { phaseGuide } from "./phase-guide.ts";
  * system prompt's state section; calling the tool returns the guidance for it.
  */
 export const DEVSYS_TOOL_DESCRIPTION =
-  "What the development system expects in the current workflow phase (named in the system prompt's Current state), and which tools to use. Call it when unsure what comes next.";
+  "What the development system expects in the current workflow phase (the system prompt names it as of the start of the prompt; this call is live), and which tools to use. Call it when unsure what comes next.";
 
 /** `devsys`: a model-only tool that returns the current phase's guidance. */
 export function createPhaseTool(deps: { state: SessionState }): ToolDefinition {

package/src/context/intent-trigger.ts

@@ -1,6 +1,6 @@
 import type { DevsysState } from "../core/types.ts";
 import type { Jev } from "../jev/client.ts";
-import { type Intent, judgeIntent } from "../jev/questions/intent.ts";
+import { type Intent, judgeIntent, requestText } from "../jev/questions/intent.ts";
 
 /** Probability at or above which an intent is acted on. */
 export const INTENT_AT = 0.7;
@@ -35,7 +35,7 @@ export async function intentLineFor(
   state: DevsysState,
   jev: Jev,
 ): Promise<string | undefined> {
-  if (state.phase !== "idle" || prompt.trim() === "" || jev.availability() === "offline") {
+  if (state.phase !== "idle" || requestText(prompt) === "" || jev.availability() === "offline") {
     return undefined;
   }
   const judged = await judgeIntent(jev, { prompt, phase: state.phase });

package/src/context/system-prompt.ts

@@ -28,7 +28,7 @@ export function buildPrinciplesSection(nonNegotiables: string): string {
  */
 export function buildStateSection(state: DevsysState): string {
   const lines = [
-    "## Current state",
+    "## Workflow state at the start of this prompt (call the devsys tool for the live phase)",
     `- phase: ${state.phase}`,
     `- sizing: ${state.sizing ?? "none"}`,
     `- active slice: ${state.activeSlice ?? "none"}`,

package/src/jev/questions/test-change.ts

@@ -79,7 +79,7 @@ export async function judgeTestChange(
       ? { before: input.before ?? "", after: "" }
       : changeWindow(input.before ?? "", input.after);
   const state = {
-    path: input.path,
+    path: clip(input.path).slice(0, 400),
     before: clip(windowed.before),
     after: input.after === undefined ? "" : clip(windowed.after),
     deleted: input.after === undefined,

package/src/subagents/VENDORED.md

@@ -41,3 +41,4 @@ SOFTWARE.
 6. Empty scope (I5 review): `orch/runtime.ts` `availableScopedModels` treats an empty `/scoped-models` as no restriction (pi's meaning) instead of "matches nothing", because every devsys agent declares `models:` and would otherwise be unspawnable by default. Pinned models are also re-applied on resume (`resolveInitialSettings`), since a restored session may hold only inherited history.
 7. Bundled agents outside `src/subagents`: `agents/researcher.md` (devsys `models:` family list) and `agents/reviewer.md` (devsys `models:` list and packet format; description rewritten; upstream `icon` and `modelSuggestions` removed) differ from upstream; `agents/{advisor,implementer,lens-*}.md` are new. The agent file format itself is unchanged.
 8. Thread cap (I7b.1): `orch/manager.ts` `archiveFinished` replaces the hard "Total thread limit reached" check. At `maxThreads` the oldest finished (completed/failed/stopped) thread without a retained descendant is dropped from the registry to make room; the limit is refused only when every kept thread is live, paused or a parent of one. Upstream kept finished threads forever, so long autonomous runs stalled on their own history. A dropped thread cannot be resumed with `agent_steer` (its session file stays on disk).
+9. Registry writes (OOM fix): `index.ts` `persist` signs the registry with `orch/persist-signature.ts` `registrySignature`, which leaves a starting or running thread's status line and partial output out of the change signature. Upstream wrote the whole registry (every thread, about 12 KB each) on every status change; pi never rewrites session entries, so one session grew to 2.3 GB (5,909 snapshots) and pi died loading it. Writes now happen when a thread is added or removed, starts, settles, pauses or is stopped.

package/src/subagents/index.ts

@@ -1,21 +1,31 @@
-// @ts-nocheck: vendored upstream pi-subagent-manager 0.14.0 compiled under looser options; see src/subagents/VENDORED.md
+// biome-ignore-all lint/suspicious/noUnnecessaryConditions: vendored upstream TUI/config code with no tests; rewriting it risks behaviour (see src/subagents/VENDORED.md)
+// biome-ignore-all lint/suspicious/useAwait: vendored upstream TUI/config code with no tests; rewriting it risks behaviour (see src/subagents/VENDORED.md)
+// biome-ignore-all lint/complexity/noVoid: vendored upstream TUI/config code with no tests; rewriting it risks behaviour (see src/subagents/VENDORED.md)
+// biome-ignore-all lint/style/noNestedTernary: vendored upstream TUI/config code with no tests; rewriting it risks behaviour (see src/subagents/VENDORED.md)
+// biome-ignore-all lint/style/noNonNullAssertion: vendored upstream TUI/config code with no tests; rewriting it risks behaviour (see src/subagents/VENDORED.md)
+// biome-ignore-all lint/complexity/noExcessiveCognitiveComplexity: vendored upstream TUI/config code with no tests; rewriting it risks behaviour (see src/subagents/VENDORED.md)
 import { randomUUID } from "node:crypto";
-import { buildSessionContext, getAgentDir } from "@earendil-works/pi-coding-agent";
 import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import { buildSessionContext, getAgentDir } from "@earendil-works/pi-coding-agent";
 import { fuzzyFilter } from "@earendil-works/pi-tui";
-import { ConfigStore } from "./prefs/config.ts";
-import { IMPORT_REQUEST_PREFIX, markImportOffered, offerAgentImport } from "./prefs/agent-import.ts";
+import { createInheritedToolSource } from "./orch/inherited-tools.ts";
 import { ThreadManager } from "./orch/manager.ts";
+import { registrySignature } from "./orch/persist-signature.ts";
+import { subagentPrompt } from "./orch/prompt.ts";
+import { createDriverFactory } from "./orch/runtime.ts";
+import { agentTools } from "./orch/tools.ts";
+import {
+  IMPORT_REQUEST_PREFIX,
+  markImportOffered,
+  offerAgentImport,
+} from "./prefs/agent-import.ts";
+import { ConfigStore } from "./prefs/config.ts";
 import { loadManagerSettings } from "./prefs/settings.ts";
+import type { SavedThread, ThreadEvent } from "./types.ts";
+import { AgentNavigationEditor } from "./ui/agent-navigation-editor.ts";
 import { configureAgents } from "./ui/settings-ui.ts";
 import { AgentNavigationController, showAgentStatus } from "./ui/status-ui.ts";
-import { AgentNavigationEditor } from "./ui/agent-navigation-editor.ts";
-import { createDriverFactory } from "./orch/runtime.ts";
-import { createInheritedToolSource } from "./orch/inherited-tools.ts";
-import { subagentPrompt } from "./orch/prompt.ts";
-import { agentTools } from "./orch/tools.ts";
 import { editAgentTypes, updateWidget } from "./ui/ui.ts";
-import type { SavedThread, ThreadEvent } from "./types.ts";
 
 const REGISTRY_ENTRY = "pi-subagent:registry:v1";
 const ROOT_MAILBOX_ENTRY = "pi-subagent:root-mailbox:v1";
@@ -81,32 +91,55 @@ export default function piSubagent(pi: ExtensionAPI): void {
     if (existingFactory && existingFactory !== navigationEditorFactory) return;
     // The host only transfers raw text, not cursor/undo/expanded-paste state.
     if (ctx.ui.getEditorText?.()) return;
-    const history = buildSessionContext(ctx.sessionManager.getBranch()).messages.flatMap((message) => {
-      if (message.role !== "user") return [];
-      const content = message.content;
-      return [typeof content === "string" ? content : content
-        .filter((block) => block.type === "text").map((block) => block.text).join("")];
-    });
+    const history = buildSessionContext(ctx.sessionManager.getBranch()).messages.flatMap(
+      (message) => {
+        if (message.role !== "user") return [];
+        const content = message.content;
+        return [
+          typeof content === "string"
+            ? content
+            : content
+                .filter((block) => block.type === "text")
+                .map((block) => block.text)
+                .join(""),
+        ];
+      },
+    );
     navigationEditorFactory = (tui, theme, keys) => {
       const editor = new AgentNavigationEditor(tui, theme, keys, {
         // Initial startup hydrates after session_start; replacement editors do not.
-        initialHistory: seedHistory ? history : undefined,
+        ...(seedHistory ? { initialHistory: history } : {}),
         generation: () => navigationGeneration,
-        canOpen: () => manager !== undefined && context?.mode === "tui" &&
-          limits.subagentMode !== "off" && !navigation.isOpen &&
-          manager.list().some((thread) => thread.path !== "/root" &&
-            (thread.state === "starting" || thread.state === "running")),
-        openTree: () => manager && context
-          ? navigation.open(context, manager.scope("/root"), undefined, limits.nerdFontIcons) : undefined,
-        canCollapse: () => manager !== undefined && context?.mode === "tui" &&
-          limits.subagentMode !== "off" && !navigation.isOpen && !widgetCollapsed &&
-          limits.widgetMode === "full" && widgetThreads().some((thread) => thread.path !== "/root"),
+        canOpen: () =>
+          manager !== undefined &&
+          context?.mode === "tui" &&
+          limits.subagentMode !== "off" &&
+          !navigation.isOpen &&
+          manager
+            .list()
+            .some(
+              (thread) =>
+                thread.path !== "/root" &&
+                (thread.state === "starting" || thread.state === "running"),
+            ),
+        openTree: () =>
+          manager && context
+            ? navigation.open(context, manager.scope("/root"), undefined, limits.nerdFontIcons)
+            : undefined,
+        canCollapse: () =>
+          manager !== undefined &&
+          context?.mode === "tui" &&
+          limits.subagentMode !== "off" &&
+          !navigation.isOpen &&
+          !widgetCollapsed &&
+          limits.widgetMode === "full" &&
+          widgetThreads().some((thread) => thread.path !== "/root"),
         collapseWidget: () => {
           widgetCollapsed = true;
           if (context) refreshWidget(context);
         },
-        onError: (error) => context?.ui.notify(
-          error instanceof Error ? error.message : String(error), "error"),
+        onError: (error) =>
+          context?.ui.notify(error instanceof Error ? error.message : String(error), "error"),
       });
       navigationEditor = editor;
       return editor;
@@ -120,7 +153,10 @@ export default function piSubagent(pi: ExtensionAPI): void {
     // Migration requires the root's file tools and must not delegate source prompts.
     if (limits.subagentMode === "orchestration") {
       if (!firstRun)
-        ctx.ui.notify("Agent import requires Opportunistic mode. Change Subagent Mode in /agents settings.", "info");
+        ctx.ui.notify(
+          "Agent import requires Opportunistic mode. Change Subagent Mode in /agents settings.",
+          "info",
+        );
       return;
     }
     if (importInProgress) return;
@@ -140,12 +176,15 @@ export default function piSubagent(pi: ExtensionAPI): void {
     if (!manager) throw new Error("Subagent threads are not initialized; start a Pi session first");
     return manager;
   };
-  const widgetThreads = () => requireManager().list().filter((thread) => {
-    // Resuming a retained session makes it visible for this task, even after it settles again.
-    if (thread.state === "starting" || thread.state === "running")
-      hiddenWidgetThreads.delete(thread.path);
-    return !hiddenWidgetThreads.has(thread.path);
-  });
+  const widgetThreads = () =>
+    requireManager()
+      .list()
+      .filter((thread) => {
+        // Resuming a retained session makes it visible for this task, even after it settles again.
+        if (thread.state === "starting" || thread.state === "running")
+          hiddenWidgetThreads.delete(thread.path);
+        return !hiddenWidgetThreads.has(thread.path);
+      });
   const requireContext = () => {
     if (!context) throw new Error("No active Pi session");
     return context;
@@ -153,7 +192,7 @@ export default function piSubagent(pi: ExtensionAPI): void {
   const persist = () => {
     if (!manager) return;
     const threads = manager.saved();
-    const signature = JSON.stringify(threads);
+    const signature = registrySignature(threads);
     if (signature !== persistenceSignature) {
       pi.appendEntry(REGISTRY_ENTRY, {
         version: 1,
@@ -187,44 +226,72 @@ export default function piSubagent(pi: ExtensionAPI): void {
   };
   const pendingSummaries = (ctx: ExtensionContext) => {
     const entries = ctx.sessionManager.getBranch();
-    const summarized = new Set(entries.flatMap((entry) => {
-      if (entry.type !== "custom_message" || entry.customType !== ROOT_SUMMARY_MESSAGE) return [];
-      const ids = (entry.details as { mailboxIds?: unknown } | undefined)?.mailboxIds;
-      return Array.isArray(ids) ? ids.filter((id): id is string => typeof id === "string") : [];
-    }));
+    const summarized = new Set(
+      entries.flatMap((entry) => {
+        if (entry.type !== "custom_message" || entry.customType !== ROOT_SUMMARY_MESSAGE) return [];
+        const ids = (entry.details as { mailboxIds?: unknown } | undefined)?.mailboxIds;
+        return Array.isArray(ids) ? ids.filter((id): id is string => typeof id === "string") : [];
+      }),
+    );
     return entries.flatMap((entry) => {
       if (entry.type !== "custom" || entry.customType !== ROOT_MAILBOX_ENTRY) return [];
       const notification = entry.data as RootNotification | undefined;
       return notification?.rootSessionId === ctx.sessionManager.getSessionId() &&
-        notification.finalRecap === true && typeof notification.content === "string" &&
+        notification.finalRecap === true &&
+        typeof notification.content === "string" &&
         typeof notification.details?.mailboxId === "string" &&
-        !summarized.has(notification.details.mailboxId) ? [notification] : [];
+        !summarized.has(notification.details.mailboxId)
+        ? [notification]
+        : [];
     });
   };
   const scheduleSummary = (ctx: ExtensionContext) => {
-    if (!limits.finalRecap || limits.subagentMode === "off" || summaryRunning || suspendingSummaries ||
-        summaryTimer !== undefined || ctx.isIdle() === false || !pendingSummaries(ctx).length) return;
+    if (
+      !limits.finalRecap ||
+      limits.subagentMode === "off" ||
+      summaryRunning ||
+      suspendingSummaries ||
+      summaryTimer !== undefined ||
+      ctx.isIdle() === false ||
+      pendingSummaries(ctx).length === 0
+    )
+      return;
     const currentGeneration = generation;
     // Coalesce siblings finishing together and leave the settlement boundary before
     // starting a new turn. Recheck settings/session/idle state after the deferral.
     summaryTimer = setTimeout(() => {
       summaryTimer = undefined;
-      if (currentGeneration !== generation || context?.sessionManager !== ctx.sessionManager ||
-          !limits.finalRecap || limits.subagentMode === "off" || suspendingSummaries || ctx.isIdle() === false) return;
+      if (
+        currentGeneration !== generation ||
+        context?.sessionManager !== ctx.sessionManager ||
+        !limits.finalRecap ||
+        limits.subagentMode === "off" ||
+        suspendingSummaries ||
+        ctx.isIdle() === false
+      )
+        return;
       const notifications = pendingSummaries(ctx);
-      if (!notifications.length) return;
+      if (notifications.length === 0) return;
       summaryRunning = true;
       try {
-        pi.sendMessage({
-          customType: ROOT_SUMMARY_MESSAGE,
-          content: "Summarize the newly finished asynchronous subagent results for the user. " +
-            "Their results are in the preceding subagent notifications; use agent_output if more detail is needed. " +
-            "Give a concise main-thread answer covering results and any failures. " +
-            "Do not spawn or resume agents solely to summarize.\nAgents: " +
-            [...new Set(notifications.map((notification) => notification.details.path))].join(", "),
-          display: false,
-          details: { mailboxIds: notifications.map((notification) => notification.details.mailboxId) },
-        }, { triggerTurn: true });
+        pi.sendMessage(
+          {
+            customType: ROOT_SUMMARY_MESSAGE,
+            content:
+              "Summarize the newly finished asynchronous subagent results for the user. " +
+              "Their results are in the preceding subagent notifications; use agent_output if more detail is needed. " +
+              "Give a concise main-thread answer covering results and any failures. " +
+              "Do not spawn or resume agents solely to summarize.\nAgents: " +
+              [...new Set(notifications.map((notification) => notification.details.path))].join(
+                ", ",
+              ),
+            display: false,
+            details: {
+              mailboxIds: notifications.map((notification) => notification.details.mailboxId),
+            },
+          },
+          { triggerTurn: true },
+        );
       } catch (error) {
         summaryRunning = false;
         warnDelivery(error);
@@ -278,7 +345,10 @@ export default function piSubagent(pi: ExtensionAPI): void {
           content: message,
           // Foreground results already handled by a busy main thread do not need
           // another turn. Completions during a summary do need a follow-up summary.
-          finalRecap: limits.finalRecap && !suspendingSummaries && event.kind === "settled" &&
+          finalRecap:
+            limits.finalRecap &&
+            !suspendingSummaries &&
+            event.kind === "settled" &&
             (thread.state === "completed" || thread.state === "failed") &&
             (requireContext().isIdle() || summaryRunning),
           details: {
@@ -335,8 +405,14 @@ export default function piSubagent(pi: ExtensionAPI): void {
   const refreshWidget = (ctx: ExtensionContext) => {
     if (limits.subagentMode === "off") {
       if (ctx.hasUI) ctx.ui.setWidget("pi-subagent", undefined);
-    } else updateWidget(ctx, widgetThreads(), widgetCollapsed ? "minimal" : limits.widgetMode,
-      limits.nerdFontIcons, rootTurnEnded);
+    } else
+      updateWidget(
+        ctx,
+        widgetThreads(),
+        widgetCollapsed ? "minimal" : limits.widgetMode,
+        limits.nerdFontIcons,
+        rootTurnEnded,
+      );
   };
   pi.on("agent_start", async (_event, ctx) => {
     // A new task must not resurrect the previous task's settled agents. Automatic
@@ -356,7 +432,7 @@ export default function piSubagent(pi: ExtensionAPI): void {
     // A migration can span clarification turns. Reload after each root turn, without converting files.
     if (limits.subagentMode === "off" || !migrationRequested) return;
     store.reload();
-    if (store.diagnostics.length) context?.ui.notify(store.diagnostics.join("\n"), "warning");
+    if (store.diagnostics.length > 0) context?.ui.notify(store.diagnostics.join("\n"), "warning");
   });
   syncTools();
   pi.on("before_agent_start", async (event, ctx) => {
@@ -369,12 +445,17 @@ export default function piSubagent(pi: ExtensionAPI): void {
         ctx.ui.notify(`Could not save import onboarding state: ${String(error)}`, "warning");
       }
     }
-    return {
-      systemPrompt: `${event.systemPrompt}\n\n${prompt}`,
-    };
+    // A section, not a returned `systemPrompt`: returning one makes pi force the whole prompt as
+    // the head of every request, so any change to our state section would miss the prompt cache.
+    event.systemPromptOptions.sections["pi-subagent"] = prompt;
+    return undefined;
   });
 
-  const attachSession = async (ctx: ExtensionContext, installEditor = false, seedHistory = true) => {
+  const attachSession = async (
+    ctx: ExtensionContext,
+    installEditor = false,
+    seedHistory = true,
+  ) => {
     cancelSummary();
     const token = ++generation;
     closeNavigation(installEditor);
@@ -437,7 +518,7 @@ export default function piSubagent(pi: ExtensionAPI): void {
     restoreRootMailbox(ctx);
     refreshWidget(ctx);
     const diagnostics = [...store.diagnostics, ...settingsDiagnostics];
-    if (limits.subagentMode !== "off" && diagnostics.length)
+    if (limits.subagentMode !== "off" && diagnostics.length > 0)
       ctx.ui.notify(diagnostics.join("\n"), "warning");
   };
   pi.on("session_start", async (event, ctx) => {
@@ -511,10 +592,11 @@ export default function piSubagent(pi: ExtensionAPI): void {
         refreshWidget(ctx);
         restoreRootMailbox(ctx);
         ctx.ui.notify(
-          diagnostics.length
+          diagnostics.length > 0
             ? diagnostics.join("\n")
             : `Loaded ${store.list().length} agent types; maximum ${limits.maxLevels} levels`,
-          diagnostics.length ? "warning" : "info",
+
+          diagnostics.length > 0 ? "warning" : "info",
         );
       } else if (!command || command === "settings") {
         await configureAgents(ctx, {
@@ -527,13 +609,23 @@ export default function piSubagent(pi: ExtensionAPI): void {
             syncTools();
             refreshWidget(ctx);
             restoreRootMailbox(ctx);
-            if (diagnostics.length) ctx.ui.notify(diagnostics.join("\n"), "warning");
+            if (diagnostics.length > 0) ctx.ui.notify(diagnostics.join("\n"), "warning");
           },
         });
       } else if (command === "status") {
-        await showAgentStatus(ctx, requireManager().scope("/root"), undefined, limits.nerdFontIcons);
+        await showAgentStatus(
+          ctx,
+          requireManager().scope("/root"),
+          undefined,
+          limits.nerdFontIcons,
+        );
       } else if (command === "tree") {
-        await navigation.open(ctx, requireManager().scope("/root"), rest.join(" ") || undefined, limits.nerdFontIcons);
+        await navigation.open(
+          ctx,
+          requireManager().scope("/root"),
+          rest.join(" ") || undefined,
+          limits.nerdFontIcons,
+        );
       } else
         ctx.ui.notify(
           "Usage: /agents [tree [path] | status | settings | types | import | reload]",

package/src/subagents/orch/persist-signature.ts

@@ -0,0 +1,24 @@
+import type { SavedThread } from "../types.ts";
+
+const LIVE = new Set(["starting", "running"]);
+
+/**
+ * What decides whether the registry needs writing again.
+ *
+ * pi appends every `appendEntry` to the session file and never rewrites it, and each
+ * registry entry holds every thread (about 12 KB each). A running thread changes its
+ * status line on every tool call, so signing the full snapshot wrote hundreds of
+ * kilobytes per tool call: one session reached 2.3 GB and pi ran out of memory loading
+ * it. A live thread's status and output are not needed to restore it (a reload marks it
+ * interrupted), so they stay out of the signature. Starting, settling, pausing and
+ * adding or removing a thread still write.
+ */
+export function registrySignature(threads: readonly SavedThread[]): string {
+  return JSON.stringify(
+    threads.map((thread) =>
+      LIVE.has(thread.view.state)
+        ? { ...thread, view: { ...thread.view, status: "", output: undefined } }
+        : thread,
+    ),
+  );
+}