@fitchmultz/pi-subagents 0.50.0 → 0.51.0

package/src/agent/notifications.ts

@@ -1,12 +1,12 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
 import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
-import { receiptId } from "./receipts.ts";
+import { completionReceipts } from "./receipts.ts";
 import { readOutputPage } from "../shared/output-page.ts";
-import { isRecord } from "../shared/unknown.ts";
+import { journalStamp } from "../shared/native-journal.ts";
 import type { Launch, Result } from "./contracts.ts";
 import { launchAt, list, runDirectory } from "./storage.ts";
-import { readResult } from "./usage.ts";
+import { readCanonicalResult, readResult } from "./usage.ts";
 
 function queuedAdmissions(): Set<string> {
   const store = globalThis as Record<string, unknown>;
@@ -25,7 +25,7 @@ function queuedAdmissions(): Set<string> {
   return queued;
 }
 const admissions = queuedAdmissions();
-export function hold(owner: string, id: string): boolean {
+function hold(owner: string, id: string): boolean {
   const key = `${owner}/${id}`;
   if (admissions.has(key)) {
     return false;
@@ -39,93 +39,218 @@ export function release(owner: string, id: string): void {
 export function isHeld(owner: string, id: string): boolean {
   return admissions.has(`${owner}/${id}`);
 }
-export function settle(owner: string, pending: boolean): void {
+/** Returns whether any admission was released for reconciliation. */
+export function settle(owner: string, pending: boolean): boolean {
   if (pending) {
-    return;
+    return false;
   }
+  let released = false;
   for (const key of admissions) {
     if (key.startsWith(`${owner}/`)) {
       admissions.delete(key);
+      released = true;
     }
   }
+  return released;
 }
-function matches(value: unknown, launch: Launch): boolean {
-  return (
-    isRecord(value) &&
-    value.owner === launch.owner &&
-    value.id === launch.id &&
-    value.completed === true
-  );
+export function published(ctx: ExtensionContext, launch: Launch): string | undefined {
+  return completionReceipts(ctx.sessionManager.getSessionFile(), launch.owner).get(launch.id);
 }
-function toolReceipt(message: unknown, launch: Launch): boolean {
-  if (!isRecord(message) || message.role !== "toolResult") {
-    return false;
-  }
-  return (
-    (message.toolName === "delegate" || message.toolName === "agent_runs") &&
-    isRecord(message.details) &&
-    message.details.receivedResult === true &&
-    matches(message.details, launch)
-  );
-}
-function receipt(body: unknown, launch: Launch): boolean {
-  if (!isRecord(body)) {
-    return false;
-  }
-  if (body.type === "custom_message" && body.customType === "agent-completion") {
-    return matches(body.details, launch);
-  }
-  return body.type === "message" && toolReceipt(body.message, launch);
+export interface Completion {
+  readonly owner: string;
+  readonly id: string;
+  readonly message: Parameters<ExtensionAPI["sendMessage"]>[0];
 }
-export function published(ctx: ExtensionContext, launch: Launch): string | undefined {
-  return receiptId(ctx.sessionManager.getSessionFile(), launch.owner, (body) =>
-    receipt(body, launch),
-  );
+export function deliver(
+  pi: ExtensionAPI,
+  ctx: ExtensionContext,
+  notices: readonly Completion[],
+): void {
+  const idle = ctx.isIdle();
+  for (const [index, notice] of notices.entries()) {
+    hold(notice.owner, notice.id);
+    try {
+      // Pi defers settlement-triggered runs separately. Publish the idle batch first,
+      // then trigger once so no unstarted wakeup can outlive its admission hold.
+      pi.sendMessage(notice.message, {
+        triggerTurn: !idle || index === notices.length - 1,
+        deliverAs: "steer",
+      });
+    } catch (error) {
+      release(notice.owner, notice.id);
+      throw error;
+    }
+  }
 }
-function send(pi: ExtensionAPI, base: string, launch: Launch, result: Result): void {
+function notification(base: string, launch: Launch, result: Result): Completion {
   const file = path.join(runDirectory(base, launch.id), "output.txt");
   const page = fs.existsSync(file) ? readOutputPage(file, { offset: 0, length: 32768 }) : undefined;
   const more =
     page?.nextOffset !== undefined
       ? `\n[More output: agent_runs inspect id=${launch.id} offset=${page.nextOffset}; ${page.size} bytes total.]`
       : "";
-  hold(launch.owner, launch.id);
-  try {
-    pi.sendMessage(
-      {
-        customType: "agent-completion",
-        display: true,
-        content: `Agent run ${launch.id}: ${result.status}. ${result.error ?? ""}\n${page?.text ?? ""}${more}\nNative journal: ${launch.sessionFile}`,
-        details: {
-          owner: launch.owner,
-          id: launch.id,
-          completed: true,
-          result,
-          outputFile: file,
-          page,
-        },
+  return {
+    owner: launch.owner,
+    id: launch.id,
+    message: {
+      customType: "agent-completion",
+      display: true,
+      content: `Agent run ${launch.id}: ${result.status}. ${result.error ?? ""}\n${page?.text ?? ""}${more}\nNative journal: ${launch.sessionFile}`,
+      details: {
+        owner: launch.owner,
+        id: launch.id,
+        completed: true,
+        result,
+        outputFile: file,
+        page,
       },
-      { triggerTurn: true, deliverAs: "followUp" },
-    );
-  } catch (error) {
-    admissions.delete(`${launch.owner}/${launch.id}`);
-    throw error;
+    },
+  };
+}
+interface PendingJournal {
+  readonly file: string;
+  readonly stamp: string | undefined;
+}
+interface Loaded {
+  readonly launch: Launch;
+  readonly result: Result | undefined;
+  readonly journal?: PendingJournal;
+}
+interface Settled {
+  readonly launch: string;
+  readonly result: string;
+  readonly pending?: Loaded;
+  readonly endedAt: number;
+  readonly createdAt: number;
+}
+interface Completed {
+  readonly id: string;
+  readonly endedAt: number;
+  readonly createdAt: number;
+  readonly loaded?: Loaded;
+}
+// Validated settled runs by file stamp, so polling does not reparse unchanged history.
+const settledRuns = new Map<string, Map<string, Settled>>();
+function stamp(file: string): string | undefined {
+  const stat = fs.statSync(file, { bigint: true, throwIfNoEntry: false });
+  return stat === undefined ? undefined : journalStamp(stat);
+}
+function pendingJournal(file: string): PendingJournal | undefined {
+  try {
+    return { file, stamp: stamp(file) };
+  } catch {
+    // Unavailable metadata cannot validate a cached attempt or hide execution.
+    return undefined;
   }
 }
-export function notify(pi: ExtensionAPI, ctx: ExtensionContext, base: string, owner: string): void {
-  for (const id of list(base)) {
-    const launch = launchAt(base, id, owner);
-    const result = readResult(base, launch);
-    if (!result) {
-      continue;
+function unchangedPending(pending: Loaded | undefined): boolean {
+  if (pending === undefined) {
+    return true;
+  }
+  const journal = pending.journal;
+  if (journal === undefined) {
+    return false;
+  }
+  const current = pendingJournal(journal.file);
+  return current !== undefined && current.stamp === journal.stamp;
+}
+function load(base: string, id: string, owner: string): Loaded {
+  const launch = launchAt(base, id, owner);
+  const result = readCanonicalResult(base, launch);
+  if (result?.accountingPending === undefined) {
+    return { launch, result };
+  }
+  // Only pending runs need this pre-read stamp to observe concurrent publication.
+  // readResult revalidates canonical state afterward: a miss intentionally reads result.json twice.
+  const journal = pendingJournal(launch.sessionFile);
+  return { launch, result: readResult(base, launch), journal };
+}
+function cacheableResult(loaded: Loaded): boolean {
+  return loaded.result?.accountingPending === undefined || loaded.journal !== undefined;
+}
+interface Scan {
+  readonly base: string;
+  readonly owner: string;
+  readonly known: ReadonlyMap<string, Settled>;
+  readonly settled: Map<string, Settled>;
+}
+function completedRun({ base, owner, known, settled }: Scan, id: string): Completed | undefined {
+  const directory = runDirectory(base, id);
+  const launch = stamp(path.join(directory, "launch.json"));
+  const result = stamp(path.join(directory, "result.json"));
+  const prior = known.get(id);
+  if (
+    prior !== undefined &&
+    prior.launch === launch &&
+    prior.result === result &&
+    unchangedPending(prior.pending)
+  ) {
+    settled.set(id, prior);
+    return { id, ...prior, loaded: prior.pending };
+  }
+  const loaded = load(base, id, owner);
+  if (loaded.result === undefined) {
+    return;
+  }
+  const order = { endedAt: loaded.result.endedAt, createdAt: loaded.launch.createdAt };
+  // Pending accounting retries on child changes, even after the completion receipt.
+  // Ordinary settled history retains its launch/result-only idle probes.
+  const pending = loaded.result.accountingPending === undefined ? undefined : loaded;
+  if (launch !== undefined && result !== undefined && cacheableResult(loaded)) {
+    settled.set(id, { launch, result, pending, ...order });
+  }
+  return { id, ...order, loaded };
+}
+function completed(base: string, owner: string, awaited: Set<string>): Completed[] {
+  const key = JSON.stringify([base, owner]);
+  const scan = {
+    base,
+    owner,
+    known: settledRuns.get(key) ?? new Map<string, Settled>(),
+    settled: new Map<string, Settled>(),
+  };
+  const runs = list(base).flatMap((id) => {
+    const run = completedRun(scan, id);
+    if (run === undefined) {
+      awaited.add(path.join(runDirectory(base, id), "result.json"));
+    } else if (run.loaded?.result?.accountingPending !== undefined) {
+      awaited.add(run.loaded.launch.sessionFile);
     }
-    const key = `${owner}/${id}`;
-    if (published(ctx, launch) !== undefined) {
+    return run ?? [];
+  });
+  settledRuns.delete(key);
+  settledRuns.set(key, scan.settled);
+  if (settledRuns.size > 8) {
+    settledRuns.delete(settledRuns.keys().next().value ?? key);
+  }
+  return runs.sort((a, b) => {
+    if (a.endedAt !== b.endedAt) {
+      return a.endedAt - b.endedAt;
+    }
+    return a.createdAt !== b.createdAt ? a.createdAt - b.createdAt : a.id.localeCompare(b.id);
+  });
+}
+export interface Reconciliation {
+  readonly notices: Completion[];
+  /** Canonical files whose publication can change what this owner must deliver. */
+  readonly awaited: ReadonlySet<string>;
+}
+export function notifications(ctx: ExtensionContext, base: string, owner: string): Reconciliation {
+  const notices: Completion[] = [];
+  const awaited = new Set<string>();
+  const runs = completed(base, owner, awaited);
+  const receipts =
+    runs.length === 0 ? undefined : completionReceipts(ctx.sessionManager.getSessionFile(), owner);
+  for (const run of runs) {
+    const key = `${owner}/${run.id}`;
+    if (receipts?.get(run.id) !== undefined) {
       admissions.delete(key);
       continue;
     }
-    if (!admissions.has(key)) {
-      send(pi, base, launch, result);
+    const loaded = admissions.has(key) ? undefined : (run.loaded ?? load(base, run.id, owner));
+    if (loaded?.result !== undefined) {
+      notices.push(notification(base, loaded.launch, loaded.result));
     }
   }
+  return { notices, awaited };
 }

package/src/agent/observation-state.ts

@@ -0,0 +1,277 @@
+import type {
+  AgentSession,
+  AgentSessionEvent,
+  SessionMessageEntry,
+  ToolExecutionEndEvent,
+  ToolExecutionStartEvent,
+  ToolExecutionUpdateEvent,
+} from "@earendil-works/pi-coding-agent";
+import type { AssistantMessage } from "@earendil-works/pi-ai";
+import { boundedJson } from "../shared/bounded-json.ts";
+
+const VALUE_BUDGET = { bytes: 64 * 1024, nodes: 4096, depth: 32 };
+/** Longer native identifiers are left to the journal rather than keyed live. */
+export const NAME_CHARS = 1024;
+const RETAINED = { settlements: 128, tools: 256 };
+
+type MessageEvent = Extract<
+  AgentSessionEvent,
+  { type: "message_start" | "message_update" | "message_end" }
+>;
+export type ToolEvent = ToolExecutionStartEvent | ToolExecutionUpdateEvent | ToolExecutionEndEvent;
+const WRITING = { text: "writing", thinking: "thinking", toolCall: "calling" } as const;
+
+/** A live in-flight assistant message: a transient native reference, never a receipt. */
+export interface LiveMessage {
+  readonly key: number;
+  message: AssistantMessage;
+  ended: boolean;
+}
+/** A native message found in the session tree: `[seq, entry id, live message key, tool call id]`. */
+export interface Settlement {
+  readonly seq: number;
+  readonly json: string;
+}
+
+export class LiveTool {
+  partial?: unknown;
+  rev = 0;
+  ended?: { readonly isError: boolean; readonly result: unknown };
+  readonly id: string;
+  readonly name: string;
+  readonly args: unknown;
+  private argsJson?: string;
+  private partialJson?: { readonly rev: number; readonly json: string };
+  private resultJson?: string;
+  constructor(id: string, name: string, args: unknown) {
+    this.id = id;
+    this.name = name;
+    this.args = args;
+  }
+  /** Replaceable progress: only the latest reference is projected at the next flush. */
+  progress(value: unknown): void {
+    this.partial = value;
+    this.rev++;
+  }
+  projectedArgs(project: (value: unknown) => string): string {
+    this.argsJson ??= project(this.args);
+    return this.argsJson;
+  }
+  projectedPartial(project: (value: unknown) => string): string {
+    if (this.partialJson?.rev !== this.rev) {
+      this.partialJson = { rev: this.rev, json: project(this.partial) };
+    }
+    return this.partialJson.json;
+  }
+  projectedResult(project: (value: unknown) => string): string {
+    this.resultJson ??= project(this.ended?.result);
+    return this.resultJson;
+  }
+}
+
+/**
+ * Single-writer actor state. Sparse tool hooks always keep the tool table, because native state
+ * retains neither progress, unpublished parallel results nor nested calls. Token-rate message
+ * events arrive only while a detail viewer observes; attaching seeds from public native state.
+ */
+export class LiveState {
+  version = 0;
+  activityVersion = 0;
+  message?: LiveMessage;
+  readonly tools = new Map<string, LiveTool>();
+  readonly settlements: Settlement[] = [];
+  settled = 0;
+  /** Live items dropped by a retention bound or a branch change; viewers resynchronize. */
+  evicted = 0;
+  events = 0;
+  toolEvents = 0;
+  progressUpdates = 0;
+  projections = 0;
+  unavailable = 0;
+  private observing = false;
+  private nextKey = 0;
+  private queued = false;
+  private closed = false;
+  private watermark: string | null;
+  private readonly session: AgentSession;
+  private readonly changed: () => void;
+  private readonly failed: (error: unknown) => void;
+  constructor(session: AgentSession, changed: () => void, failed: (error: unknown) => void) {
+    this.session = session;
+    this.changed = changed;
+    this.failed = failed;
+    this.watermark = session.sessionManager.getLeafId();
+  }
+  /** Start or stop the transient message view; tools persist across viewers. */
+  observe(on: boolean): void {
+    this.resolve();
+    this.observing = on;
+    this.message = undefined;
+    this.settlements.length = 0;
+    if (on) {
+      const streaming = this.session.state.streamingMessage;
+      if (streaming?.role === "assistant") {
+        this.assistant("message_start", streaming);
+      }
+    }
+    this.version++;
+    this.activityVersion++;
+  }
+  apply(event: AgentSessionEvent): void {
+    this.events++;
+    if (
+      event.type === "message_start" ||
+      event.type === "message_update" ||
+      event.type === "message_end"
+    ) {
+      this.messageEvent(event);
+    }
+  }
+  private messageEvent(event: MessageEvent): void {
+    if (event.message.role === "assistant") {
+      this.assistant(event.type, event.message);
+    }
+    if (event.type === "message_end" && !this.queued) {
+      this.queued = true;
+      // Native message_end listeners run just before the same job appends the entry.
+      queueMicrotask(this.settle);
+    }
+  }
+  private assistant(type: MessageEvent["type"], message: AssistantMessage): void {
+    let live = this.message;
+    if (type === "message_start" || live === undefined) {
+      // A previous message still here was never found in the session tree.
+      this.evicted += live === undefined ? 0 : 1;
+      live = { key: ++this.nextKey, message, ended: false };
+      this.message = live;
+    }
+    live.message = message;
+    live.ended = type === "message_end";
+    this.version++;
+  }
+  tool(event: ToolEvent): void {
+    this.toolEvents++;
+    this.version++;
+    if (event.type === "tool_execution_start") {
+      this.resolve();
+      this.startTool(event.toolCallId, event.toolName, event.args);
+      return;
+    }
+    const tool = this.tools.get(event.toolCallId);
+    if (event.type === "tool_execution_update") {
+      tool?.progress(event.partialResult);
+      this.progressUpdates++;
+    } else if (tool) {
+      tool.ended = { isError: event.isError, result: event.result };
+      this.activityVersion++;
+    }
+  }
+  private startTool(id: string, name: string, args: unknown): void {
+    if (id.length > NAME_CHARS) {
+      this.unavailable++;
+      return;
+    }
+    const oldest = this.tools.size >= RETAINED.tools ? this.tools.keys().next() : undefined;
+    if (oldest?.done === false) {
+      this.tools.delete(oldest.value);
+      this.evicted++;
+    }
+    this.tools.set(id, new LiveTool(id, name, args));
+    this.activityVersion++;
+  }
+  private readonly settle = (): void => {
+    this.queued = false;
+    try {
+      if (!this.closed && this.resolve()) {
+        this.changed();
+      }
+    } catch (error) {
+      this.failed(error);
+    }
+  };
+  /**
+   * Hand native messages appended since the last walk over to their session entries. A live
+   * message without an entry is unresolved publication and stays live. Returns whether anything
+   * changed.
+   */
+  resolve(): boolean {
+    const { appended, departed } = this.appended();
+    this.evicted += departed && this.observing ? 1 : 0;
+    for (const entry of appended) {
+      this.publish(entry, !departed && this.observing);
+    }
+    this.settlements.splice(0, Math.max(0, this.settlements.length - RETAINED.settlements));
+    const changed = appended.length > 0;
+    this.version += changed ? 1 : 0;
+    this.activityVersion += changed ? 1 : 0;
+    return changed;
+  }
+  /** Message entries since the previous walk, in journal order; each append is visited once. */
+  private appended(): { appended: SessionMessageEntry[]; departed: boolean } {
+    const manager = this.session.sessionManager;
+    const appended: SessionMessageEntry[] = [];
+    let entry = manager.getLeafEntry();
+    const leaf = entry?.id ?? null;
+    while (entry !== undefined && entry.id !== this.watermark) {
+      if (entry.type === "message") {
+        appended.push(entry);
+      }
+      entry = entry.parentId === null ? undefined : manager.getEntry(entry.parentId);
+    }
+    // A leaf off the previous branch walked to the root once; its entries are not new.
+    const departed = entry === undefined && this.watermark !== null;
+    this.watermark = leaf;
+    return { appended: appended.reverse(), departed };
+  }
+  private publish({ id, message }: SessionMessageEntry, settle: boolean): void {
+    const call = message.role === "toolResult" ? message.toolCallId : "";
+    const tool = call.length > 0 && call.length <= NAME_CHARS ? call : null;
+    // A published result also retires the calls it made through ctx.executeTool().
+    const retired = (key: string): boolean =>
+      tool !== null && (key === tool || key.startsWith(`${tool}/`));
+    for (const live of [...this.tools.keys()].filter(retired)) {
+      this.tools.delete(live);
+    }
+    const key = this.message?.message === message ? this.message.key : null;
+    this.message = key === null ? this.message : undefined;
+    if (settle) {
+      const seq = ++this.settled;
+      this.settlements.push({ seq, json: JSON.stringify([seq, id, key, tool]) });
+    }
+  }
+  dispose(): void {
+    this.closed = true;
+  }
+  readonly project = (value: unknown): string => {
+    this.projections++;
+    const result = boundedJson(value, VALUE_BUDGET);
+    if ("json" in result) {
+      return `{"json":${result.json}}`;
+    }
+    this.unavailable++;
+    return `{"unavailable":"${result.unavailable}"}`;
+  };
+  /** Sampled from public native state, so compact peers never need token-rate events. */
+  activity(busy: boolean): string {
+    const { session } = this;
+    const running = [...this.tools.values()].filter((tool) => tool.ended === undefined);
+    const streaming = session.state.streamingMessage;
+    const content = streaming?.role === "assistant" ? streaming.content : undefined;
+    const last = content?.at(-1)?.type;
+    const phases: Array<[boolean, string]> = [
+      [running.length > 0, "tool"],
+      [content !== undefined, last === undefined ? "responding" : WRITING[last]],
+      [session.isCompacting, "compacting"],
+      [session.isRetrying, "retrying"],
+      [busy || session.isStreaming, "working"],
+    ];
+    const phase = phases.find(([active]) => active)?.[1] ?? "idle";
+    return JSON.stringify({
+      phase,
+      tools: running.slice(0, 3).map((tool) => tool.name.slice(0, 64)),
+      running: running.length,
+      model: this.session.model?.id,
+      thinking: this.session.thinkingLevel,
+    });
+  }
+}