@fitchmultz/pi-subagents 0.50.0 → 0.51.0

package/src/agent/journal-tail.ts

@@ -1,14 +1,10 @@
 import * as fs from "node:fs";
+import { NativeJournal, type NativeRecord } from "../shared/native-journal.ts";
+import { nativeProjection } from "../shared/journal-projections.ts";
+import type { Projection } from "../shared/json-projection.ts";
+import { isRecord, recordAt, type UnknownRecord } from "../shared/unknown.ts";
 
-const READ_BYTES = 65536;
-const LINE_BYTES = 32768;
-const KEEP_LINES = 80;
-
-function record(value: unknown): value is Record<string, unknown> {
-  return typeof value === "object" && value !== null && !Array.isArray(value);
-}
 export function plainText(value: string, length = 512): string {
-  // Journal content is untrusted terminal input, never terminal control sequences.
   return Array.from(value.slice(0, length))
     .map((char) => {
       const code = char.codePointAt(0) ?? 0;
@@ -16,169 +12,161 @@ export function plainText(value: string, length = 512): string {
     })
     .join("");
 }
-function contentLines(content: unknown): string[] {
-  if (typeof content === "string") {
-    return [plainText(content)];
+const PAGE_CHARS = 32 * 1024;
+const callFields = new Set(["type", "id", "name"]);
+const textFields = new Set(["text", "thinking"]);
+const metadata: Projection = (keys, root) => {
+  if (
+    keys[0] === "message" &&
+    keys[1] === "content" &&
+    keys.length > 3 &&
+    !callFields.has(String(keys[3]))
+  ) {
+    return false;
   }
-  if (!Array.isArray(content)) {
-    return [];
+  return nativeProjection(keys, root);
+};
+function contentIndex(keys: readonly (string | number)[]): number | undefined {
+  if (keys[0] !== "message" || keys[1] !== "content") {
+    return undefined;
   }
-  return content.slice(0, 12).flatMap((block: unknown) => {
-    if (!record(block)) {
-      return [];
-    }
-    if (block.type === "text" && typeof block.text === "string") {
-      return [plainText(block.text)];
+  if (keys.length === 2) {
+    return -1;
+  }
+  return typeof keys[2] === "number" && textFields.has(String(keys[3])) ? keys[2] : undefined;
+}
+class TextPage {
+  private chars = 0;
+  private carry?: { index: number; text: string };
+  private readonly text = new Map<number, string>();
+  private readonly start: number;
+  constructor(page: number) {
+    this.start = page * PAGE_CHARS;
+  }
+  add(index: number, chunk: string): void {
+    const previous = this.carry;
+    this.carry = undefined;
+    if (previous && previous.index !== index) {
+      this.take(previous.index, previous.text);
     }
-    if (block.type === "toolCall" && typeof block.name === "string") {
-      return [`tool ${plainText(block.name, 80)} ${plainText(JSON.stringify(block.arguments))}`];
+    const text = previous?.index === index ? previous.text + chunk : chunk;
+    const last = text.charCodeAt(text.length - 1);
+    const pending = last >= 0xd800 && last <= 0xdbff;
+    this.take(index, pending ? text.slice(0, -1) : text);
+    if (pending) {
+      this.carry = { index, text: text.slice(-1) };
     }
-    return block.type === "thinking"
-      ? []
-      : [`[${typeof block.type === "string" ? plainText(block.type, 40) : "non-text"} omitted]`];
-  });
-}
-function activity(value: unknown): string[] {
-  if (!record(value) || value.type !== "message" || !record(value.message)) {
-    return [];
   }
-  const message = value.message;
-  if (message.role === "assistant") {
-    const lines = contentLines(message.content).map((line) => `assistant: ${line}`);
-    if (typeof message.errorMessage === "string") {
-      lines.push(`assistant error: ${plainText(message.errorMessage)}`);
+  private take(index: number, text: string): void {
+    const points = Array.from(text);
+    const from = Math.max(0, this.start - this.chars);
+    const to = Math.min(points.length, this.start + PAGE_CHARS - this.chars);
+    if (to > from) {
+      this.text.set(index, (this.text.get(index) ?? "") + points.slice(from, to).join(""));
     }
-    return lines;
+    this.chars += points.length;
   }
-  if (message.role === "toolResult") {
-    const label = message.isError === true ? "tool error" : "tool result";
-    const name = typeof message.toolName === "string" ? plainText(message.toolName, 80) : "unknown";
-    return [`${label} ${name}:`, ...contentLines(message.content)];
+  finish(value: UnknownRecord, page: number): ConversationPage {
+    if (this.carry) {
+      this.take(this.carry.index, this.carry.text);
+    }
+    const message = { ...recordAt(value, "message") };
+    if (typeof message.content === "string") {
+      message.content = this.text.get(-1) ?? "";
+    }
+    if (Array.isArray(message.content)) {
+      for (const [index, block] of (message.content as unknown[]).entries()) {
+        if (isRecord(block) && typeof block.type === "string" && textFields.has(block.type)) {
+          block[block.type] = this.text.get(index) ?? "";
+        }
+      }
+    }
+    return { message, page, pages: Math.max(1, Math.ceil(this.chars / PAGE_CHARS)) };
   }
-  return [];
+}
+export interface ConversationPage {
+  message: UnknownRecord;
+  page: number;
+  pages: number;
 }
 
-/** A lossy display tail; the native journal remains the complete source of truth. */
-export class JournalTail {
-  readonly lines: string[] = [];
-  notice = "Waiting for published native activity.";
-  private offset = 0;
-  private partial: Buffer = Buffer.alloc(0);
-  private dropping = false;
-  private identity = "";
-  private stamp = 0;
-  private readonly file: string;
-  private readonly sessionId: string;
+/** Native indexing owns ancestry and publication; only requested message pages are hydrated. */
+export class ConversationJournal {
+  readonly native: NativeJournal;
+  private branch?: { length: number; leaf: string | null | undefined; entries: NativeRecord[] };
   constructor(file: string, sessionId: string) {
-    this.file = file;
-    this.sessionId = sessionId;
-  }
-  private reset(size: number): void {
-    this.lines.length = 0;
-    this.offset = 0;
-    this.partial = Buffer.alloc(0);
-    this.dropping = false;
-    if (size > READ_BYTES) {
-      this.skip(size);
-    }
-  }
-  private skip(size: number): void {
-    this.offset = size - READ_BYTES;
-    this.partial = Buffer.alloc(0);
-    this.dropping = true;
-    this.lines.push("[Earlier native activity omitted; full journal retained.]");
-  }
-  private header(fd: number): void {
-    const head = Buffer.alloc(4096);
-    const count = fs.readSync(fd, head, 0, head.length, 0);
-    const lf = head.indexOf(10, 0);
-    if (lf < 0 || lf >= count) {
-      throw new Error("Native journal header is incomplete or too large.");
+    const stat = fs.lstatSync(file);
+    if (!stat.isFile() || stat.isSymbolicLink()) {
+      throw new Error("Child journal is not a regular file.");
     }
-    const value: unknown = JSON.parse(head.subarray(0, lf).toString("utf8"));
-    if (!record(value) || value.type !== "session" || value.id !== this.sessionId) {
+    this.native = new NativeJournal(file, "live", true, {
+      readBytes: 256 * 1024,
+      select: metadata,
+    });
+    if (this.native.records[0]?.value.id !== sessionId) {
       throw new Error("Native journal identity does not match this child.");
     }
   }
-  private line(line: Buffer): void {
-    if (line.length > LINE_BYTES) {
-      this.lines.push("[Large native entry omitted; read the full journal.]");
-      return;
-    }
-    try {
-      const value: unknown = JSON.parse(line.toString("utf8"));
-      this.lines.push(...activity(value));
-    } catch {
-      this.lines.push("[Malformed published native entry omitted.]");
+  entries(requested?: string | null): NativeRecord[] {
+    const leaf =
+      typeof requested === "string" && !this.native.byId.has(requested) ? undefined : requested;
+    if (this.branch?.length !== this.native.records.length || this.branch.leaf !== leaf) {
+      this.branch = {
+        length: this.native.records.length,
+        leaf,
+        entries: this.native.branch(leaf).filter((entry) => entry.value.type === "message"),
+      };
     }
+    return this.branch.entries;
   }
-  private consume(chunk: Buffer): void {
-    const bytes = Buffer.concat([this.partial, chunk]);
-    let start = 0;
-    let lf = bytes.indexOf(10);
-    while (lf >= 0) {
-      if (!this.dropping) {
-        this.line(bytes.subarray(start, lf));
-      }
-      this.dropping = false;
-      start = lf + 1;
-      lf = bytes.indexOf(10, start);
-    }
-    this.partial = bytes.subarray(start);
-    if (this.partial.length > LINE_BYTES) {
-      this.partial = Buffer.alloc(0);
-      this.dropping = true;
-      this.lines.push("[Large native entry omitted; read the full journal.]");
-    }
-    this.lines.splice(0, Math.max(0, this.lines.length - KEEP_LINES));
-    if (this.partial.length > 0) {
-      this.notice = "Waiting for newline publication of the latest native entry.";
-    }
+  async call(id: string, result: string, signal?: AbortSignal): Promise<UnknownRecord | undefined> {
+    const contains = (entry: NativeRecord) => {
+      const content = recordAt(entry.value, "message")?.content;
+      return (
+        Array.isArray(content) &&
+        content.some(
+          (part: unknown) => isRecord(part) && part.type === "toolCall" && part.id === id,
+        )
+      );
+    };
+    const entries = this.branch?.entries ?? [];
+    const entry = entries
+      .slice(
+        0,
+        entries.findIndex((item) => item.value.id === result),
+      )
+      .findLast(contains);
+    const content = entry && (await this.page(entry, 0, { signal })).message.content;
+    return Array.isArray(content)
+      ? content.find(
+          (part: unknown): part is UnknownRecord =>
+            isRecord(part) && part.type === "toolCall" && part.id === id,
+        )
+      : undefined;
   }
-  private changed(stat: fs.Stats, identity: string): boolean {
-    return (
-      identity !== this.identity ||
-      stat.size < this.offset ||
-      (stat.size === this.offset && stat.mtimeMs !== this.stamp)
+  async page(
+    entry: NativeRecord,
+    page = 0,
+    options: { signal?: AbortSignal; hideThinking?: boolean } = {},
+  ): Promise<ConversationPage> {
+    const text = new TextPage(page);
+    const value = await this.native.project(
+      entry,
+      (keys, root) => {
+        if (contentIndex(keys) !== undefined && keys.length > 2) {
+          return 0;
+        }
+        return nativeProjection(keys, root);
+      },
+      (keys, chunk) => {
+        const index = contentIndex(keys);
+        if (index === undefined || (options.hideThinking === true && keys[3] === "thinking")) {
+          return;
+        }
+        text.add(index, chunk);
+      },
+      options.signal,
     );
-  }
-  private observe(fd: number, stat: fs.Stats): void {
-    const identity = `${stat.dev}:${stat.ino}`;
-    if (this.changed(stat, identity)) {
-      this.reset(stat.size);
-    } else if (stat.size - this.offset > READ_BYTES) {
-      this.skip(stat.size);
-    }
-    this.identity = identity;
-    this.stamp = stat.mtimeMs;
-    const chunk = Buffer.alloc(Math.min(READ_BYTES, stat.size - this.offset));
-    const count = fs.readSync(fd, chunk, 0, chunk.length, this.offset);
-    this.offset += count;
-    this.notice =
-      this.offset > READ_BYTES ? "Recent journal tail only; full journal retained." : "";
-    this.consume(chunk.subarray(0, count));
-    if (this.lines.length === 0 && this.notice.length === 0) {
-      this.notice = "Waiting for published native activity.";
-    }
-  }
-  poll(): void {
-    let fd: number | undefined;
-    try {
-      fd = fs.openSync(this.file, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW);
-      const stat = fs.fstatSync(fd);
-      if (!stat.isFile()) {
-        throw new Error("Native journal is not a regular file.");
-      }
-      this.header(fd);
-      this.observe(fd, stat);
-    } catch (error) {
-      this.lines.length = 0;
-      this.identity = "";
-      this.notice = error instanceof Error ? plainText(error.message) : "Journal unavailable.";
-    } finally {
-      if (fd !== undefined) {
-        fs.closeSync(fd);
-      }
-    }
+    return text.finish(value, page);
   }
 }

package/src/agent/launch.ts

@@ -3,16 +3,22 @@ import * as path from "node:path";
 import { randomUUID } from "node:crypto";
 import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
 import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
-import type { Launch, DelegateInput } from "./contracts.ts";
+import type { Launch } from "./contracts.ts";
+import type { DelegateInput } from "./tool-schemas.ts";
 import { loadSDK, packageRoot, runtimeEntry } from "./host.ts";
 import { runDirectory, write } from "./storage.ts";
 
-function selectedModel(input: DelegateInput, context: ExtensionContext) {
+function selectedModel(
+  input: DelegateInput,
+  context: ExtensionContext,
+  thinking: ThinkingLevel,
+  sdk: Awaited<ReturnType<typeof loadSDK>>,
+) {
   if (input.model === undefined) {
     if (!context.model) {
       throw new Error("No selected model; no fallback was used.");
     }
-    return context.model;
+    return { model: context.model, thinking };
   }
   const slash = input.model.indexOf("/");
   if (slash < 1) {
@@ -22,10 +28,28 @@ function selectedModel(input: DelegateInput, context: ExtensionContext) {
     input.model.slice(0, slash),
     input.model.slice(slash + 1),
   );
-  if (!model) {
+  if (model) {
+    return { model, thinking };
+  }
+  if (typeof sdk.resolveModelReference !== "function") {
+    throw new Error(
+      `Model ${input.model} is unavailable; no fallback was used. Check the provider/model ID. This host supports literal IDs only. If you intended a thinking suffix, update Pi for native thinking-suffix support.`,
+    );
+  }
+  const resolved = sdk.resolveModelReference(
+    {
+      text: input.model,
+      provider: input.model.slice(0, slash),
+      thinkingSuffix: "strict",
+      fuzzy: false,
+    },
+    context.modelRegistry.getAll(),
+    () => true,
+  );
+  if (resolved.status !== "resolved") {
     throw new Error(`Model ${input.model} is unavailable; no fallback was used.`);
   }
-  return model;
+  return { model: resolved.model, thinking: resolved.thinkingLevel ?? thinking };
 }
 function targetTrust(
   sdk: Awaited<ReturnType<typeof loadSDK>>,
@@ -63,7 +87,7 @@ export async function prepareLaunch(options: {
   if (!fs.statSync(cwd).isDirectory()) {
     throw new Error("Delegation cwd is not a directory.");
   }
-  const model = selectedModel(input, context);
+  const selection = selectedModel(input, context, thinking, sdk);
   const trusted = targetTrust(sdk, cwd, context, agentDir);
   const id = randomUUID();
   const directory = runDirectory(base, id);
@@ -98,9 +122,9 @@ export async function prepareLaunch(options: {
     sessionId: manager.getSessionId(),
     prompt: input.prompt,
     context: input.context ?? "fresh",
-    provider: model.provider,
-    model: model.id,
-    thinking,
+    provider: selection.model.provider,
+    model: selection.model.id,
+    thinking: selection.thinking,
     trusted,
     name: input.name,
     createdAt: Date.now(),

package/src/agent/legacy.ts

@@ -1,12 +1,13 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
-import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
 import { journalStamp } from "../shared/native-journal.ts";
 import { receiptId } from "./receipts.ts";
 import { resolveTempRootDir } from "../shared/types.ts";
 import { isRecord } from "../shared/unknown.ts";
-import { hold, release, isHeld } from "./notifications.ts";
+import { release, isHeld, type Completion } from "./notifications.ts";
 import { missing } from "./storage.ts";
+import { errorText } from "./host.ts";
 
 interface LegacyInput {
   file: string;
@@ -17,29 +18,53 @@ interface LegacyInput {
   owner: string;
   single: boolean;
 }
-function legacyInput(file: string, owner: string): LegacyInput | undefined {
+interface IgnoredInputs {
+  readonly owner: string;
+  readonly directory: string;
+  readonly ignored: Map<string, string>;
+}
+interface LegacyScan extends IgnoredInputs {
+  readonly known: ReadonlyMap<string, string> | undefined;
+}
+// Retain only foreign classifications, never their saved output, for each live native manager.
+const ignoredInputs = new WeakMap<ExtensionContext["sessionManager"], IgnoredInputs>();
+function fileStamp(file: string): string | undefined {
   const stat = fs.lstatSync(file, { bigint: true });
-  if (!stat.isFile() || stat.isSymbolicLink()) {
+  return stat.isFile() && !stat.isSymbolicLink() ? journalStamp(stat) : undefined;
+}
+function legacyInput(file: string, scan: LegacyScan): LegacyInput | undefined {
+  const stamp = fileStamp(file);
+  if (stamp === undefined) {
+    return;
+  }
+  if (scan.known?.get(file) === stamp) {
+    scan.ignored.set(file, stamp);
     return;
   }
   const raw = fs.readFileSync(file, "utf8");
   const data: unknown = JSON.parse(raw);
-  if (!isRecord(data) || data.sessionId !== owner) {
+  if (!isRecord(data) || data.sessionId !== scan.owner) {
+    if (fileStamp(file) === stamp) {
+      scan.ignored.set(file, stamp);
+    }
     return;
   }
+  return ownedInput({ file, raw, stamp, owner: scan.owner }, data);
+}
+function ownedInput(
+  source: Pick<LegacyInput, "file" | "raw" | "stamp" | "owner">,
+  data: Record<string, unknown>,
+): LegacyInput | undefined {
   const id = typeof data.runId === "string" ? data.runId : data.id;
-  if (typeof id !== "string" || path.basename(file, ".json") !== id) {
+  if (typeof id !== "string" || path.basename(source.file, ".json") !== id) {
     return;
   }
   const legacy = `legacy:${id}:${typeof data.timestamp === "number" ? data.timestamp : "unknown"}`;
   const completionId = typeof data.completionId === "string" ? data.completionId : legacy;
   return {
-    file,
-    raw,
-    stamp: journalStamp(stat),
+    ...source,
     id,
     key: `completion:${completionId}`,
-    owner,
     single: data.mode === "single",
   };
 }
@@ -104,7 +129,7 @@ function published(ctx: ExtensionContext, input: LegacyInput): boolean {
     undefined
   );
 }
-function deliver(pi: ExtensionAPI, ctx: ExtensionContext, input: LegacyInput): void {
+function notification(ctx: ExtensionContext, input: LegacyInput): Completion | undefined {
   const identity = `legacy:${input.id}:${input.key}`;
   if (published(ctx, input)) {
     release(input.owner, identity);
@@ -120,26 +145,54 @@ function deliver(pi: ExtensionAPI, ctx: ExtensionContext, input: LegacyInput): v
   if (isHeld(input.owner, identity)) {
     return;
   }
-  hold(input.owner, identity);
+  return {
+    owner: input.owner,
+    id: identity,
+    message: {
+      customType: "subagent-notify",
+      display: true,
+      // Full saved bytes, not a replacement summary. No owner/run record is invented.
+      content: input.raw,
+      details: { completion: { runId: input.id, key: input.key, ownerSessionId: input.owner } },
+    },
+  };
+}
+function legacyNotice(
+  ctx: ExtensionContext,
+  file: string,
+  scan: LegacyScan,
+): Completion | undefined {
   try {
-    pi.sendMessage(
-      {
-        customType: "subagent-notify",
-        display: true,
-        // Full saved bytes, not a replacement summary. No owner/run record is invented.
-        content: input.raw,
-        details: { completion: { runId: input.id, key: input.key, ownerSessionId: input.owner } },
-      },
-      { triggerTurn: true, deliverAs: "followUp" },
-    );
+    const input = legacyInput(file, scan);
+    return input ? notification(ctx, input) : undefined;
   } catch (error) {
-    release(input.owner, identity);
-    throw error;
+    console.error(`Skipped legacy completion ${file}: ${errorText(error)}`);
+    return;
   }
 }
-export function notifyLegacy(pi: ExtensionAPI, ctx: ExtensionContext): void {
-  const root = resolveTempRootDir();
-  const directory = path.join(root, "async-subagent-results");
+function startScan(ctx: ExtensionContext, directory: string): LegacyScan {
+  const manager = ctx.sessionManager;
+  const owner = manager.getSessionId();
+  const prior = ignoredInputs.get(manager);
+  const scan: LegacyScan = {
+    directory,
+    owner,
+    known: prior?.directory === directory && prior.owner === owner ? prior.ignored : undefined,
+    ignored: new Map(),
+  };
+  // Replace even on a failed directory scan; vanished/unsafe inputs keep no cached decision.
+  ignoredInputs.set(manager, { directory, owner, ignored: scan.ignored });
+  return scan;
+}
+/** Retired releases created this directory at session start, before any of their runners. */
+export function legacyDirectory(): string {
+  return path.join(resolveTempRootDir(), "async-subagent-results");
+}
+export function legacyNotifications(ctx: ExtensionContext): Completion[] {
+  const notices: Completion[] = [];
+  const directory = legacyDirectory();
+  const root = path.dirname(directory);
+  const scan = startScan(ctx, directory);
   let names: string[];
   try {
     for (const candidate of [root, directory]) {
@@ -151,7 +204,7 @@ export function notifyLegacy(pi: ExtensionAPI, ctx: ExtensionContext): void {
     names = fs.readdirSync(directory);
   } catch (error) {
     if (missing(error)) {
-      return;
+      return notices;
     }
     throw error;
   }
@@ -159,9 +212,10 @@ export function notifyLegacy(pi: ExtensionAPI, ctx: ExtensionContext): void {
     if (!name.endsWith(".json")) {
       continue;
     }
-    const input = legacyInput(path.join(directory, name), ctx.sessionManager.getSessionId());
-    if (input) {
-      deliver(pi, ctx, input);
+    const notice = legacyNotice(ctx, path.join(directory, name), scan);
+    if (notice) {
+      notices.push(notice);
     }
   }
+  return notices;
 }