@jameslovespancakes/pi-plus 1.0.12 → 1.0.13

package/src/core/config.ts

@@ -20,10 +20,17 @@ export interface RemoteSection {
   workers: Record<string, unknown>[];
 }
 
+export type BetterCompactMode = "off" | "on" | "jev";
+
+export interface CompactSection {
+  better: BetterCompactMode;
+}
+
 export interface PiPlusConfig {
   env: Record<string, string>;
   policy: PolicySection;
   remote: RemoteSection;
+  compact: CompactSection;
 }
 
 const DEFAULTS: PiPlusConfig = {
@@ -34,6 +41,7 @@ const DEFAULTS: PiPlusConfig = {
     deny: [],
   },
   remote: { injectStatus: true, workers: [] },
+  compact: { better: "off" },
 };
 
 /** Legacy file -> section, applied only when that section is still absent. */
@@ -103,6 +111,7 @@ export function configPath(): string {
 }
 
 function normalize(raw: Partial<PiPlusConfig> | undefined): PiPlusConfig {
+  const better = raw?.compact?.better;
   return {
     env: raw?.env && typeof raw.env === "object" ? { ...raw.env } : {},
     policy: {
@@ -117,6 +126,9 @@ function normalize(raw: Partial<PiPlusConfig> | undefined): PiPlusConfig {
       defaults: raw?.remote?.defaults,
       workers: Array.isArray(raw?.remote?.workers) ? raw.remote.workers : [],
     },
+    compact: {
+      better: better === "on" || better === "jev" ? better : "off",
+    },
   };
 }
 

package/src/domains/agents/format.ts

@@ -0,0 +1,97 @@
+const DELIVERY_TEXT_MAX = 2_000;
+
+export interface SnapshotAgent {
+  sessionId: string;
+  alias?: string;
+  host: string;
+  branch?: string;
+  repo?: string;
+  commit?: string;
+  state: "idle" | "thinking" | "tool";
+  lastTool?: string;
+  coordinator?: string | null;
+  reports?: string[];
+}
+
+export interface SnapshotSelf {
+  branch?: string;
+  repo?: string;
+  commit?: string;
+}
+
+export interface SnapshotCoordination {
+  coordinator?: string | null;
+  reports?: string[];
+  repoThread?: string;
+}
+
+export interface DeliveryMessage {
+  senderId: string;
+  senderAlias?: string;
+  text: string;
+  priority: "normal" | "urgent";
+}
+
+export interface DeliveryThread {
+  id: string;
+  title?: string;
+}
+
+export interface BoardDelivery {
+  message: DeliveryMessage;
+  thread: DeliveryThread;
+}
+
+function cleanLine(value: unknown, max: number): string {
+  const text = String(value ?? "").replace(/[\u0000-\u001f\u007f]/g, " ").replace(/\s+/g, " ").trim();
+  return text.length <= max ? text : `${text.slice(0, max - 1)}…`;
+}
+
+function compactTool(lastTool?: string): string | undefined {
+  if (!lastTool) return undefined;
+  const separator = lastTool.indexOf(":");
+  return cleanLine(separator < 0 ? lastTool : lastTool.slice(0, separator), 32);
+}
+
+export function formatBoardSnapshot(
+  agents: readonly SnapshotAgent[],
+  self: SnapshotSelf,
+  coordination: SnapshotCoordination,
+): string {
+  const commit = self.commit?.slice(0, 12) || "no-commit";
+  const meta = [
+    `${self.branch || "-"}@${commit}`,
+    `${agents.length} peer${agents.length === 1 ? "" : "s"}`,
+    coordination.repoThread ? `room=${coordination.repoThread}` : undefined,
+    coordination.coordinator ? `reports-to=${coordination.coordinator}` : undefined,
+    coordination.reports?.length ? `reports=${coordination.reports.join(",")}` : undefined,
+  ].filter(Boolean).join(" ");
+  const lines = [`[board ${meta}]`];
+  for (const agent of agents.slice(0, 12)) {
+    const agentCommit = agent.commit?.slice(0, 12) || "no-commit";
+    const sameLine = agent.repo && self.repo === agent.repo && agent.branch === self.branch;
+    const drift = sameLine && agent.commit && self.commit && agent.commit !== self.commit ? " !commit" : "";
+    const tool = agent.state === "tool" ? compactTool(agent.lastTool) : undefined;
+    const activity = tool ? `tool:${tool}` : agent.state;
+    const role = agent.reports?.length ? " coord" : agent.coordinator ? ` ->${agent.coordinator}` : "";
+    lines.push(`- ${agent.alias || agent.sessionId.slice(0, 8)}@${agent.host} ${activity} ${agent.branch || "-"}@${agentCommit}${drift}${role}`);
+  }
+  if (agents.length > 12) lines.push(`- +${agents.length - 12} more; use agent_board agents`);
+  return lines.join("\n");
+}
+
+function compactDeliveryText(text: string): string {
+  const clean = text.replace(/\r\n?/g, "\n").replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, " ").trim();
+  if (clean.length <= DELIVERY_TEXT_MAX) return clean;
+  return `${clean.slice(0, DELIVERY_TEXT_MAX - 1)}…\n[truncated; use agent_board read]`;
+}
+
+export function formatBoardDeliveries(deliveries: readonly BoardDelivery[]): string {
+  const lines = [`[board${deliveries.some(({ message }) => message.priority === "urgent") ? " urgent" : ""} x${deliveries.length}]`];
+  for (const { message, thread } of deliveries) {
+    const sender = message.senderAlias || message.senderId.slice(0, 8);
+    lines.push(`${message.priority === "urgent" ? "! " : ""}from=${sender} thread=${thread.id}${thread.title ? ` (${cleanLine(thread.title, 60)})` : ""}`);
+    lines.push(compactDeliveryText(message.text));
+  }
+  return lines.join("\n");
+}

package/src/domains/agents/index.ts

@@ -11,11 +11,14 @@ import { Type } from "typebox";
 import WebSocket from "ws";
 import { env } from "../../core/env.ts";
 import { handleBoardAdmin, registerBoardLifecycle } from "./board-setup.ts";
+import { formatBoardDeliveries, formatBoardSnapshot, type BoardDelivery } from "./format.ts";
 
 const HEARTBEAT_MS = 8_000;
 const REQUEST_TIMEOUT_MS = 2_000;
 const SNAPSHOT_CACHE_MS = 1_500;
 const MESSAGE_MAX = 8_000;
+const DELIVERY_DEBOUNCE_MS = 250;
+const DELIVERY_BATCH_MAX = 8;
 
 type State = "idle" | "thinking" | "tool";
 type Priority = "normal" | "urgent";
@@ -302,6 +305,9 @@ export default function (pi: ExtensionAPI) {
   let self: AgentInfo | undefined;
   let gitInfo: GitInfo = {};
   let snapshot: { at: number; agents: AgentInfo[] } = { at: 0, agents: [] };
+  let lastInjectedSnapshot: string | undefined;
+  let pendingDeliveries: BoardDelivery[] = [];
+  let deliveryTimer: ReturnType<typeof setTimeout> | undefined;
 
   const updateStatus = () => {
     if (!ctx?.hasUI) return;
@@ -315,43 +321,46 @@ export default function (pi: ExtensionAPI) {
     self = { ...self, ...gitInfo };
     client.presence(gitInfo);
   };
-  const formatCoordination = (): string[] => {
-    const { coordinator, reports, repoThread } = client.coordination;
-    const lines: string[] = [];
-    if (coordinator) lines.push(`You report to coordinator '${coordinator}'. Use agent_board action 'report' to send them progress, blockers, and completed work.`);
-    if (reports?.length) lines.push(`You are the coordinator for: ${reports.join(", ")}. Their reports arrive as agent-board messages; keep their work aligned.`);
-    if (repoThread) lines.push(`Shared repo room: thread '${repoThread}' automatically contains every agent working in this repository. Post repo-wide notices there with agent_board send.`);
-    return lines;
+  const snapshotAgents = async (): Promise<AgentInfo[]> => {
+    if (Date.now() - snapshot.at > SNAPSHOT_CACHE_MS) {
+      snapshot = { at: Date.now(), agents: await client.request("agents", {}, 500) };
+    }
+    return snapshot.agents;
   };
-  const formatSnapshot = (agents: AgentInfo[]): string => {
-    const coordinationLines = formatCoordination();
-    if (!agents.length) return ["[Live agent board]\nNo other Pi agents are currently running.", ...coordinationLines].join("\n");
-    const lines = agents.slice(0, 20).map((agent) => {
-      const commit = agent.commit ? agent.commit.slice(0, 12) : "no-commit";
-      const sameLine = agent.repo && gitInfo.repo === agent.repo && agent.branch === gitInfo.branch;
-      const divergent = sameLine && agent.commit && gitInfo.commit && agent.commit !== gitInfo.commit ? " ⚠ different commit" : "";
-      const activity = agent.state === "tool" && agent.lastTool ? agent.lastTool : agent.state;
-      const role = agent.reports?.length ? " [coordinator]" : agent.coordinator ? ` [reports to ${agent.coordinator}]` : "";
-      return `- ${agent.alias || agent.sessionId.slice(0, 8)}@${agent.host}${role}: ${activity}; ${agent.branch || "-"}@${commit}${divergent}${agent.lastPrompt ? `; ${agent.lastPrompt}` : ""}`;
-    });
-    return [
-      `[Live agent board, consult before working]`,
-      `Your commit: ${gitInfo.branch || "-"}@${gitInfo.commit?.slice(0, 12) || "no-commit"}`,
-      `Running collaborators:`,
-      lines.join("\n"),
-      ...coordinationLines,
-      `Avoid duplicate or outdated work. Use agent_board to coordinate when useful.`,
-    ].join("\n");
+  const flushDeliveries = () => {
+    if (deliveryTimer) clearTimeout(deliveryTimer);
+    deliveryTimer = undefined;
+    const deliveries = pendingDeliveries;
+    pendingDeliveries = [];
+    if (!ctx || !deliveries.length) return;
+    const urgent = deliveries.some(({ message }) => message.priority === "urgent");
+    if (urgent && !ctx.isIdle()) ctx.abort();
+    const details = deliveries.length === 1 ? deliveries[0] : { deliveries };
+    pi.sendMessage(
+      { customType: "agent-board", content: formatBoardDeliveries(deliveries), display: true, details },
+      { deliverAs: "steer", triggerTurn: true },
+    );
+  };
+  const queueDelivery = (delivery: BoardDelivery) => {
+    pendingDeliveries.push(delivery);
+    if (delivery.message.priority === "urgent" || pendingDeliveries.length >= DELIVERY_BATCH_MAX) {
+      flushDeliveries();
+      return;
+    }
+    if (deliveryTimer) return;
+    deliveryTimer = setTimeout(flushDeliveries, DELIVERY_DEBOUNCE_MS);
+    deliveryTimer.unref?.();
   };
 
   client.on((event) => {
-    if (event.t === "connection" || event.t === "coordination") { updateStatus(); return; }
+    if (event.t === "connection") {
+      if (event.connected) { snapshot.at = 0; lastInjectedSnapshot = undefined; }
+      updateStatus();
+      return;
+    }
+    if (event.t === "coordination") { updateStatus(); return; }
     if (event.t !== "message" || event.adminView || !ctx) return;
-    const message = event.message as BoardMessage;
-    const thread = event.thread as ThreadInfo;
-    const content = `[Live agent-board message${message.priority === "urgent" ? " URGENT" : ""}]\nFrom: ${message.senderAlias || message.senderId}\nThread: ${thread.title || thread.id}\nCommit context: you are on ${gitInfo.branch || "-"}@${gitInfo.commit?.slice(0, 12) || "no-commit"}\n\n${message.text}\n\nEvaluate this at the next decision point. If relevant, adjust your work and reply through agent_board with thread '${thread.id}'. If irrelevant, continue the current task.`;
-    if (message.priority === "urgent" && !ctx.isIdle()) ctx.abort();
-    pi.sendMessage({ customType: "agent-board", content, display: true, details: { message, thread } }, { deliverAs: "steer", triggerTurn: true });
+    queueDelivery({ message: event.message as BoardMessage, thread: event.thread as ThreadInfo });
   });
 
   pi.on("session_start", async (_event, eventCtx) => {
@@ -368,6 +377,16 @@ export default function (pi: ExtensionAPI) {
   pi.on("before_agent_start", async (event) => {
     client.presence({ state: "thinking", lastPrompt: cleanLine(event.prompt) });
     client.activity("prompt", event.prompt);
+    if (!client.connected) return;
+    try {
+      const content = formatBoardSnapshot(await snapshotAgents(), gitInfo, client.coordination);
+      if (content === lastInjectedSnapshot) return;
+      lastInjectedSnapshot = content;
+      // Persist one compact snapshot at a user-turn boundary. A transient message
+      // appended in `context` becomes Anthropic's final cache breakpoint but is
+      // absent from the next transcript, forcing a conversation-cache miss.
+      return { message: { customType: "agent-board-snapshot", content, display: false, details: {} } };
+    } catch { return; }
   });
   pi.on("tool_execution_start", async (event) => {
     const lastTool = cleanLine(`${event.toolName}: ${JSON.stringify(event.args)}`, 100);
@@ -375,20 +394,12 @@ export default function (pi: ExtensionAPI) {
   });
   pi.on("tool_execution_end", async () => client.presence({ state: "thinking" }));
   pi.on("agent_settled", async () => { client.presence({ state: "idle", lastTool: undefined }); refreshGit(); });
-  pi.on("session_shutdown", async (_event, eventCtx) => { client.stop(); eventCtx.ui.setStatus("agent-board", undefined); ctx = undefined; self = undefined; });
-
-  // A transient, late context message avoids growing the session and keeps the static prompt cache stable.
-  pi.on("context", async (event) => {
-    if (!client.connected) return;
-    try {
-      if (Date.now() - snapshot.at > SNAPSHOT_CACHE_MS) snapshot = { at: Date.now(), agents: await client.request("agents", {}, 500) };
-      return {
-        messages: [...event.messages, {
-          role: "custom" as const, customType: "agent-board-snapshot", content: formatSnapshot(snapshot.agents),
-          display: false, details: {}, timestamp: Date.now(),
-        }],
-      };
-    } catch { return; }
+  pi.on("session_compact", async () => { lastInjectedSnapshot = undefined; });
+  pi.on("session_tree", async () => { lastInjectedSnapshot = undefined; });
+  pi.on("session_shutdown", async (_event, eventCtx) => {
+    if (deliveryTimer) clearTimeout(deliveryTimer);
+    deliveryTimer = undefined; pendingDeliveries = [];
+    client.stop(); eventCtx.ui.setStatus("agent-board", undefined); ctx = undefined; self = undefined;
   });
 
   pi.registerMessageRenderer("agent-board", (message, _options, theme) => new Text(theme.fg("accent", message.content as string), 1, 0));
@@ -419,7 +430,7 @@ export default function (pi: ExtensionAPI) {
     description: "Consult and message all currently running local and remote Pi agents. Every agent record includes host, state, Git branch, and commit hash. Messages are live: idle agents wake immediately and busy agents see them at the next safe turn boundary. Stopped agents are never listed and messages are not queued for them. Agents working in the same repository share an auto-created repo room, and coordinators can be assigned so subordinate agents report their progress upward.",
     promptSnippet: "Consult live collaborators, report to your coordinator, and exchange direct, repo-room, or group messages",
     promptGuidelines: [
-      "Consult the transient Live agent board context before starting work; avoid duplicating work and compare Git commit hashes before relying on another agent's changes.",
+      "Consult the latest compact Live agent board snapshot before starting work; avoid duplicating work and compare Git commit hashes before relying on another agent's changes.",
       "Use agent_board to inform collaborators when their commit or assumptions appear outdated, and reply only when coordination is useful.",
       "If the board snapshot names a coordinator for you, use agent_board action 'report' to send them meaningful progress, blockers, and completed work instead of staying silent.",
       "Use agent_board action 'set_coordinator' when the user puts one agent in charge of others, and 'coordinators' to inspect the current reporting structure.",

package/src/domains/compact/archive.ts

@@ -0,0 +1,140 @@
+import { createHash } from "node:crypto";
+import { existsSync, readFileSync } from "node:fs";
+import { basename } from "node:path";
+import { agentPath, writeJson } from "../../core/store.ts";
+import type { ArchiveRecord, SemanticChunk, SuperContextArchive } from "./types.ts";
+
+function safeSessionId(sessionId: string): string {
+  const readable = sessionId.replace(/[^A-Za-z0-9._-]/g, "-").slice(0, 80) || "session";
+  const suffix = createHash("sha256").update(sessionId).digest("hex").slice(0, 10);
+  return `${readable}-${suffix}`;
+}
+
+export function archivePath(sessionId: string): string {
+  return agentPath("super-context", "archives", `${safeSessionId(sessionId)}.json`);
+}
+
+function emptyArchive(sessionId: string): SuperContextArchive {
+  return { version: 1, sessionId, records: [], checkpoints: {} };
+}
+
+function isArchiveRecord(value: unknown): value is ArchiveRecord {
+  if (!value || typeof value !== "object") return false;
+  const record = value as Partial<ArchiveRecord>;
+  return typeof record.id === "string"
+    && typeof record.hash === "string"
+    && typeof record.text === "string"
+    && typeof record.role === "string"
+    && typeof record.ordinal === "number"
+    && typeof record.archivedAt === "string";
+}
+
+/** Refuse to overwrite a corrupt archive: source recoverability wins over convenience. */
+export function loadArchive(sessionId: string): SuperContextArchive {
+  const path = archivePath(sessionId);
+  if (!existsSync(path)) return emptyArchive(sessionId);
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(readFileSync(path, "utf8"));
+  } catch (error) {
+    throw new Error(`Super Context archive is unreadable; refusing to overwrite ${path}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
+  }
+  if (!parsed || typeof parsed !== "object") throw new Error(`Invalid Super Context archive: ${path}`);
+  const candidate = parsed as Partial<SuperContextArchive>;
+  if (candidate.version !== 1 || candidate.sessionId !== sessionId || !Array.isArray(candidate.records)) {
+    throw new Error(`Unsupported Super Context archive format: ${path}`);
+  }
+  if (!candidate.records.every(isArchiveRecord)) throw new Error(`Invalid Super Context archive records: ${path}`);
+  const checkpoints = candidate.checkpoints;
+  if (!checkpoints || typeof checkpoints !== "object" || Array.isArray(checkpoints)) {
+    throw new Error(`Invalid Super Context archive checkpoints: ${path}`);
+  }
+  for (const ids of Object.values(checkpoints)) {
+    if (!Array.isArray(ids) || ids.some((id) => typeof id !== "string")) {
+      throw new Error(`Invalid Super Context archive checkpoint: ${path}`);
+    }
+  }
+  return candidate as SuperContextArchive;
+}
+
+function addChunks(archive: SuperContextArchive, chunks: readonly SemanticChunk[]): void {
+  const byId = new Map(archive.records.map((record) => [record.id, record]));
+  let ordinal = archive.records.reduce((maximum, record) => Math.max(maximum, record.ordinal), -1) + 1;
+  const archivedAt = new Date().toISOString();
+  for (const chunk of chunks) {
+    const existing = byId.get(chunk.id);
+    if (existing) {
+      if (existing.hash !== chunk.hash || existing.text !== chunk.text) {
+        throw new Error(`Super Context archive ID collision for ${chunk.id}`);
+      }
+      continue;
+    }
+    const record: ArchiveRecord = { ...chunk, ordinal, archivedAt };
+    ordinal += 1;
+    archive.records.push(record);
+    byId.set(record.id, record);
+  }
+}
+
+function checkpointHash(recordIds: readonly string[]): string {
+  const digest = createHash("sha256").update(recordIds.join("\0")).digest("hex");
+  return `SCC-${digest.slice(0, 20)}`;
+}
+
+export interface PersistedCheckpoint {
+  archive: SuperContextArchive;
+  checkpointId: string;
+  records: ArchiveRecord[];
+  duplicateChunks: number;
+  file: string;
+}
+
+/** Add-only persistence for immutable records and content-addressed checkpoints. */
+export function persistCheckpoint(
+  sessionId: string,
+  chunks: readonly SemanticChunk[],
+  priorRecordIds: readonly string[],
+): PersistedCheckpoint {
+  const archive = loadArchive(sessionId);
+  const currentIds = chunks.map((chunk) => chunk.id);
+  const allRecordIds = [...priorRecordIds, ...currentIds];
+  const duplicateChunks = allRecordIds.length - new Set(allRecordIds).size;
+  addChunks(archive, chunks);
+
+  const knownIds = new Set(archive.records.map((record) => record.id));
+  const recordIds = [...new Set(allRecordIds)];
+  const missing = recordIds.filter((id) => !knownIds.has(id));
+  if (missing.length > 0) throw new Error(`Super Context archive is missing ${missing.length} source record(s)`);
+
+  const checkpointId = checkpointHash(recordIds);
+  const existing = archive.checkpoints[checkpointId];
+  if (existing && JSON.stringify(existing) !== JSON.stringify(recordIds)) {
+    throw new Error(`Super Context checkpoint collision for ${checkpointId}`);
+  }
+  archive.checkpoints[checkpointId] ??= recordIds;
+
+  const file = archivePath(sessionId);
+  if (!writeJson(file, archive, false, 0o600)) throw new Error(`Failed to persist Super Context archive: ${file}`);
+  const byId = new Map(archive.records.map((record) => [record.id, record]));
+  return {
+    archive,
+    checkpointId,
+    records: recordIds.map((id) => byId.get(id)!),
+    duplicateChunks,
+    file,
+  };
+}
+
+export function checkpointRecords(archive: SuperContextArchive, checkpointId: string | undefined): ArchiveRecord[] {
+  if (!checkpointId) return [];
+  const ids = archive.checkpoints[checkpointId];
+  if (!ids) return [];
+  const byId = new Map(archive.records.map((record) => [record.id, record]));
+  const records = ids.map((id) => byId.get(id)).filter((record): record is ArchiveRecord => record !== undefined);
+  return records.length === ids.length ? records : [];
+}
+
+export function archiveDisplayName(file: string): string {
+  return basename(file);
+}

package/src/domains/compact/chunking.ts

@@ -0,0 +1,166 @@
+import { createHash } from "node:crypto";
+import type { AgentMessage } from "@earendil-works/pi-agent-core";
+import type { Message } from "@earendil-works/pi-ai";
+import { convertToLlm } from "@earendil-works/pi-coding-agent";
+import type { ChunkRole, SemanticChunk, SourceItem } from "./types.ts";
+
+export const TARGET_CHUNK_TOKENS = 512;
+export const MIN_CHUNK_TOKENS = 128;
+export const MAX_CHUNK_TOKENS = 1024;
+export const CHUNK_OVERLAP_TOKENS = 32;
+
+const PROTECTION_PATTERN = /(?:\b(?:must|never|required?|constraint|blocked|blocker|decision|correction|instead|uncommitted|rollback|next steps?|todo|in progress|do not|don't|cannot|can't)\b|\b(?:error|failed?|failure|exception|timeout|timed out|denied|refused)\b|\b(?:modified|created|deleted|renamed|implemented|verified|reproduced)\b)/i;
+const EXACT_PATTERN = /(?:[A-Za-z]:[\\/][^\s`"']+|(?:^|\s)(?:\.\.?[\\/]|~[\\/])[^\s`"']+|\b[\w.-]+\.(?:ts|tsx|js|jsx|mjs|cjs|json|md|py|rs|go|java|cs|cpp|c|h|yaml|yml|toml|lock|sql)\b|`[^`\n]{1,160}`|\b[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*\(\)|\b(?:exit code|status code|HTTP)\s*[:=]?\s*\d+\b|\b\d+(?:\.\d+)?%\b)/im;
+const QUARANTINE_PATTERN = /(?:ignore (?:all )?(?:prior|previous|earlier|system|developer) instructions?|system (?:override|message|prompt)|developer (?:override|message)|you are (?:chatgpt|the assistant)|follow these instructions?|assistant must|reveal (?:the )?(?:system|developer) prompt|include (?:the )?exact token|output (?:only )?(?:the )?(?:token|string)|prompt injection|INJECTION_[A-Z0-9_-]+)/i;
+
+function estimateTokens(text: string): number {
+  return Math.max(1, Math.ceil(text.length / 4));
+}
+
+function blockText(content: Message["content"]): string {
+  if (typeof content === "string") return content;
+  return content
+    .map((block) => {
+      if (block.type === "text") return block.text;
+      if (block.type === "image") return `[image ${block.mimeType}]`;
+      return "";
+    })
+    .filter(Boolean)
+    .join("\n");
+}
+
+function serializeMessage(message: Message): string {
+  if (message.role === "user") return `[User]\n${blockText(message.content)}`;
+  if (message.role === "toolResult") {
+    const state = message.isError ? " error" : "";
+    return `[Tool result: ${message.toolName}${state}]\n${blockText(message.content)}`;
+  }
+  const parts: string[] = [];
+  for (const block of message.content) {
+    if (block.type === "text") parts.push(`[Assistant]\n${block.text}`);
+    else if (block.type === "thinking") parts.push(`[Assistant thinking]\n${block.thinking}`);
+    else if (block.type === "toolCall") {
+      parts.push(`[Assistant tool call]\n${block.name}(${JSON.stringify(block.arguments)})`);
+    }
+  }
+  if (message.errorMessage) parts.push(`[Assistant error]\n${message.errorMessage}`);
+  return parts.join("\n\n");
+}
+
+function roleFor(message: AgentMessage, converted: Message): ChunkRole {
+  if (message.role === "user") return "user";
+  if (message.role === "assistant") return "assistant";
+  if (message.role === "toolResult" || message.role === "bashExecution") return "tool";
+  if (message.role === "compactionSummary" || message.role === "branchSummary") return "summary";
+  if (message.role === "custom") return "custom";
+  if (converted.role === "toolResult") return "tool";
+  if (converted.role === "assistant") return "assistant";
+  return "custom";
+}
+
+/** Convert Pi messages without applying Pi's tool-output truncation. */
+export function sourceItemsFromMessages(messages: readonly AgentMessage[]): SourceItem[] {
+  const items: SourceItem[] = [];
+  for (const message of messages) {
+    for (const converted of convertToLlm([message])) {
+      const text = serializeMessage(converted).trim();
+      if (!text) continue;
+      items.push({ role: roleFor(message, converted), text, source: "conversation" });
+    }
+  }
+  return items;
+}
+
+export function legacySummaryItem(summary: string): SourceItem {
+  return { role: "summary", text: `[Legacy compaction summary]\n${summary}`, source: "legacy-summary" };
+}
+
+export function isQuarantined(role: ChunkRole, text: string): boolean {
+  return role === "tool" && QUARANTINE_PATTERN.test(text);
+}
+
+export function isProtected(role: ChunkRole, text: string): boolean {
+  if (isQuarantined(role, text)) return false;
+  if (role === "user" || role === "summary") return true;
+  return PROTECTION_PATTERN.test(text);
+}
+
+export function isExactHeavy(text: string): boolean {
+  return EXACT_PATTERN.test(text) || PROTECTION_PATTERN.test(text);
+}
+
+function splitLargeItem(item: SourceItem): SourceItem[] {
+  const maxChars = MAX_CHUNK_TOKENS * 4;
+  const overlapChars = CHUNK_OVERLAP_TOKENS * 4;
+  if (item.text.length <= maxChars) return [item];
+
+  const pieces: SourceItem[] = [];
+  let start = 0;
+  while (start < item.text.length) {
+    let end = Math.min(item.text.length, start + maxChars);
+    if (end < item.text.length) {
+      const newline = item.text.lastIndexOf("\n", end);
+      const sentence = item.text.lastIndexOf(". ", end);
+      const boundary = Math.max(newline, sentence);
+      if (boundary > start + Math.floor(maxChars / 2)) end = boundary + 1;
+    }
+    pieces.push({ ...item, text: item.text.slice(start, end) });
+    if (end >= item.text.length) break;
+    start = Math.max(start + 1, end - overlapChars);
+  }
+  return pieces;
+}
+
+function safetyKey(item: SourceItem): string {
+  return [item.role, item.source, isProtected(item.role, item.text), isQuarantined(item.role, item.text)].join(":");
+}
+
+function makeChunk(items: readonly SourceItem[]): SemanticChunk {
+  const text = items.map((item) => item.text).join("\n\n");
+  const role = items[0]!.role;
+  const source = items[0]!.source;
+  const hash = createHash("sha256").update(role).update("\0").update(source).update("\0").update(text).digest("hex");
+  return {
+    id: `SC-${hash.slice(0, 16)}`,
+    hash,
+    role,
+    text,
+    tokens: estimateTokens(text),
+    source,
+    protected: isProtected(role, text),
+    exactHeavy: isExactHeavy(text),
+    quarantined: isQuarantined(role, text),
+  };
+}
+
+/**
+ * Semantic/trust-aware chunking. Role and safety boundaries are never mixed;
+ * undersized chunks are retained when combining them would cross a boundary.
+ */
+export function semanticChunks(sourceItems: readonly SourceItem[]): SemanticChunk[] {
+  const units = sourceItems.flatMap(splitLargeItem);
+  const chunks: SemanticChunk[] = [];
+  let current: SourceItem[] = [];
+  let currentTokens = 0;
+
+  const flush = () => {
+    if (current.length > 0) chunks.push(makeChunk(current));
+    current = [];
+    currentTokens = 0;
+  };
+
+  for (const unit of units) {
+    const unitTokens = estimateTokens(unit.text);
+    const compatible = current.length === 0 || safetyKey(current[0]!) === safetyKey(unit);
+    const combinedTokens = currentTokens + unitTokens;
+    if (!compatible || (current.length > 0 && currentTokens >= MIN_CHUNK_TOKENS && combinedTokens > TARGET_CHUNK_TOKENS)) {
+      flush();
+    }
+    if (current.length > 0 && currentTokens + unitTokens > MAX_CHUNK_TOKENS) flush();
+    current.push(unit);
+    currentTokens += unitTokens;
+    if (currentTokens >= TARGET_CHUNK_TOKENS) flush();
+  }
+  flush();
+  return chunks;
+}