@chendpoc/pi-memory 0.1.13 → 0.2.1

package/src/preflight/strip.ts

@@ -1,10 +1,5 @@
-const PRIVATE_MEMORY_OPEN = "<private_memory>";
-const PRIVATE_MEMORY_CLOSE = "</private_memory>";
+import { PRIVATE_MEMORY_CLOSE, PRIVATE_MEMORY_OPEN } from "../constants/preflight.js";
 
-/**
- * Remove all well-formed <private_memory>…</private_memory> blocks from text.
- * Unterminated open markers are left untouched (fail-safe).
- */
 export function stripPrivateMemory(text: string): string {
   let s = text;
   for (;;) {
@@ -27,10 +22,6 @@ export function stripPrivateMemory(text: string): string {
   }
 }
 
-/**
- * Inject private memory context before the user payload in a scaffolded message.
- * Mirrors Kocoro injectPrivateMemoryContext.
- */
 export function injectPrivateMemoryContext(
   scaffolded: string,
   userPayload: string,
@@ -38,6 +29,7 @@ export function injectPrivateMemoryContext(
 ): string {
   const ctx = privateContext.trim();
   if (!ctx) return scaffolded;
+
   if (userPayload && scaffolded.endsWith(userPayload)) {
     return (
       scaffolded.slice(0, scaffolded.length - userPayload.length) +
@@ -46,5 +38,6 @@ export function injectPrivateMemoryContext(
       userPayload
     );
   }
-  return scaffolded + "\n\n" + ctx;
+
+  return `${scaffolded}\n\n${ctx}`;
 }

package/src/shutdown/enqueue.ts

@@ -0,0 +1,29 @@
+import { SHUTDOWN_QUEUE_FILE } from "../constants/memory.js";
+import { serializeJsonlFrame } from "../ipc/jsonlFramer.js";
+import { appendText, joinPath } from "../utils/fs.js";
+
+import type { SessionShutdownEvent } from "@earendil-works/pi-coding-agent";
+
+export type ShutdownQueueEntry = {
+  sessionFile: string;
+  parentSession?: string;
+  reason: SessionShutdownEvent["reason"];
+  isSubagent: boolean;
+  enqueuedAt: string;
+};
+
+export function shutdownQueuePath(agentDir: string): string {
+  return joinPath(agentDir, SHUTDOWN_QUEUE_FILE);
+}
+
+export async function enqueueShutdownMetadata(
+  agentDir: string,
+  entry: ShutdownQueueEntry,
+): Promise<void> {
+  await appendText(shutdownQueuePath(agentDir), serializeJsonlFrame(entry));
+}
+
+export function readParentSession(header: Record<string, unknown> | null): string | undefined {
+  const parent = header?.parentSession ?? header?.parent_session;
+  return typeof parent === "string" && parent.trim().length > 0 ? parent.trim() : undefined;
+}

package/src/shutdown/extractPrompt.ts

@@ -0,0 +1,27 @@
+export function buildShutdownMemoryExportPrompt(conversationText: string): string {
+  return `Extract durable cross-session facts from this coding-agent conversation.
+
+Output markdown ONLY under these optional subsections:
+### Preferences
+### Conventions
+### Findings
+### Todos
+
+Rules:
+- One bullet per standalone fact (no pronouns like "we" without context).
+- Skip subsections with nothing new; output nothing if there are no durable facts.
+- Do NOT include session progress, next actions, or ephemeral tool noise.
+- Do NOT include <private_memory> blocks.
+
+<conversation>
+${conversationText}
+</conversation>`;
+}
+
+/** Wrap shutdown LLM output for parseMemoryExport(). */
+export function wrapShutdownExportMarkdown(body: string): string {
+  const trimmed = body.trim();
+  if (!trimmed) return "";
+  if (/^##\s+Memory Export\s*$/im.test(trimmed)) return trimmed;
+  return `## Memory Export\n${trimmed}`;
+}

package/src/shutdown/processed.ts

@@ -0,0 +1,45 @@
+import { SHUTDOWN_PROCESSED_FILE } from "../constants/memory.js";
+import { joinPath, readText, writeText } from "../utils/fs.js";
+
+export type ShutdownProcessedState = {
+  processed: string[];
+};
+
+export type ShutdownSkipReason =
+  | "missing_session"
+  | "compaction_already_ingested"
+  | "subagent_no_delta"
+  | "no_export"
+  | "no_llm"
+  | "ingested_compaction"
+  | "ingested_llm";
+
+export function shutdownProcessedPath(agentDir: string): string {
+  return joinPath(agentDir, SHUTDOWN_PROCESSED_FILE);
+}
+
+export async function readShutdownProcessedState(agentDir: string): Promise<ShutdownProcessedState> {
+  const raw = await readText(shutdownProcessedPath(agentDir));
+  if (!raw.trim()) return { processed: [] };
+  try {
+    const parsed = JSON.parse(raw) as ShutdownProcessedState;
+    return { processed: Array.isArray(parsed.processed) ? parsed.processed : [] };
+  } catch {
+    return { processed: [] };
+  }
+}
+
+export async function markShutdownProcessed(
+  agentDir: string,
+  sessionFile: string,
+): Promise<void> {
+  const state = await readShutdownProcessedState(agentDir);
+  if (!state.processed.includes(sessionFile)) {
+    state.processed.push(sessionFile);
+  }
+  await writeText(shutdownProcessedPath(agentDir), JSON.stringify(state, null, 2));
+}
+
+export function isShutdownProcessed(state: ShutdownProcessedState, sessionFile: string): boolean {
+  return state.processed.includes(sessionFile);
+}

package/src/shutdown/readQueue.ts

@@ -0,0 +1,26 @@
+import { parseJsonlLine } from "../ipc/jsonlFramer.js";
+import { readText } from "../utils/fs.js";
+
+import { shutdownQueuePath, type ShutdownQueueEntry } from "./enqueue.js";
+
+/** Latest queue row per session file (deduped). */
+export async function readShutdownQueueEntries(agentDir: string): Promise<ShutdownQueueEntry[]> {
+  const raw = await readText(shutdownQueuePath(agentDir));
+  if (!raw.trim()) return [];
+
+  const bySession = new Map<string, ShutdownQueueEntry>();
+
+  for (const line of raw.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try {
+      const entry = parseJsonlLine<ShutdownQueueEntry>(trimmed);
+      if (!entry.sessionFile?.trim()) continue;
+      bySession.set(entry.sessionFile, entry);
+    } catch {
+      // skip malformed lines
+    }
+  }
+
+  return [...bySession.values()];
+}

package/src/shutdown/runDrainJob.ts

@@ -0,0 +1,202 @@
+import type { LlmClient } from "../adapters/llm/types.js";
+import {
+  filterCompactionDelta,
+  shouldSkipSubagentCompactionIngest,
+} from "../compact/subagentDelta.js";
+import { parseMemoryExport } from "../compact/parseMemoryExport.js";
+import { reindex } from "../sidecar/client.js";
+import { ensureSidecarRunning } from "../sidecar/sidecarManager.js";
+import { resolveSidecarPaths } from "../sidecar/paths.js";
+import { sidecarQueryCache } from "../preflight/queryCache.js";
+import type { MemoryStore } from "../store/memoryStore.js";
+import { canRead, pathExists } from "../utils/fs.js";
+
+import {
+  buildShutdownMemoryExportPrompt,
+  wrapShutdownExportMarkdown,
+} from "./extractPrompt.js";
+import { markShutdownProcessed, readShutdownProcessedState, isShutdownProcessed } from "./processed.js";
+import { readShutdownQueueEntries } from "./readQueue.js";
+import type { ShutdownQueueEntry } from "./enqueue.js";
+import { readLatestCompactionSummary, readSessionConversationText } from "./sessionReader.js";
+
+export type DrainShutdownQueueStats = {
+  queued: number;
+  skipped: number;
+  ingested: number;
+  appended: number;
+  indexGeneration?: number;
+};
+
+export type RunDrainShutdownQueueOptions = {
+  store: MemoryStore;
+  agentDir: string;
+  llm?: LlmClient | null;
+  reindex?: boolean;
+};
+
+export type RunDrainShutdownQueueResult =
+  | { status: "empty" }
+  | { status: "drained"; stats: DrainShutdownQueueStats }
+  | { status: "failed"; error: Error };
+
+export async function runDrainShutdownQueueJob(
+  opts: RunDrainShutdownQueueOptions,
+): Promise<RunDrainShutdownQueueResult> {
+  const queue = await readShutdownQueueEntries(opts.agentDir);
+  if (queue.length === 0) return { status: "empty" };
+
+  const processedState = await readShutdownProcessedState(opts.agentDir);
+  const stats: DrainShutdownQueueStats = {
+    queued: queue.length,
+    skipped: 0,
+    ingested: 0,
+    appended: 0,
+  };
+
+  try {
+    for (const entry of queue) {
+      const sessionFile = entry.sessionFile.trim();
+      if (isShutdownProcessed(processedState, sessionFile)) {
+        stats.skipped += 1;
+        continue;
+      }
+
+      const result = await drainQueueEntry({
+        entry,
+        store: opts.store,
+        agentDir: opts.agentDir,
+        llm: opts.llm ?? null,
+      });
+      stats.appended += result.appended;
+      if (result.appended > 0) {
+        stats.ingested += 1;
+      } else {
+        stats.skipped += 1;
+      }
+    }
+
+    if (stats.appended > 0 && opts.reindex !== false) {
+      stats.indexGeneration = await syncSidecarIndex(opts.agentDir, opts.store);
+    }
+
+    return { status: "drained", stats };
+  } catch (error) {
+    return {
+      status: "failed",
+      error: error instanceof Error ? error : new Error(String(error)),
+    };
+  }
+}
+
+async function drainQueueEntry(opts: {
+  entry: ShutdownQueueEntry;
+  store: MemoryStore;
+  agentDir: string;
+  llm: LlmClient | null;
+}): Promise<{ appended: number }> {
+  const sessionFile = opts.entry.sessionFile.trim();
+  if (!sessionFile) {
+    return { appended: 0 };
+  }
+
+  if (!pathExists(sessionFile) || !canRead(sessionFile)) {
+    await markShutdownProcessed(opts.agentDir, sessionFile);
+    return { appended: 0 };
+  }
+
+  const compaction = await readLatestCompactionSummary(sessionFile);
+  if (compaction && (await opts.store.hasProcessedCompaction(compaction.compactionId))) {
+    await markShutdownProcessed(opts.agentDir, sessionFile);
+    return { appended: 0 };
+  }
+
+  if (compaction) {
+    const appended = await ingestFromSummary({
+      store: opts.store,
+      agentDir: opts.agentDir,
+      sessionFile,
+      summary: compaction.summary,
+      compactionId: compaction.compactionId,
+      subagent: opts.entry.isSubagent,
+    });
+    return { appended };
+  }
+
+  if (!opts.llm) {
+    await markShutdownProcessed(opts.agentDir, sessionFile);
+    return { appended: 0 };
+  }
+
+  const conversationText = await readSessionConversationText(sessionFile);
+  if (!conversationText?.trim()) {
+    await markShutdownProcessed(opts.agentDir, sessionFile);
+    return { appended: 0 };
+  }
+
+  const prompt = buildShutdownMemoryExportPrompt(conversationText);
+  const raw = await opts.llm.complete(prompt);
+  const summary = wrapShutdownExportMarkdown(raw);
+  if (!summary.trim()) {
+    await markShutdownProcessed(opts.agentDir, sessionFile);
+    return { appended: 0 };
+  }
+
+  const appended = await ingestFromSummary({
+    store: opts.store,
+    agentDir: opts.agentDir,
+    sessionFile,
+    summary,
+    subagent: opts.entry.isSubagent,
+  });
+  return { appended };
+}
+
+async function ingestFromSummary(opts: {
+  store: MemoryStore;
+  agentDir: string;
+  sessionFile: string;
+  summary: string;
+  compactionId?: string;
+  subagent?: boolean;
+}): Promise<number> {
+  const parsed = parseMemoryExport(opts.summary);
+  if (parsed.length === 0) {
+    if (opts.compactionId) {
+      await opts.store.markCompactionProcessed(opts.compactionId);
+    }
+    await markShutdownProcessed(opts.agentDir, opts.sessionFile);
+    return 0;
+  }
+
+  let entries = parsed;
+  if (opts.subagent) {
+    const existing = await opts.store.listEntries();
+    entries = filterCompactionDelta(parsed, existing);
+    if (shouldSkipSubagentCompactionIngest(parsed, entries)) {
+      if (opts.compactionId) {
+        await opts.store.markCompactionProcessed(opts.compactionId);
+      }
+      await markShutdownProcessed(opts.agentDir, opts.sessionFile);
+      return 0;
+    }
+  }
+
+  if (entries.length > 0) {
+    await opts.store.appendMany(entries, { mode: "ifAbsent" });
+  }
+
+  if (opts.compactionId) {
+    await opts.store.markCompactionProcessed(opts.compactionId);
+  }
+  await markShutdownProcessed(opts.agentDir, opts.sessionFile);
+  return entries.length;
+}
+
+async function syncSidecarIndex(agentDir: string, store: MemoryStore): Promise<number> {
+  const sidecar = resolveSidecarPaths(agentDir);
+  await ensureSidecarRunning(sidecar);
+  const result = await reindex(sidecar.socketPath, await store.exportForIndex());
+  sidecarQueryCache.onReindexComplete(agentDir, result.index_generation);
+  return result.index_generation;
+}

package/src/shutdown/sessionReader.ts

@@ -0,0 +1,61 @@
+import type { AgentMessage } from "@earendil-works/pi-agent-core";
+import {
+  buildSessionContext,
+  convertToLlm,
+  getLatestCompactionEntry,
+  parseSessionEntries,
+  serializeConversation,
+  type FileEntry,
+  type SessionEntry,
+} from "@earendil-works/pi-coding-agent";
+
+import { stripPrivateMemory } from "../preflight/strip.js";
+import { readTextRequired } from "../utils/fs.js";
+
+function stripPrivateMemoryFromMessages(messages: AgentMessage[]): AgentMessage[] {
+  return messages.map((message) => {
+    if (message.role !== "user") return message;
+    if (typeof message.content === "string") {
+      return { ...message, content: stripPrivateMemory(message.content) };
+    }
+    return {
+      ...message,
+      content: message.content.map((block) =>
+        block.type === "text" ? { ...block, text: stripPrivateMemory(block.text) } : block,
+      ),
+    } as AgentMessage;
+  });
+}
+
+function toSessionEntries(fileEntries: FileEntry[]): SessionEntry[] {
+  return fileEntries.filter((entry): entry is SessionEntry => entry.type !== "session");
+}
+
+function resolveLeafId(entries: SessionEntry[]): string | null {
+  return entries.at(-1)?.id ?? null;
+}
+
+export async function readSessionConversationText(sessionFile: string): Promise<string | null> {
+  const content = await readTextRequired(sessionFile);
+  const fileEntries = parseSessionEntries(content);
+  const entries = toSessionEntries(fileEntries);
+  if (entries.length === 0) return null;
+
+  const context = buildSessionContext(entries, resolveLeafId(entries));
+  const messages = stripPrivateMemoryFromMessages(context.messages);
+  if (messages.length === 0) return null;
+
+  return serializeConversation(convertToLlm(messages));
+}
+
+export async function readLatestCompactionSummary(sessionFile: string): Promise<{
+  compactionId: string;
+  summary: string;
+} | null> {
+  const content = await readTextRequired(sessionFile);
+  const fileEntries = parseSessionEntries(content);
+  const entries = toSessionEntries(fileEntries);
+  const compaction = getLatestCompactionEntry(entries);
+  if (!compaction?.summary?.trim()) return null;
+  return { compactionId: compaction.id, summary: compaction.summary };
+}