@chendpoc/pi-memory 0.1.13 → 0.2.0

package/src/shutdown/processed.ts

@@ -0,0 +1,45 @@
+import { SHUTDOWN_PROCESSED_FILE } from "../constants/memory.js";
+import { joinPath, readText, writeText } from "../utils/fs.js";
+
+export type ShutdownProcessedState = {
+  processed: string[];
+};
+
+export type ShutdownSkipReason =
+  | "missing_session"
+  | "compaction_already_ingested"
+  | "subagent_no_delta"
+  | "no_export"
+  | "no_llm"
+  | "ingested_compaction"
+  | "ingested_llm";
+
+export function shutdownProcessedPath(agentDir: string): string {
+  return joinPath(agentDir, SHUTDOWN_PROCESSED_FILE);
+}
+
+export async function readShutdownProcessedState(agentDir: string): Promise<ShutdownProcessedState> {
+  const raw = await readText(shutdownProcessedPath(agentDir));
+  if (!raw.trim()) return { processed: [] };
+  try {
+    const parsed = JSON.parse(raw) as ShutdownProcessedState;
+    return { processed: Array.isArray(parsed.processed) ? parsed.processed : [] };
+  } catch {
+    return { processed: [] };
+  }
+}
+
+export async function markShutdownProcessed(
+  agentDir: string,
+  sessionFile: string,
+): Promise<void> {
+  const state = await readShutdownProcessedState(agentDir);
+  if (!state.processed.includes(sessionFile)) {
+    state.processed.push(sessionFile);
+  }
+  await writeText(shutdownProcessedPath(agentDir), JSON.stringify(state, null, 2));
+}
+
+export function isShutdownProcessed(state: ShutdownProcessedState, sessionFile: string): boolean {
+  return state.processed.includes(sessionFile);
+}

package/src/shutdown/readQueue.ts

@@ -0,0 +1,26 @@
+import { parseJsonlLine } from "../ipc/jsonlFramer.js";
+import { readText } from "../utils/fs.js";
+
+import { shutdownQueuePath, type ShutdownQueueEntry } from "./enqueue.js";
+
+/** Latest queue row per session file (deduped). */
+export async function readShutdownQueueEntries(agentDir: string): Promise<ShutdownQueueEntry[]> {
+  const raw = await readText(shutdownQueuePath(agentDir));
+  if (!raw.trim()) return [];
+
+  const bySession = new Map<string, ShutdownQueueEntry>();
+
+  for (const line of raw.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try {
+      const entry = parseJsonlLine<ShutdownQueueEntry>(trimmed);
+      if (!entry.sessionFile?.trim()) continue;
+      bySession.set(entry.sessionFile, entry);
+    } catch {
+      // skip malformed lines
+    }
+  }
+
+  return [...bySession.values()];
+}

package/src/shutdown/runDrainJob.ts

@@ -0,0 +1,202 @@
+import type { LlmClient } from "../adapters/llm/types.js";
+import {
+  filterCompactionDelta,
+  shouldSkipSubagentCompactionIngest,
+} from "../compact/subagentDelta.js";
+import { parseMemoryExport } from "../compact/parseMemoryExport.js";
+import { reindex } from "../sidecar/client.js";
+import { ensureSidecarRunning } from "../sidecar/sidecarManager.js";
+import { resolveSidecarPaths } from "../sidecar/paths.js";
+import { sidecarQueryCache } from "../preflight/queryCache.js";
+import type { MemoryStore } from "../store/memoryStore.js";
+import { canRead, pathExists } from "../utils/fs.js";
+
+import {
+  buildShutdownMemoryExportPrompt,
+  wrapShutdownExportMarkdown,
+} from "./extractPrompt.js";
+import { markShutdownProcessed, readShutdownProcessedState, isShutdownProcessed } from "./processed.js";
+import { readShutdownQueueEntries } from "./readQueue.js";
+import type { ShutdownQueueEntry } from "./enqueue.js";
+import { readLatestCompactionSummary, readSessionConversationText } from "./sessionReader.js";
+
+export type DrainShutdownQueueStats = {
+  queued: number;
+  skipped: number;
+  ingested: number;
+  appended: number;
+  indexGeneration?: number;
+};
+
+export type RunDrainShutdownQueueOptions = {
+  store: MemoryStore;
+  agentDir: string;
+  llm?: LlmClient | null;
+  reindex?: boolean;
+};
+
+export type RunDrainShutdownQueueResult =
+  | { status: "empty" }
+  | { status: "drained"; stats: DrainShutdownQueueStats }
+  | { status: "failed"; error: Error };
+
+export async function runDrainShutdownQueueJob(
+  opts: RunDrainShutdownQueueOptions,
+): Promise<RunDrainShutdownQueueResult> {
+  const queue = await readShutdownQueueEntries(opts.agentDir);
+  if (queue.length === 0) return { status: "empty" };
+
+  const processedState = await readShutdownProcessedState(opts.agentDir);
+  const stats: DrainShutdownQueueStats = {
+    queued: queue.length,
+    skipped: 0,
+    ingested: 0,
+    appended: 0,
+  };
+
+  try {
+    for (const entry of queue) {
+      const sessionFile = entry.sessionFile.trim();
+      if (isShutdownProcessed(processedState, sessionFile)) {
+        stats.skipped += 1;
+        continue;
+      }
+
+      const result = await drainQueueEntry({
+        entry,
+        store: opts.store,
+        agentDir: opts.agentDir,
+        llm: opts.llm ?? null,
+      });
+      stats.appended += result.appended;
+      if (result.appended > 0) {
+        stats.ingested += 1;
+      } else {
+        stats.skipped += 1;
+      }
+    }
+
+    if (stats.appended > 0 && opts.reindex !== false) {
+      stats.indexGeneration = await syncSidecarIndex(opts.agentDir, opts.store);
+    }
+
+    return { status: "drained", stats };
+  } catch (error) {
+    return {
+      status: "failed",
+      error: error instanceof Error ? error : new Error(String(error)),
+    };
+  }
+}
+
+async function drainQueueEntry(opts: {
+  entry: ShutdownQueueEntry;
+  store: MemoryStore;
+  agentDir: string;
+  llm: LlmClient | null;
+}): Promise<{ appended: number }> {
+  const sessionFile = opts.entry.sessionFile.trim();
+  if (!sessionFile) {
+    return { appended: 0 };
+  }
+
+  if (!pathExists(sessionFile) || !canRead(sessionFile)) {
+    await markShutdownProcessed(opts.agentDir, sessionFile);
+    return { appended: 0 };
+  }
+
+  const compaction = await readLatestCompactionSummary(sessionFile);
+  if (compaction && (await opts.store.hasProcessedCompaction(compaction.compactionId))) {
+    await markShutdownProcessed(opts.agentDir, sessionFile);
+    return { appended: 0 };
+  }
+
+  if (compaction) {
+    const appended = await ingestFromSummary({
+      store: opts.store,
+      agentDir: opts.agentDir,
+      sessionFile,
+      summary: compaction.summary,
+      compactionId: compaction.compactionId,
+      subagent: opts.entry.isSubagent,
+    });
+    return { appended };
+  }
+
+  if (!opts.llm) {
+    await markShutdownProcessed(opts.agentDir, sessionFile);
+    return { appended: 0 };
+  }
+
+  const conversationText = await readSessionConversationText(sessionFile);
+  if (!conversationText?.trim()) {
+    await markShutdownProcessed(opts.agentDir, sessionFile);
+    return { appended: 0 };
+  }
+
+  const prompt = buildShutdownMemoryExportPrompt(conversationText);
+  const raw = await opts.llm.complete(prompt);
+  const summary = wrapShutdownExportMarkdown(raw);
+  if (!summary.trim()) {
+    await markShutdownProcessed(opts.agentDir, sessionFile);
+    return { appended: 0 };
+  }
+
+  const appended = await ingestFromSummary({
+    store: opts.store,
+    agentDir: opts.agentDir,
+    sessionFile,
+    summary,
+    subagent: opts.entry.isSubagent,
+  });
+  return { appended };
+}
+
+async function ingestFromSummary(opts: {
+  store: MemoryStore;
+  agentDir: string;
+  sessionFile: string;
+  summary: string;
+  compactionId?: string;
+  subagent?: boolean;
+}): Promise<number> {
+  const parsed = parseMemoryExport(opts.summary);
+  if (parsed.length === 0) {
+    if (opts.compactionId) {
+      await opts.store.markCompactionProcessed(opts.compactionId);
+    }
+    await markShutdownProcessed(opts.agentDir, opts.sessionFile);
+    return 0;
+  }
+
+  let entries = parsed;
+  if (opts.subagent) {
+    const existing = await opts.store.listEntries();
+    entries = filterCompactionDelta(parsed, existing);
+    if (shouldSkipSubagentCompactionIngest(parsed, entries)) {
+      if (opts.compactionId) {
+        await opts.store.markCompactionProcessed(opts.compactionId);
+      }
+      await markShutdownProcessed(opts.agentDir, opts.sessionFile);
+      return 0;
+    }
+  }
+
+  if (entries.length > 0) {
+    await opts.store.appendMany(entries, { mode: "ifAbsent" });
+  }
+
+  if (opts.compactionId) {
+    await opts.store.markCompactionProcessed(opts.compactionId);
+  }
+  await markShutdownProcessed(opts.agentDir, opts.sessionFile);
+  return entries.length;
+}
+
+async function syncSidecarIndex(agentDir: string, store: MemoryStore): Promise<number> {
+  const sidecar = resolveSidecarPaths(agentDir);
+  await ensureSidecarRunning(sidecar);
+  const result = await reindex(sidecar.socketPath, await store.exportForIndex());
+  sidecarQueryCache.onReindexComplete(agentDir, result.index_generation);
+  return result.index_generation;
+}

package/src/shutdown/sessionReader.ts

@@ -0,0 +1,61 @@
+import type { AgentMessage } from "@earendil-works/pi-agent-core";
+import {
+  buildSessionContext,
+  convertToLlm,
+  getLatestCompactionEntry,
+  parseSessionEntries,
+  serializeConversation,
+  type FileEntry,
+  type SessionEntry,
+} from "@earendil-works/pi-coding-agent";
+
+import { stripPrivateMemory } from "../preflight/strip.js";
+import { readTextRequired } from "../utils/fs.js";
+
+function stripPrivateMemoryFromMessages(messages: AgentMessage[]): AgentMessage[] {
+  return messages.map((message) => {
+    if (message.role !== "user") return message;
+    if (typeof message.content === "string") {
+      return { ...message, content: stripPrivateMemory(message.content) };
+    }
+    return {
+      ...message,
+      content: message.content.map((block) =>
+        block.type === "text" ? { ...block, text: stripPrivateMemory(block.text) } : block,
+      ),
+    } as AgentMessage;
+  });
+}
+
+function toSessionEntries(fileEntries: FileEntry[]): SessionEntry[] {
+  return fileEntries.filter((entry): entry is SessionEntry => entry.type !== "session");
+}
+
+function resolveLeafId(entries: SessionEntry[]): string | null {
+  return entries.at(-1)?.id ?? null;
+}
+
+export async function readSessionConversationText(sessionFile: string): Promise<string | null> {
+  const content = await readTextRequired(sessionFile);
+  const fileEntries = parseSessionEntries(content);
+  const entries = toSessionEntries(fileEntries);
+  if (entries.length === 0) return null;
+
+  const context = buildSessionContext(entries, resolveLeafId(entries));
+  const messages = stripPrivateMemoryFromMessages(context.messages);
+  if (messages.length === 0) return null;
+
+  return serializeConversation(convertToLlm(messages));
+}
+
+export async function readLatestCompactionSummary(sessionFile: string): Promise<{
+  compactionId: string;
+  summary: string;
+} | null> {
+  const content = await readTextRequired(sessionFile);
+  const fileEntries = parseSessionEntries(content);
+  const entries = toSessionEntries(fileEntries);
+  const compaction = getLatestCompactionEntry(entries);
+  if (!compaction?.summary?.trim()) return null;
+  return { compactionId: compaction.id, summary: compaction.summary };
+}

package/src/sidecar/client.ts

@@ -1,166 +1,114 @@
-import { randomBytes } from "node:crypto";
-import http from "node:http";
+// Agent 侧：发送请求到 sidecar
+import { randomUUID } from "node:crypto";
+import net from "node:net";
 
 import {
-  classifyHTTP,
-  classifyTransportError,
-  ErrTransport,
-  type ErrorClass,
-} from "../errclass.js";
-import type {
-  HealthPayload,
-  QueryIntent,
-  QueryRequest,
-  ReloadResponse,
-  ResponseEnvelope,
-} from "../types.js";
+  SIDECAR_PING_TIMEOUT_MS,
+  SIDECAR_QUERY_TIMEOUT_MS,
+  SIDECAR_REINDEX_TIMEOUT_MS,
+} from "../constants/timing.js";
+import { JsonlFramer, parseJsonlLine, serializeJsonlFrame } from "../ipc/jsonlFramer.js";
+import { isErrorResponse, type IndexDocument, type IndexStats, type SidecarResponse } from "./protocol.js";
 
-function requestId(): string {
-  return `req-${randomBytes(6).toString("hex")}`;
-}
+export function sidecarRequest<T extends SidecarResponse>(
+  socketPath: string,
+  frame: Record<string, unknown>,
+  timeoutMs = SIDECAR_QUERY_TIMEOUT_MS,
+): Promise<T> {
+  return new Promise((resolve, reject) => {
+    const socket = net.connect(socketPath);
+    const framer = new JsonlFramer();
 
-export class SidecarClient {
-  constructor(
-    private readonly socketPath: string,
-    private readonly timeoutMs: number,
-  ) {}
+    const timer = setTimeout(() => {
+      socket.destroy();
+      reject(new Error("Sidecar request timed out"));
+    }, timeoutMs);
 
-  async health(signal?: AbortSignal): Promise<HealthPayload> {
-    return this.do<HealthPayload>("GET", "/health", undefined, signal);
-  }
+    socket.on("connect", () => {
+      socket.write(serializeJsonlFrame(frame));
+    });
+
+    socket.on("data", (chunk) => {
+      for (const line of framer.push(chunk.toString())) {
+        clearTimeout(timer);
+        socket.end();
+
+        let response: SidecarResponse;
+        try {
+          response = parseJsonlLine<SidecarResponse>(line);
+        } catch {
+          reject(new Error("Invalid JSON response from sidecar"));
+          return;
+        }
+
+        if (isErrorResponse(response)) {
+          reject(new Error(response.error));
+          return;
+        }
 
-  async query(
-    intent: QueryIntent,
-    signal?: AbortSignal,
-  ): Promise<{
-    env: ResponseEnvelope | null;
-    errorClass: ErrorClass;
-    transportError?: Error;
-  }> {
-    const rid = requestId();
-    const body: QueryRequest = { intent, request_id: rid };
-    try {
-      const { status, data } = await this.doRaw<ResponseEnvelope>(
-        "POST",
-        "/query",
-        body,
-        signal,
-        rid,
-      );
-      return {
-        env: data,
-        errorClass: classifyHTTP(status, data),
-      };
-    } catch (err) {
-      if (err instanceof Error && err.message.includes("transport")) {
-        return {
-          env: null,
-          errorClass: classifyTransportError(err),
-          transportError: err,
-        };
+        resolve(response as T);
+        return;
       }
-      return {
-        env: null,
-        errorClass: "unavailable",
-        transportError: err instanceof Error ? err : new Error(String(err)),
-      };
-    }
-  }
+    });
 
-  async reload(signal?: AbortSignal): Promise<ReloadResponse> {
-    return this.do<ReloadResponse>("POST", "/bundle/reload", {}, signal);
-  }
+    socket.on("error", reject);
+  });
+}
 
-  private async do<T>(
-    method: string,
-    path: string,
-    body: unknown,
-    signal?: AbortSignal,
-    requestIdHeader?: string,
-  ): Promise<T> {
-    const { data } = await this.doRaw<T>(
-      method,
-      path,
-      body,
-      signal,
-      requestIdHeader,
+export async function ping(socketPath: string): Promise<boolean> {
+  try {
+    const res = await sidecarRequest<Extract<SidecarResponse, { type: "pong" }>>(
+      socketPath,
+      { type: "ping" },
+      SIDECAR_PING_TIMEOUT_MS,
     );
-    return data;
+    return res.type === "pong";
+  } catch {
+    return false;
   }
+}
 
-  private doRaw<T>(
-    method: string,
-    path: string,
-    body: unknown,
-    signal?: AbortSignal,
-    requestIdHeader?: string,
-  ): Promise<{ status: number; data: T }> {
-    return new Promise((resolve, reject) => {
-      const payload = body != null ? JSON.stringify(body) : undefined;
-      const req = http.request(
-        {
-          socketPath: this.socketPath,
-          path,
-          method,
-          headers: {
-            "X-Request-ID": requestIdHeader ?? requestId(),
-            ...(payload
-              ? {
-                  "Content-Type": "application/json",
-                  "Content-Length": Buffer.byteLength(payload),
-                }
-              : {}),
-          },
-          timeout: this.timeoutMs,
-        },
-        (res) => {
-          const chunks: Buffer[] = [];
-          res.on("data", (c) => chunks.push(c));
-          res.on("end", () => {
-            const text = Buffer.concat(chunks).toString("utf8");
-            if (!text) {
-              reject(new Error(`${ErrTransport.message}: empty body`));
-              return;
-            }
-            try {
-              resolve({
-                status: res.statusCode ?? 0,
-                data: JSON.parse(text) as T,
-              });
-            } catch (e) {
-              reject(
-                new Error(
-                  `${ErrTransport.message}: decode: ${e instanceof Error ? e.message : e}`,
-                ),
-              );
-            }
-          });
-        },
-      );
+export async function fetchIndexStats(
+  socketPath: string,
+): Promise<{ stats: IndexStats } | { error: string }> {
+  try {
+    const res = await sidecarRequest<Extract<SidecarResponse, { type: "stats_ok" }>>(
+      socketPath,
+      { type: "stats" },
+      SIDECAR_PING_TIMEOUT_MS,
+    );
+    if (res.type !== "stats_ok") return { error: "unexpected sidecar response" };
+    const { type: _type, ...stats } = res;
+    return { stats };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return { error: message };
+  }
+}
 
-      req.on("timeout", () => {
-        req.destroy(new Error(`${ErrTransport.message}: timeout`));
-      });
-      req.on("error", (err) => {
-        reject(new Error(`${ErrTransport.message}: ${err.message}`));
-      });
+export async function query(
+  socketPath: string,
+  queryText: string,
+  timeoutMs = SIDECAR_QUERY_TIMEOUT_MS,
+) {
+  const request_id = randomUUID();
+  return sidecarRequest<Extract<SidecarResponse, { type: "result" }>>(
+    socketPath,
+    { type: "query", request_id, query: queryText },
+    timeoutMs,
+  );
+}
 
-      if (signal) {
-        if (signal.aborted) {
-          req.destroy(new Error(`${ErrTransport.message}: aborted`));
-          return;
-        }
-        signal.addEventListener(
-          "abort",
-          () => {
-            req.destroy(new Error(`${ErrTransport.message}: aborted`));
-          },
-          { once: true },
-        );
-      }
+export type ReindexResult = Extract<SidecarResponse, { type: "reindex_ok" }>;
 
-      if (payload) req.write(payload);
-      req.end();
-    });
-  }
+export async function reindex(
+  socketPath: string,
+  documents: IndexDocument[] = [],
+): Promise<ReindexResult> {
+  const request_id = randomUUID();
+  return sidecarRequest<ReindexResult>(
+    socketPath,
+    { type: "reindex", request_id, documents },
+    SIDECAR_REINDEX_TIMEOUT_MS,
+  );
 }

package/src/sidecar/paths.ts

@@ -0,0 +1,31 @@
+import { fileURLToPath } from "node:url";
+
+import { SIDECAR_DB_FILE, SIDECAR_SOCKET_FILE } from "../constants/paths.js";
+import { canRead, joinPath, pathDirname } from "../utils/fs.js";
+
+export type SidecarPaths = {
+  socketPath: string;
+  dbPath: string;
+};
+
+export function resolveSidecarPaths(agentDir: string): SidecarPaths {
+  return {
+    socketPath: joinPath(agentDir, SIDECAR_SOCKET_FILE),
+    dbPath: joinPath(agentDir, SIDECAR_DB_FILE),
+  };
+}
+
+/** Resolve compiled sidecar entry (dist/sidecar/server/process.js). */
+export function resolveSidecarEntry(): string {
+  const here = pathDirname(fileURLToPath(import.meta.url));
+  const candidates = [
+    joinPath(here, "server/process.js"),
+    joinPath(here, "../../dist/sidecar/server/process.js"),
+  ];
+
+  for (const entry of candidates) {
+    if (canRead(entry)) return entry;
+  }
+
+  throw new Error("Sidecar entry not found; run `pnpm build` first");
+}

package/src/sidecar/protocol.ts

@@ -0,0 +1,38 @@
+export type MemoryEntry = {
+  content: string;
+  relevance: number;
+  timestamp: string;
+  source: string;
+};
+
+export type IndexDocument = {
+  id: string;
+  content: string;
+  source: string;
+  timestamp: string;
+};
+
+export type SidecarRequest =
+  | { type: "ping" }
+  | { type: "stats" }
+  | { type: "query"; request_id: string; query: string }
+  | { type: "reindex"; request_id: string; documents?: IndexDocument[] };
+
+export type IndexStats = {
+  index_generation: number;
+  chunk_count: number;
+  embedding_provider?: string;
+  embedding_model?: string;
+  embedding_dim?: number;
+};
+
+export type SidecarResponse =
+  | { type: "pong" }
+  | { type: "stats_ok" } & IndexStats
+  | { type: "result"; request_id: string; results: MemoryEntry[] }
+  | { type: "reindex_ok"; request_id: string; indexed: number; index_generation: number }
+  | { type: "error"; request_id?: string; error: string };
+
+export function isErrorResponse(res: SidecarResponse): res is Extract<SidecarResponse, { type: "error" }> {
+  return res.type === "error";
+}