@chendpoc/pi-memory 0.1.13 → 0.2.0

package/src/sidecar/reindexBridge.ts

@@ -0,0 +1,52 @@
+import debounce from "lodash/debounce.js";
+
+import { DEFAULT_REINDEX_DEBOUNCE_MS } from "../constants/timing.js";
+
+import { sidecarQueryCache } from "../preflight/queryCache.js";
+import { ensureSidecarRunning, type SidecarOpts } from "./sidecarManager.js";
+import { reindex } from "./client.js";
+import type { IndexDocument } from "./protocol.js";
+
+export type ReindexScheduler = {
+  schedule(): void;
+  runNow(): Promise<void>;
+};
+
+export function createReindexScheduler(opts: {
+  sidecar: SidecarOpts;
+  agentDir: string;
+  getDocuments: () => Promise<IndexDocument[]>;
+  debounceMs?: number;
+}): ReindexScheduler {
+  let running = false;
+  let pending = false;
+
+  const runNow = async (): Promise<void> => {
+    if (running) {
+      pending = true;
+      return;
+    }
+
+    running = true;
+    try {
+      await ensureSidecarRunning(opts.sidecar);
+      const documents = await opts.getDocuments();
+      const result = await reindex(opts.sidecar.socketPath, documents);
+      sidecarQueryCache.onReindexComplete(opts.agentDir, result.index_generation);
+    } catch {
+      // fail-silent; preflight falls back to MEMORY.md
+    } finally {
+      running = false;
+      if (pending) {
+        pending = false;
+        await runNow();
+      }
+    }
+  };
+
+  const schedule = debounce(() => {
+    void runNow();
+  }, opts.debounceMs ?? DEFAULT_REINDEX_DEBOUNCE_MS);
+
+  return { schedule, runNow };
+}

package/src/sidecar/server/process.ts

@@ -0,0 +1,27 @@
+// Sidecar 子进程入口
+import { loadEnv } from "../../config/loadEnv.js";
+import { createSidecarServer } from "./server.js";
+
+loadEnv();
+
+function readArg(flag: string): string {
+  const idx = process.argv.indexOf(flag);
+  const value = idx === -1 ? undefined : process.argv[idx + 1];
+  if (!value) {
+    throw new Error(`Missing required argument: ${flag}`);
+  }
+  return value;
+}
+
+const socketPath = readArg("--socket");
+const dbPath = readArg("--db");
+
+const { shutdown } = createSidecarServer({ socketPath, dbPath });
+
+function onSignal(): void {
+  shutdown();
+  process.exit(0);
+}
+
+process.on("SIGTERM", onSignal);
+process.on("SIGINT", onSignal);

package/src/sidecar/server/query.ts

@@ -0,0 +1,16 @@
+import type { MemoryEntry, SidecarResponse } from "../protocol.js";
+import { getVecStore } from "./vec/store.js";
+
+export type QueryContext = {
+  dbPath: string;
+};
+
+export async function handleQuery(
+  requestId: string,
+  queryText: string,
+  ctx: QueryContext,
+): Promise<Extract<SidecarResponse, { type: "result" }>> {
+  const store = getVecStore(ctx.dbPath);
+  const results: MemoryEntry[] = await store.query(queryText);
+  return { type: "result", request_id: requestId, results };
+}

package/src/sidecar/server/reindex.ts

@@ -0,0 +1,21 @@
+import type { IndexDocument, SidecarResponse } from "../protocol.js";
+import { getVecStore } from "./vec/store.js";
+
+export type ReindexContext = {
+  dbPath: string;
+};
+
+export async function handleReindex(
+  requestId: string,
+  ctx: ReindexContext,
+  documents: IndexDocument[] = [],
+): Promise<Extract<SidecarResponse, { type: "reindex_ok" }>> {
+  const store = getVecStore(ctx.dbPath);
+  const outcome = await store.reindex(documents);
+  return {
+    type: "reindex_ok",
+    request_id: requestId,
+    indexed: outcome.indexed,
+    index_generation: outcome.indexGeneration,
+  };
+}

package/src/sidecar/server/server.ts

@@ -0,0 +1,105 @@
+import { createServer, type Server, type Socket } from "node:net";
+import { writeFileSync } from "node:fs";
+
+import { SIDECAR_PID_SUFFIX } from "../../constants/paths.js";
+import { JsonlFramer, parseJsonlLine, serializeJsonlFrame } from "../../ipc/jsonlFramer.js";
+import { ensureDirSync, pathDirname } from "../../utils/fs.js";
+import { cleanupSocketFiles, removeSocketFile, secureSocketPath } from "../../utils/socket.js";
+import type { SidecarRequest, SidecarResponse } from "../protocol.js";
+import { handleQuery } from "./query.js";
+import { handleReindex } from "./reindex.js";
+import { handleStats } from "./stats.js";
+
+export type SidecarServerOpts = {
+  socketPath: string;
+  dbPath: string;
+};
+
+export type SidecarServer = {
+  server: Server;
+  shutdown: () => void;
+};
+
+function writeResponse(socket: Socket, response: SidecarResponse): void {
+  socket.write(serializeJsonlFrame(response));
+}
+
+function writeError(socket: Socket, error: string, requestId?: string): void {
+  writeResponse(socket, { type: "error", request_id: requestId, error });
+}
+
+async function dispatchFrame(
+  frame: SidecarRequest,
+  socket: Socket,
+  ctx: SidecarServerOpts,
+): Promise<void> {
+  switch (frame.type) {
+    case "ping":
+      writeResponse(socket, { type: "pong" });
+      return;
+    case "stats":
+      writeResponse(socket, handleStats({ dbPath: ctx.dbPath }));
+      return;
+    case "query":
+      writeResponse(socket, await handleQuery(frame.request_id, frame.query, { dbPath: ctx.dbPath }));
+      return;
+    case "reindex":
+      writeResponse(
+        socket,
+        await handleReindex(frame.request_id, { dbPath: ctx.dbPath }, frame.documents),
+      );
+      return;
+    default: {
+      const unknown = frame as { type?: string; request_id?: string };
+      writeError(socket, `unknown frame type: ${unknown.type ?? "undefined"}`, unknown.request_id);
+    }
+  }
+}
+
+export function createSidecarServer(opts: SidecarServerOpts): SidecarServer {
+  const pidPath = opts.socketPath + SIDECAR_PID_SUFFIX;
+
+  ensureDirSync(pathDirname(opts.socketPath));
+  removeSocketFile(opts.socketPath);
+
+  const server = createServer((socket) => {
+    const framer = new JsonlFramer();
+
+    socket.on("data", (chunk) => {
+      for (const line of framer.push(chunk.toString())) {
+        let frame: SidecarRequest;
+        try {
+          frame = parseJsonlLine<SidecarRequest>(line);
+        } catch {
+          writeError(socket, "invalid JSON frame");
+          continue;
+        }
+
+        void dispatchFrame(frame, socket, opts).catch((error: unknown) => {
+          const message = error instanceof Error ? error.message : String(error);
+          const requestId = typeof frame === "object" && frame && "request_id" in frame
+            ? String(frame.request_id)
+            : undefined;
+          writeError(socket, message, requestId);
+        });
+      }
+    });
+  });
+
+  function shutdown(): void {
+    server.close();
+    cleanupSocketFiles(opts.socketPath, pidPath);
+  }
+
+  server.listen(opts.socketPath, () => {
+    writeFileSync(pidPath, String(process.pid));
+    secureSocketPath(opts.socketPath);
+  });
+
+  server.on("error", (error) => {
+    console.error("Sidecar server error:", error);
+    process.exit(1);
+  });
+
+  return { server, shutdown };
+}

package/src/sidecar/server/stats.ts

@@ -0,0 +1,23 @@
+import type { SidecarResponse } from "../protocol.js";
+import { getVecStore } from "./vec/store.js";
+
+export type StatsContext = {
+  dbPath: string;
+};
+
+export function handleStats(ctx: StatsContext): Extract<SidecarResponse, { type: "stats_ok" }> {
+  const store = getVecStore(ctx.dbPath);
+  const meta = store.getStoredEmbeddingMeta();
+  return {
+    type: "stats_ok",
+    index_generation: store.getIndexGeneration(),
+    chunk_count: store.getChunkCount(),
+    ...(meta
+      ? {
+          embedding_provider: meta.provider,
+          embedding_model: meta.model,
+          embedding_dim: meta.dim,
+        }
+      : {}),
+  };
+}

package/src/sidecar/server/vec/embedder.ts

@@ -0,0 +1 @@
+export { getEmbedder, resetEmbedderForTests, createEmbedder, type Embedder } from "../../../adapters/embed/factory.js";

package/src/sidecar/server/vec/mmr.ts

@@ -0,0 +1,55 @@
+import { MMR_LAMBDA } from "../../../constants/retrieval.js";
+
+export type ScoredCandidate = {
+  chunkId: string;
+  content: string;
+  source: string;
+  timestamp: string;
+  distance: number;
+  embedding: Float32Array;
+};
+
+export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
+  let dot = 0;
+  for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
+  return dot;
+}
+
+/** MMR re-ranking: λ defaults from retrieval config (see constants/retrieval.ts). */
+export function mmrSelect(
+  queryEmbedding: Float32Array,
+  candidates: ScoredCandidate[],
+  limit: number,
+  lambda = MMR_LAMBDA,
+): ScoredCandidate[] {
+  if (candidates.length === 0) return [];
+
+  const selected: ScoredCandidate[] = [];
+  const remaining = [...candidates];
+
+  while (selected.length < limit && remaining.length > 0) {
+    let bestIdx = 0;
+    let bestScore = -Infinity;
+
+    for (let i = 0; i < remaining.length; i++) {
+      const candidate = remaining[i]!;
+      const relevance = cosineSimilarity(queryEmbedding, candidate.embedding);
+      const maxSim = selected.length === 0
+        ? 0
+        : Math.max(...selected.map((s) => cosineSimilarity(candidate.embedding, s.embedding)));
+      const score = lambda * relevance - (1 - lambda) * maxSim;
+      if (score > bestScore) {
+        bestScore = score;
+        bestIdx = i;
+      }
+    }
+
+    selected.push(remaining.splice(bestIdx, 1)[0]!);
+  }
+
+  return selected;
+}
+
+export function distanceToRelevance(distance: number): number {
+  return Math.max(0, Math.min(1, 1 - distance));
+}

package/src/sidecar/server/vec/store.ts

@@ -0,0 +1,232 @@
+import { createRequire } from "node:module";
+
+import type { Embedder } from "../../../adapters/embed/types.js";
+import type { IndexDocument, MemoryEntry } from "../../protocol.js";
+import { ensureDirSync, pathDirname } from "../../../utils/fs.js";
+import { readRetrievalConfig } from "../../../config/retrieval.js";
+import { getEmbedder } from "./embedder.js";
+import { cosineSimilarity, distanceToRelevance, mmrSelect, type ScoredCandidate } from "./mmr.js";
+
+const require = createRequire(import.meta.url);
+
+type ChunkRow = {
+  chunk_id: string;
+  content: string;
+  source: string;
+  timestamp: string;
+  embedding: Buffer;
+};
+
+export type StoredEmbeddingMeta = {
+  model: string;
+  provider: string;
+  dim: number;
+};
+
+export type ReindexOutcome = {
+  indexed: number;
+  indexGeneration: number;
+};
+
+function embeddingToBlob(embedding: Float32Array): Buffer {
+  return Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
+}
+
+function blobToEmbedding(blob: Buffer): Float32Array {
+  return new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / 4);
+}
+
+export class VecStore {
+  private db: import("better-sqlite3").Database;
+
+  constructor(dbPath: string) {
+    ensureDirSync(pathDirname(dbPath));
+    const Database = require("better-sqlite3") as typeof import("better-sqlite3");
+    this.db = new Database(dbPath);
+    this.initSchema();
+  }
+
+  private initSchema(): void {
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS meta (
+        key TEXT PRIMARY KEY,
+        value TEXT NOT NULL
+      );
+      CREATE TABLE IF NOT EXISTS memory_chunks (
+        chunk_id TEXT PRIMARY KEY,
+        content TEXT NOT NULL,
+        source TEXT NOT NULL,
+        timestamp TEXT NOT NULL,
+        embedding BLOB NOT NULL
+      );
+    `);
+  }
+
+  getStoredEmbeddingMeta(): StoredEmbeddingMeta | null {
+    const read = (key: string): string | undefined =>
+      this.db.prepare("SELECT value FROM meta WHERE key = ?").pluck().get(key) as string | undefined;
+
+    const model = read("embedding_model");
+    const provider = read("embedding_provider");
+    const dimRaw = read("embedding_dim");
+    if (!model || !provider || !dimRaw) return null;
+
+    const dim = Number.parseInt(dimRaw, 10);
+    if (!Number.isFinite(dim)) return null;
+    return { model, provider, dim };
+  }
+
+  embeddingMetaMatches(embedder: Embedder): boolean {
+    const stored = this.getStoredEmbeddingMeta();
+    if (!stored) return true;
+    return (
+      stored.model === embedder.model &&
+      stored.provider === embedder.provider &&
+      stored.dim === embedder.dim
+    );
+  }
+
+  getIndexGeneration(): number {
+    const raw = this.db.prepare("SELECT value FROM meta WHERE key = 'index_generation'").pluck().get();
+    return Number(raw ?? "0");
+  }
+
+  getChunkCount(): number {
+    const row = this.db.prepare("SELECT COUNT(*) AS count FROM memory_chunks").get() as { count: number };
+    return row.count;
+  }
+
+  private clearChunksIfEmbeddingMismatch(embedder: Embedder): void {
+    if (this.embeddingMetaMatches(embedder)) return;
+    this.db.prepare("DELETE FROM memory_chunks").run();
+  }
+
+  private bumpIndexGeneration(): number {
+    const generation = this.getIndexGeneration() + 1;
+    this.db
+      .prepare("INSERT INTO meta(key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value")
+      .run("index_generation", String(generation));
+    return generation;
+  }
+
+  private writeEmbeddingMeta(embedder: Embedder): void {
+    const upsert = this.db.prepare(
+      "INSERT INTO meta(key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value",
+    );
+    upsert.run("embedding_model", embedder.model);
+    upsert.run("embedding_provider", embedder.provider);
+    upsert.run("embedding_dim", String(embedder.dim));
+  }
+
+  async reindex(documents: IndexDocument[]): Promise<ReindexOutcome> {
+    const embedder = getEmbedder();
+    this.clearChunksIfEmbeddingMismatch(embedder);
+
+    if (documents.length === 0) {
+      const indexGeneration = this.getIndexGeneration();
+      this.writeEmbeddingMeta(embedder);
+      return { indexed: 0, indexGeneration };
+    }
+
+    const embeddings = await embedder.embedBatch(documents.map((doc) => doc.content));
+
+    const sync = this.db.transaction((docs: IndexDocument[], vectors: Float32Array[]) => {
+      const incomingIds = new Set(docs.map((doc) => doc.id));
+      const existing = this.db.prepare("SELECT chunk_id FROM memory_chunks").all() as Array<{ chunk_id: string }>;
+
+      const deleteChunk = this.db.prepare("DELETE FROM memory_chunks WHERE chunk_id = ?");
+      for (const row of existing) {
+        if (!incomingIds.has(row.chunk_id)) {
+          deleteChunk.run(row.chunk_id);
+        }
+      }
+
+      const upsert = this.db.prepare(`
+        INSERT INTO memory_chunks(chunk_id, content, source, timestamp, embedding)
+        VALUES (?, ?, ?, ?, ?)
+        ON CONFLICT(chunk_id) DO UPDATE SET
+          content = excluded.content,
+          source = excluded.source,
+          timestamp = excluded.timestamp,
+          embedding = excluded.embedding
+      `);
+
+      for (let i = 0; i < docs.length; i++) {
+        const doc = docs[i]!;
+        upsert.run(doc.id, doc.content, doc.source, doc.timestamp, embeddingToBlob(vectors[i]!));
+      }
+
+      const indexGeneration = this.bumpIndexGeneration();
+      this.writeEmbeddingMeta(embedder);
+      return { indexed: docs.length, indexGeneration };
+    });
+
+    return sync(documents, embeddings);
+  }
+
+  async query(queryText: string, topK?: number): Promise<MemoryEntry[]> {
+    const retrieval = readRetrievalConfig();
+    const limit = topK ?? retrieval.topK;
+    const embedder = getEmbedder();
+    this.clearChunksIfEmbeddingMismatch(embedder);
+
+    const queryEmbedding = await embedder.embed(queryText);
+
+    const rows = this.db
+      .prepare("SELECT chunk_id, content, source, timestamp, embedding FROM memory_chunks")
+      .all() as ChunkRow[];
+
+    if (rows.length === 0) return [];
+
+    const candidates: ScoredCandidate[] = [];
+    for (const row of rows) {
+      const embedding = blobToEmbedding(row.embedding);
+      if (embedding.length !== queryEmbedding.length) continue;
+      const similarity = cosineSimilarity(queryEmbedding, embedding);
+      if (similarity < retrieval.minRelevance) continue;
+      candidates.push({
+        chunkId: row.chunk_id,
+        content: row.content,
+        source: row.source,
+        timestamp: row.timestamp,
+        distance: 1 - similarity,
+        embedding,
+      });
+    }
+
+    if (candidates.length === 0) return [];
+
+    candidates.sort((a, b) => a.distance - b.distance);
+    const poolSize = Math.min(limit * retrieval.candidatePoolMultiplier, candidates.length);
+    const pool = candidates.slice(0, poolSize);
+    const selected = mmrSelect(queryEmbedding, pool, limit, retrieval.mmrLambda);
+
+    return selected.map((item) => ({
+      content: item.content,
+      source: item.source,
+      timestamp: item.timestamp,
+      relevance: distanceToRelevance(item.distance),
+    }));
+  }
+
+  close(): void {
+    this.db.close();
+  }
+}
+
+const stores = new Map<string, VecStore>();
+
+export function getVecStore(dbPath: string): VecStore {
+  let store = stores.get(dbPath);
+  if (!store) {
+    store = new VecStore(dbPath);
+    stores.set(dbPath, store);
+  }
+  return store;
+}
+
+/** @internal test hook */
+export function resetVecStoreForTests(): void {
+  for (const store of stores.values()) store.close();
+  stores.clear();
+}

package/src/sidecar/sidecarManager.ts

@@ -0,0 +1,134 @@
+// Agent 侧：connect-or-create、spawn lock、execa 生命周期
+import { readFileSync, unlinkSync, writeFileSync } from "node:fs";
+
+import { execa } from "execa";
+
+import { SIDECAR_SPAWN_LOCK_FILE } from "../constants/paths.js";
+import {
+  SIDECAR_FORCE_KILL_DELAY_MS,
+  SIDECAR_SPAWN_LOCK_STALE_MS,
+  SIDECAR_START_TIMEOUT_MS,
+} from "../constants/timing.js";
+import { ensureDirSync, joinPath, pathDirname, pathExists } from "../utils/fs.js";
+import { nowMs } from "../utils/time.js";
+import { ping } from "./client.js";
+import { resolveSidecarEntry } from "./paths.js";
+import { canConnect, waitUntilReady } from "./utils.js";
+
+export { resolveSidecarEntry } from "./paths.js";
+
+const START_TIMEOUT_MS = SIDECAR_START_TIMEOUT_MS;
+
+let instance: SidecarManager | undefined;
+
+export type SidecarOpts = {
+  entry?: string;
+  socketPath: string;
+  dbPath: string;
+};
+
+/** 上层唯一入口：确保 sidecar 在跑（attach 或 spawn） */
+export async function ensureSidecarRunning(opts: SidecarOpts): Promise<void> {
+  const resolved = { ...opts, entry: opts.entry ?? resolveSidecarEntry() };
+  ensureDirSync(pathDirname(resolved.socketPath));
+
+  if (await canConnect(resolved.socketPath)) return;
+
+  if (!acquireSpawnLock(resolved.socketPath)) {
+    await waitUntilReady(() => canConnect(resolved.socketPath), START_TIMEOUT_MS);
+    return;
+  }
+
+  try {
+    if (await canConnect(resolved.socketPath)) return;
+    await getInstance().spawn(resolved);
+  } finally {
+    releaseSpawnLock(resolved.socketPath);
+  }
+}
+
+export async function stopSidecar(): Promise<void> {
+  await getInstance().stop();
+}
+
+function getInstance(): SidecarManager {
+  instance ??= new SidecarManager();
+  return instance;
+}
+
+class SidecarManager {
+  private child?: ReturnType<typeof execa>;
+
+  async spawn(opts: Required<SidecarOpts>): Promise<void> {
+    if (await ping(opts.socketPath)) return;
+
+    this.child = execa(
+      process.execPath,
+      [opts.entry, "--socket", opts.socketPath, "--db", opts.dbPath],
+      {
+        stdio: "ignore",
+        cleanup: true,
+        forceKillAfterDelay: SIDECAR_FORCE_KILL_DELAY_MS,
+      },
+    );
+
+    this.child.catch(() => {});
+
+    await waitUntilReady(() => ping(opts.socketPath), START_TIMEOUT_MS);
+  }
+
+  async stop(): Promise<void> {
+    this.child?.kill("SIGTERM");
+    await this.child;
+    this.child = undefined;
+  }
+}
+
+function spawnLockPath(socketPath: string): string {
+  return joinPath(pathDirname(socketPath), SIDECAR_SPAWN_LOCK_FILE);
+}
+
+function acquireSpawnLock(socketPath: string): boolean {
+  const lockPath = spawnLockPath(socketPath);
+  for (let i = 0; i < 5; i++) {
+    try {
+      writeFileSync(lockPath, `${process.pid}\n${nowMs()}\n`, { flag: "wx" });
+      return true;
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+      if (isLockStale(lockPath)) {
+        try {
+          unlinkSync(lockPath);
+        } catch {}
+        continue;
+      }
+      return false;
+    }
+  }
+  return false;
+}
+
+function isLockStale(lockPath: string): boolean {
+  if (!pathExists(lockPath)) return false;
+  try {
+    const [pidLine = "", tsLine = "0"] = readFileSync(lockPath, "utf8").trim().split("\n");
+    const pid = Number.parseInt(pidLine, 10);
+    const ts = Number.parseInt(tsLine, 10);
+    if (Number.isFinite(pid)) {
+      try {
+        process.kill(pid, 0);
+      } catch {
+        return true;
+      }
+    }
+    return !Number.isFinite(ts) || nowMs() - ts > SIDECAR_SPAWN_LOCK_STALE_MS;
+  } catch {
+    return true;
+  }
+}
+
+function releaseSpawnLock(socketPath: string): void {
+  try {
+    unlinkSync(spawnLockPath(socketPath));
+  } catch {}
+}

package/src/sidecar/utils.ts

@@ -0,0 +1,28 @@
+import net from "node:net";
+
+export function waitUntilReady(predicate: () => Promise<boolean>, timeout: number): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        reject(new Error(`Timeout waiting for predicate after ${timeout}ms`));
+      }, timeout);
+      const interval = setInterval(async () => {
+        if (await predicate()) {
+          clearInterval(interval);
+          clearTimeout(timer);
+          resolve();
+        }
+      }, 100);
+    });
+  }
+
+export function canConnect(socketPath: string): Promise<boolean> {
+  return new Promise((resolve) => {
+    const socket = net.connect(socketPath);
+    socket.on("connect", () => {
+      resolve(true);
+    });
+    socket.on("error", () => {
+      resolve(false);
+    });
+  });
+}