@chendpoc/pi-memory 0.1.13 → 0.2.1

package/src/sidecar/server/vec/store.ts

@@ -0,0 +1,232 @@
+import { createRequire } from "node:module";
+
+import type { Embedder } from "../../../adapters/embed/types.js";
+import type { IndexDocument, MemoryEntry } from "../../protocol.js";
+import { ensureDirSync, pathDirname } from "../../../utils/fs.js";
+import { readRetrievalConfig } from "../../../config/retrieval.js";
+import { getEmbedder } from "./embedder.js";
+import { cosineSimilarity, distanceToRelevance, mmrSelect, type ScoredCandidate } from "./mmr.js";
+
+const require = createRequire(import.meta.url);
+
+type ChunkRow = {
+  chunk_id: string;
+  content: string;
+  source: string;
+  timestamp: string;
+  embedding: Buffer;
+};
+
+export type StoredEmbeddingMeta = {
+  model: string;
+  provider: string;
+  dim: number;
+};
+
+export type ReindexOutcome = {
+  indexed: number;
+  indexGeneration: number;
+};
+
+function embeddingToBlob(embedding: Float32Array): Buffer {
+  return Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
+}
+
+function blobToEmbedding(blob: Buffer): Float32Array {
+  return new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / 4);
+}
+
+export class VecStore {
+  private db: import("better-sqlite3").Database;
+
+  constructor(dbPath: string) {
+    ensureDirSync(pathDirname(dbPath));
+    const Database = require("better-sqlite3") as typeof import("better-sqlite3");
+    this.db = new Database(dbPath);
+    this.initSchema();
+  }
+
+  private initSchema(): void {
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS meta (
+        key TEXT PRIMARY KEY,
+        value TEXT NOT NULL
+      );
+      CREATE TABLE IF NOT EXISTS memory_chunks (
+        chunk_id TEXT PRIMARY KEY,
+        content TEXT NOT NULL,
+        source TEXT NOT NULL,
+        timestamp TEXT NOT NULL,
+        embedding BLOB NOT NULL
+      );
+    `);
+  }
+
+  getStoredEmbeddingMeta(): StoredEmbeddingMeta | null {
+    const read = (key: string): string | undefined =>
+      this.db.prepare("SELECT value FROM meta WHERE key = ?").pluck().get(key) as string | undefined;
+
+    const model = read("embedding_model");
+    const provider = read("embedding_provider");
+    const dimRaw = read("embedding_dim");
+    if (!model || !provider || !dimRaw) return null;
+
+    const dim = Number.parseInt(dimRaw, 10);
+    if (!Number.isFinite(dim)) return null;
+    return { model, provider, dim };
+  }
+
+  embeddingMetaMatches(embedder: Embedder): boolean {
+    const stored = this.getStoredEmbeddingMeta();
+    if (!stored) return true;
+    return (
+      stored.model === embedder.model &&
+      stored.provider === embedder.provider &&
+      stored.dim === embedder.dim
+    );
+  }
+
+  getIndexGeneration(): number {
+    const raw = this.db.prepare("SELECT value FROM meta WHERE key = 'index_generation'").pluck().get();
+    return Number(raw ?? "0");
+  }
+
+  getChunkCount(): number {
+    const row = this.db.prepare("SELECT COUNT(*) AS count FROM memory_chunks").get() as { count: number };
+    return row.count;
+  }
+
+  private clearChunksIfEmbeddingMismatch(embedder: Embedder): void {
+    if (this.embeddingMetaMatches(embedder)) return;
+    this.db.prepare("DELETE FROM memory_chunks").run();
+  }
+
+  private bumpIndexGeneration(): number {
+    const generation = this.getIndexGeneration() + 1;
+    this.db
+      .prepare("INSERT INTO meta(key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value")
+      .run("index_generation", String(generation));
+    return generation;
+  }
+
+  private writeEmbeddingMeta(embedder: Embedder): void {
+    const upsert = this.db.prepare(
+      "INSERT INTO meta(key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value",
+    );
+    upsert.run("embedding_model", embedder.model);
+    upsert.run("embedding_provider", embedder.provider);
+    upsert.run("embedding_dim", String(embedder.dim));
+  }
+
+  async reindex(documents: IndexDocument[]): Promise<ReindexOutcome> {
+    const embedder = getEmbedder();
+    this.clearChunksIfEmbeddingMismatch(embedder);
+
+    if (documents.length === 0) {
+      const indexGeneration = this.getIndexGeneration();
+      this.writeEmbeddingMeta(embedder);
+      return { indexed: 0, indexGeneration };
+    }
+
+    const embeddings = await embedder.embedBatch(documents.map((doc) => doc.content));
+
+    const sync = this.db.transaction((docs: IndexDocument[], vectors: Float32Array[]) => {
+      const incomingIds = new Set(docs.map((doc) => doc.id));
+      const existing = this.db.prepare("SELECT chunk_id FROM memory_chunks").all() as Array<{ chunk_id: string }>;
+
+      const deleteChunk = this.db.prepare("DELETE FROM memory_chunks WHERE chunk_id = ?");
+      for (const row of existing) {
+        if (!incomingIds.has(row.chunk_id)) {
+          deleteChunk.run(row.chunk_id);
+        }
+      }
+
+      const upsert = this.db.prepare(`
+        INSERT INTO memory_chunks(chunk_id, content, source, timestamp, embedding)
+        VALUES (?, ?, ?, ?, ?)
+        ON CONFLICT(chunk_id) DO UPDATE SET
+          content = excluded.content,
+          source = excluded.source,
+          timestamp = excluded.timestamp,
+          embedding = excluded.embedding
+      `);
+
+      for (let i = 0; i < docs.length; i++) {
+        const doc = docs[i]!;
+        upsert.run(doc.id, doc.content, doc.source, doc.timestamp, embeddingToBlob(vectors[i]!));
+      }
+
+      const indexGeneration = this.bumpIndexGeneration();
+      this.writeEmbeddingMeta(embedder);
+      return { indexed: docs.length, indexGeneration };
+    });
+
+    return sync(documents, embeddings);
+  }
+
+  async query(queryText: string, topK?: number): Promise<MemoryEntry[]> {
+    const retrieval = readRetrievalConfig();
+    const limit = topK ?? retrieval.topK;
+    const embedder = getEmbedder();
+    this.clearChunksIfEmbeddingMismatch(embedder);
+
+    const queryEmbedding = await embedder.embed(queryText);
+
+    const rows = this.db
+      .prepare("SELECT chunk_id, content, source, timestamp, embedding FROM memory_chunks")
+      .all() as ChunkRow[];
+
+    if (rows.length === 0) return [];
+
+    const candidates: ScoredCandidate[] = [];
+    for (const row of rows) {
+      const embedding = blobToEmbedding(row.embedding);
+      if (embedding.length !== queryEmbedding.length) continue;
+      const similarity = cosineSimilarity(queryEmbedding, embedding);
+      if (similarity < retrieval.minRelevance) continue;
+      candidates.push({
+        chunkId: row.chunk_id,
+        content: row.content,
+        source: row.source,
+        timestamp: row.timestamp,
+        distance: 1 - similarity,
+        embedding,
+      });
+    }
+
+    if (candidates.length === 0) return [];
+
+    candidates.sort((a, b) => a.distance - b.distance);
+    const poolSize = Math.min(limit * retrieval.candidatePoolMultiplier, candidates.length);
+    const pool = candidates.slice(0, poolSize);
+    const selected = mmrSelect(queryEmbedding, pool, limit, retrieval.mmrLambda);
+
+    return selected.map((item) => ({
+      content: item.content,
+      source: item.source,
+      timestamp: item.timestamp,
+      relevance: distanceToRelevance(item.distance),
+    }));
+  }
+
+  close(): void {
+    this.db.close();
+  }
+}
+
+const stores = new Map<string, VecStore>();
+
+export function getVecStore(dbPath: string): VecStore {
+  let store = stores.get(dbPath);
+  if (!store) {
+    store = new VecStore(dbPath);
+    stores.set(dbPath, store);
+  }
+  return store;
+}
+
+/** @internal test hook */
+export function resetVecStoreForTests(): void {
+  for (const store of stores.values()) store.close();
+  stores.clear();
+}

package/src/sidecar/sidecarManager.ts

@@ -0,0 +1,134 @@
+// Agent 侧：connect-or-create、spawn lock、execa 生命周期
+import { readFileSync, unlinkSync, writeFileSync } from "node:fs";
+
+import { execa } from "execa";
+
+import { SIDECAR_SPAWN_LOCK_FILE } from "../constants/paths.js";
+import {
+  SIDECAR_FORCE_KILL_DELAY_MS,
+  SIDECAR_SPAWN_LOCK_STALE_MS,
+  SIDECAR_START_TIMEOUT_MS,
+} from "../constants/timing.js";
+import { ensureDirSync, joinPath, pathDirname, pathExists } from "../utils/fs.js";
+import { nowMs } from "../utils/time.js";
+import { ping } from "./client.js";
+import { resolveSidecarEntry } from "./paths.js";
+import { canConnect, waitUntilReady } from "./utils.js";
+
+export { resolveSidecarEntry } from "./paths.js";
+
+const START_TIMEOUT_MS = SIDECAR_START_TIMEOUT_MS;
+
+let instance: SidecarManager | undefined;
+
+export type SidecarOpts = {
+  entry?: string;
+  socketPath: string;
+  dbPath: string;
+};
+
+/** 上层唯一入口：确保 sidecar 在跑（attach 或 spawn） */
+export async function ensureSidecarRunning(opts: SidecarOpts): Promise<void> {
+  const resolved = { ...opts, entry: opts.entry ?? resolveSidecarEntry() };
+  ensureDirSync(pathDirname(resolved.socketPath));
+
+  if (await canConnect(resolved.socketPath)) return;
+
+  if (!acquireSpawnLock(resolved.socketPath)) {
+    await waitUntilReady(() => canConnect(resolved.socketPath), START_TIMEOUT_MS);
+    return;
+  }
+
+  try {
+    if (await canConnect(resolved.socketPath)) return;
+    await getInstance().spawn(resolved);
+  } finally {
+    releaseSpawnLock(resolved.socketPath);
+  }
+}
+
+export async function stopSidecar(): Promise<void> {
+  await getInstance().stop();
+}
+
+function getInstance(): SidecarManager {
+  instance ??= new SidecarManager();
+  return instance;
+}
+
+class SidecarManager {
+  private child?: ReturnType<typeof execa>;
+
+  async spawn(opts: Required<SidecarOpts>): Promise<void> {
+    if (await ping(opts.socketPath)) return;
+
+    this.child = execa(
+      process.execPath,
+      [opts.entry, "--socket", opts.socketPath, "--db", opts.dbPath],
+      {
+        stdio: "ignore",
+        cleanup: true,
+        forceKillAfterDelay: SIDECAR_FORCE_KILL_DELAY_MS,
+      },
+    );
+
+    this.child.catch(() => {});
+
+    await waitUntilReady(() => ping(opts.socketPath), START_TIMEOUT_MS);
+  }
+
+  async stop(): Promise<void> {
+    this.child?.kill("SIGTERM");
+    await this.child;
+    this.child = undefined;
+  }
+}
+
+function spawnLockPath(socketPath: string): string {
+  return joinPath(pathDirname(socketPath), SIDECAR_SPAWN_LOCK_FILE);
+}
+
+function acquireSpawnLock(socketPath: string): boolean {
+  const lockPath = spawnLockPath(socketPath);
+  for (let i = 0; i < 5; i++) {
+    try {
+      writeFileSync(lockPath, `${process.pid}\n${nowMs()}\n`, { flag: "wx" });
+      return true;
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+      if (isLockStale(lockPath)) {
+        try {
+          unlinkSync(lockPath);
+        } catch {}
+        continue;
+      }
+      return false;
+    }
+  }
+  return false;
+}
+
+function isLockStale(lockPath: string): boolean {
+  if (!pathExists(lockPath)) return false;
+  try {
+    const [pidLine = "", tsLine = "0"] = readFileSync(lockPath, "utf8").trim().split("\n");
+    const pid = Number.parseInt(pidLine, 10);
+    const ts = Number.parseInt(tsLine, 10);
+    if (Number.isFinite(pid)) {
+      try {
+        process.kill(pid, 0);
+      } catch {
+        return true;
+      }
+    }
+    return !Number.isFinite(ts) || nowMs() - ts > SIDECAR_SPAWN_LOCK_STALE_MS;
+  } catch {
+    return true;
+  }
+}
+
+function releaseSpawnLock(socketPath: string): void {
+  try {
+    unlinkSync(spawnLockPath(socketPath));
+  } catch {}
+}

package/src/sidecar/utils.ts

@@ -0,0 +1,28 @@
+import net from "node:net";
+
+export function waitUntilReady(predicate: () => Promise<boolean>, timeout: number): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        reject(new Error(`Timeout waiting for predicate after ${timeout}ms`));
+      }, timeout);
+      const interval = setInterval(async () => {
+        if (await predicate()) {
+          clearInterval(interval);
+          clearTimeout(timer);
+          resolve();
+        }
+      }, 100);
+    });
+  }
+
+export function canConnect(socketPath: string): Promise<boolean> {
+  return new Promise((resolve) => {
+    const socket = net.connect(socketPath);
+    socket.on("connect", () => {
+      resolve(true);
+    });
+    socket.on("error", () => {
+      resolve(false);
+    });
+  });
+}

package/src/sidecar/warmup.ts

@@ -0,0 +1,50 @@
+import { fetchIndexStats, ping, query } from "./client.js";
+import { SIDECAR_WARMUP_QUERY_TIMEOUT_MS } from "../constants/timing.js";
+import { debugMemory } from "../utils/debugLog.js";
+import { nowMs } from "../utils/time.js";
+
+export type WarmSidecarOptions = {
+  queryTimeoutMs?: number;
+};
+
+/** Pre-open sidecar DB and optionally smoke-test embed + vec paths. Fail-silent. */
+export async function warmSidecar(
+  socketPath: string,
+  options: WarmSidecarOptions = {},
+): Promise<void> {
+  const startedAt = nowMs();
+
+  if (!(await ping(socketPath))) {
+    debugMemory("sidecar", "warm", { ok: false, reason: "ping" });
+    return;
+  }
+
+  const statsStartedAt = nowMs();
+  const statsResult = await fetchIndexStats(socketPath);
+  const statsMs = nowMs() - statsStartedAt;
+
+  if ("error" in statsResult) {
+    debugMemory("sidecar", "warm", { ok: false, stats_ms: statsMs, reason: "stats" });
+    return;
+  }
+
+  let queryMs = 0;
+  if (statsResult.stats.chunk_count > 0) {
+    const queryStartedAt = nowMs();
+    const queryTimeoutMs = options.queryTimeoutMs ?? SIDECAR_WARMUP_QUERY_TIMEOUT_MS;
+    try {
+      await query(socketPath, ".", queryTimeoutMs);
+    } catch {
+      // warm query is best-effort
+    }
+    queryMs = nowMs() - queryStartedAt;
+  }
+
+  debugMemory("sidecar", "warm", {
+    ok: true,
+    stats_ms: statsMs,
+    query_ms: queryMs,
+    chunks: statsResult.stats.chunk_count,
+    total_ms: nowMs() - startedAt,
+  });
+}

package/src/store/backend.ts

@@ -0,0 +1,66 @@
+import lockfile from "proper-lockfile";
+
+import {
+  MEMORY_LOCK_MAX_TIMEOUT_MS,
+  MEMORY_LOCK_MIN_TIMEOUT_MS,
+  MEMORY_LOCK_RETRIES,
+} from "../constants/timing.js";
+import {
+  ensureDir,
+  ensureFile,
+  joinPath,
+  listDir,
+  pathDirname,
+  readText,
+  removeFile,
+  writeText,
+} from "../utils/fs.js";
+
+import { isAutoOverflowFile } from "./paths.js";
+
+export class MarkdownMemoryBackend {
+  constructor(private readonly memoryFile: string) {}
+
+  async ensureAgentDir(): Promise<void> {
+    await ensureDir(pathDirname(this.memoryFile));
+  }
+
+  async readText(path: string): Promise<string> {
+    return readText(path);
+  }
+
+  async writeText(path: string, content: string): Promise<void> {
+    await writeText(path, content);
+  }
+
+  async listAutoFiles(agentDir: string): Promise<string[]> {
+    const names = await listDir(agentDir);
+    return names.filter(isAutoOverflowFile).sort();
+  }
+
+  autoFilePath(agentDir: string, fileName: string): string {
+    return joinPath(agentDir, fileName);
+  }
+
+  async deleteAutoFile(path: string): Promise<void> {
+    await removeFile(path);
+  }
+
+  async withMemoryLock<T>(fn: () => Promise<T>): Promise<T> {
+    await this.ensureAgentDir();
+    await ensureFile(this.memoryFile);
+
+    const release = await lockfile.lock(this.memoryFile, {
+      retries: {
+        retries: MEMORY_LOCK_RETRIES,
+        minTimeout: MEMORY_LOCK_MIN_TIMEOUT_MS,
+        maxTimeout: MEMORY_LOCK_MAX_TIMEOUT_MS,
+      },
+    });
+    try {
+      return await fn();
+    } finally {
+      await release();
+    }
+  }
+}

package/src/store/index.ts

@@ -0,0 +1,5 @@
+export { MemoryStore, createMemoryStore } from "./memoryStore.js";
+export { MarkdownMemoryBackend } from "./backend.js";
+export { getAgentPaths, resolveAgentDir } from "./paths.js";
+export * from "./types.js";
+export * from "../constants/memory.js";

package/src/store/indexChunks.ts

@@ -0,0 +1,89 @@
+import type { IndexDocument } from "../sidecar/protocol.js";
+import type { ChunkingConfig } from "../config/chunking.js";
+import { CHUNKING_DISABLED_MAX_CHARS } from "../constants/chunking.js";
+import { pathBasename } from "../utils/fs.js";
+import type { MemorySection, ParsedEntry } from "./types.js";
+
+function formatChunkContent(section: MemorySection, body: string): string {
+  return `[${section}] ${body.trim()}`;
+}
+
+/**
+ * Split long text on paragraph / line / sentence boundaries, then hard-wrap at maxChars.
+ */
+export function splitTextByMaxChars(text: string, maxChars: number): string[] {
+  const trimmed = text.trim();
+  if (!trimmed) return [];
+  if (maxChars <= 0 || trimmed.length <= maxChars) return [trimmed];
+
+  const parts: string[] = [];
+  let remaining = trimmed;
+
+  while (remaining.length > maxChars) {
+    let cut = maxChars;
+    const minCut = Math.floor(maxChars * 0.35);
+
+    const paragraph = remaining.lastIndexOf("\n\n", cut);
+    if (paragraph >= minCut) {
+      cut = paragraph;
+    } else {
+      const line = remaining.lastIndexOf("\n", cut);
+      if (line >= minCut) {
+        cut = line;
+      } else {
+        const sentence = remaining.lastIndexOf(". ", cut);
+        if (sentence >= minCut) {
+          cut = sentence + 1;
+        } else {
+          const space = remaining.lastIndexOf(" ", cut);
+          if (space >= minCut) {
+            cut = space;
+          }
+        }
+      }
+    }
+
+    const part = remaining.slice(0, cut).trim();
+    if (part) parts.push(part);
+    remaining = remaining.slice(cut).trim();
+    if (!part && remaining.length > 0) {
+      parts.push(remaining.slice(0, maxChars).trim());
+      remaining = remaining.slice(maxChars).trim();
+    }
+  }
+
+  if (remaining) parts.push(remaining);
+  return parts.length > 0 ? parts : [trimmed];
+}
+
+export function buildIndexDocuments(
+  entries: ParsedEntry[],
+  chunking: ChunkingConfig,
+): IndexDocument[] {
+  const docs: IndexDocument[] = [];
+  const maxChars =
+    chunking.maxChars <= CHUNKING_DISABLED_MAX_CHARS
+      ? CHUNKING_DISABLED_MAX_CHARS
+      : chunking.maxChars;
+
+  for (const entry of entries) {
+    const bodies =
+      maxChars <= CHUNKING_DISABLED_MAX_CHARS
+        ? [entry.content.trim()].filter(Boolean)
+        : splitTextByMaxChars(entry.content, maxChars);
+
+    if (bodies.length === 0) continue;
+
+    bodies.forEach((body, index) => {
+      const id = bodies.length === 1 ? entry.id : `${entry.id}#${index}`;
+      docs.push({
+        id,
+        content: formatChunkContent(entry.section, body),
+        source: pathBasename(entry.sourceFile),
+        timestamp: entry.timestamp,
+      });
+    });
+  }
+
+  return docs;
+}

package/src/store/markdown/format.ts

@@ -0,0 +1,16 @@
+import type { MemorySection, StoreMemoryEntry } from "../types.js";
+
+export function formatEntryLine(entry: StoreMemoryEntry): string {
+  const prefix = entry.userAuthored ? "[user] " : "";
+  const meta = `<!-- id:${entry.id}${entry.userAuthored ? " user" : ""} ts:${entry.timestamp} -->`;
+  return `- ${prefix}${entry.content.trim()} ${meta}`;
+}
+
+export function formatSectionHeader(section: MemorySection): string {
+  return `## ${section}`;
+}
+
+export function countLines(content: string): number {
+  if (!content) return 0;
+  return content.split("\n").length;
+}

package/src/store/markdown/parse.ts

@@ -0,0 +1,63 @@
+import { pathBasename } from "../../utils/fs.js";
+import { epochTimestamp } from "../../utils/time.js";
+
+import { OVERFLOW_POINTER_RE } from "../../constants/memory.js";
+import { MEMORY_SECTIONS, type MemorySection, type ParsedEntry } from "../types.js";
+
+const SECTION_RE = /^##\s+(Preferences|Conventions|Findings|Todos)\s*$/;
+const ENTRY_META_RE = /<!--\s*id:([^\s]+)(?:\s+user)?(?:\s+ts:([^\s]+))?\s*-->/;
+const USER_PREFIX_RE = /^\[user\]\s+/;
+
+export function parseMemoryMarkdown(content: string, sourceFile: string): ParsedEntry[] {
+  const entries: ParsedEntry[] = [];
+  let section: MemorySection | undefined;
+  const lines = content.split("\n");
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i]!;
+    const sectionMatch = line.match(SECTION_RE);
+    if (sectionMatch) {
+      section = sectionMatch[1] as MemorySection;
+      continue;
+    }
+
+    if (!line.startsWith("- ") || !section) continue;
+
+    const pointerMatch = line.match(OVERFLOW_POINTER_RE);
+    if (pointerMatch) continue;
+
+    const metaMatch = line.match(ENTRY_META_RE);
+    const id = metaMatch?.[1] ?? `${pathBasename(sourceFile)}:${i + 1}`;
+    const timestamp = metaMatch?.[2] ?? epochTimestamp();
+    let body = line.slice(2).replace(ENTRY_META_RE, "").trim();
+    const userAuthored = USER_PREFIX_RE.test(body);
+    if (userAuthored) body = body.replace(USER_PREFIX_RE, "").trim();
+
+    if (!body) continue;
+
+    entries.push({
+      id,
+      section,
+      content: body,
+      userAuthored: userAuthored || undefined,
+      timestamp,
+      sourceFile,
+      line: i + 1,
+    });
+  }
+
+  return entries;
+}
+
+export function listOverflowPointers(content: string): string[] {
+  const files: string[] = [];
+  for (const line of content.split("\n")) {
+    const match = line.match(OVERFLOW_POINTER_RE);
+    if (match?.[1]) files.push(match[1]);
+  }
+  return files;
+}
+
+export function isKnownSection(name: string): name is MemorySection {
+  return (MEMORY_SECTIONS as readonly string[]).includes(name);
+}