@chendpoc/pi-memory 0.2.1 → 0.2.3

package/src/sidecar/server/vec/store.ts

@@ -1,232 +0,0 @@
-import { createRequire } from "node:module";
-
-import type { Embedder } from "../../../adapters/embed/types.js";
-import type { IndexDocument, MemoryEntry } from "../../protocol.js";
-import { ensureDirSync, pathDirname } from "../../../utils/fs.js";
-import { readRetrievalConfig } from "../../../config/retrieval.js";
-import { getEmbedder } from "./embedder.js";
-import { cosineSimilarity, distanceToRelevance, mmrSelect, type ScoredCandidate } from "./mmr.js";
-
-const require = createRequire(import.meta.url);
-
-type ChunkRow = {
-  chunk_id: string;
-  content: string;
-  source: string;
-  timestamp: string;
-  embedding: Buffer;
-};
-
-export type StoredEmbeddingMeta = {
-  model: string;
-  provider: string;
-  dim: number;
-};
-
-export type ReindexOutcome = {
-  indexed: number;
-  indexGeneration: number;
-};
-
-function embeddingToBlob(embedding: Float32Array): Buffer {
-  return Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
-}
-
-function blobToEmbedding(blob: Buffer): Float32Array {
-  return new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / 4);
-}
-
-export class VecStore {
-  private db: import("better-sqlite3").Database;
-
-  constructor(dbPath: string) {
-    ensureDirSync(pathDirname(dbPath));
-    const Database = require("better-sqlite3") as typeof import("better-sqlite3");
-    this.db = new Database(dbPath);
-    this.initSchema();
-  }
-
-  private initSchema(): void {
-    this.db.exec(`
-      CREATE TABLE IF NOT EXISTS meta (
-        key TEXT PRIMARY KEY,
-        value TEXT NOT NULL
-      );
-      CREATE TABLE IF NOT EXISTS memory_chunks (
-        chunk_id TEXT PRIMARY KEY,
-        content TEXT NOT NULL,
-        source TEXT NOT NULL,
-        timestamp TEXT NOT NULL,
-        embedding BLOB NOT NULL
-      );
-    `);
-  }
-
-  getStoredEmbeddingMeta(): StoredEmbeddingMeta | null {
-    const read = (key: string): string | undefined =>
-      this.db.prepare("SELECT value FROM meta WHERE key = ?").pluck().get(key) as string | undefined;
-
-    const model = read("embedding_model");
-    const provider = read("embedding_provider");
-    const dimRaw = read("embedding_dim");
-    if (!model || !provider || !dimRaw) return null;
-
-    const dim = Number.parseInt(dimRaw, 10);
-    if (!Number.isFinite(dim)) return null;
-    return { model, provider, dim };
-  }
-
-  embeddingMetaMatches(embedder: Embedder): boolean {
-    const stored = this.getStoredEmbeddingMeta();
-    if (!stored) return true;
-    return (
-      stored.model === embedder.model &&
-      stored.provider === embedder.provider &&
-      stored.dim === embedder.dim
-    );
-  }
-
-  getIndexGeneration(): number {
-    const raw = this.db.prepare("SELECT value FROM meta WHERE key = 'index_generation'").pluck().get();
-    return Number(raw ?? "0");
-  }
-
-  getChunkCount(): number {
-    const row = this.db.prepare("SELECT COUNT(*) AS count FROM memory_chunks").get() as { count: number };
-    return row.count;
-  }
-
-  private clearChunksIfEmbeddingMismatch(embedder: Embedder): void {
-    if (this.embeddingMetaMatches(embedder)) return;
-    this.db.prepare("DELETE FROM memory_chunks").run();
-  }
-
-  private bumpIndexGeneration(): number {
-    const generation = this.getIndexGeneration() + 1;
-    this.db
-      .prepare("INSERT INTO meta(key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value")
-      .run("index_generation", String(generation));
-    return generation;
-  }
-
-  private writeEmbeddingMeta(embedder: Embedder): void {
-    const upsert = this.db.prepare(
-      "INSERT INTO meta(key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value",
-    );
-    upsert.run("embedding_model", embedder.model);
-    upsert.run("embedding_provider", embedder.provider);
-    upsert.run("embedding_dim", String(embedder.dim));
-  }
-
-  async reindex(documents: IndexDocument[]): Promise<ReindexOutcome> {
-    const embedder = getEmbedder();
-    this.clearChunksIfEmbeddingMismatch(embedder);
-
-    if (documents.length === 0) {
-      const indexGeneration = this.getIndexGeneration();
-      this.writeEmbeddingMeta(embedder);
-      return { indexed: 0, indexGeneration };
-    }
-
-    const embeddings = await embedder.embedBatch(documents.map((doc) => doc.content));
-
-    const sync = this.db.transaction((docs: IndexDocument[], vectors: Float32Array[]) => {
-      const incomingIds = new Set(docs.map((doc) => doc.id));
-      const existing = this.db.prepare("SELECT chunk_id FROM memory_chunks").all() as Array<{ chunk_id: string }>;
-
-      const deleteChunk = this.db.prepare("DELETE FROM memory_chunks WHERE chunk_id = ?");
-      for (const row of existing) {
-        if (!incomingIds.has(row.chunk_id)) {
-          deleteChunk.run(row.chunk_id);
-        }
-      }
-
-      const upsert = this.db.prepare(`
-        INSERT INTO memory_chunks(chunk_id, content, source, timestamp, embedding)
-        VALUES (?, ?, ?, ?, ?)
-        ON CONFLICT(chunk_id) DO UPDATE SET
-          content = excluded.content,
-          source = excluded.source,
-          timestamp = excluded.timestamp,
-          embedding = excluded.embedding
-      `);
-
-      for (let i = 0; i < docs.length; i++) {
-        const doc = docs[i]!;
-        upsert.run(doc.id, doc.content, doc.source, doc.timestamp, embeddingToBlob(vectors[i]!));
-      }
-
-      const indexGeneration = this.bumpIndexGeneration();
-      this.writeEmbeddingMeta(embedder);
-      return { indexed: docs.length, indexGeneration };
-    });
-
-    return sync(documents, embeddings);
-  }
-
-  async query(queryText: string, topK?: number): Promise<MemoryEntry[]> {
-    const retrieval = readRetrievalConfig();
-    const limit = topK ?? retrieval.topK;
-    const embedder = getEmbedder();
-    this.clearChunksIfEmbeddingMismatch(embedder);
-
-    const queryEmbedding = await embedder.embed(queryText);
-
-    const rows = this.db
-      .prepare("SELECT chunk_id, content, source, timestamp, embedding FROM memory_chunks")
-      .all() as ChunkRow[];
-
-    if (rows.length === 0) return [];
-
-    const candidates: ScoredCandidate[] = [];
-    for (const row of rows) {
-      const embedding = blobToEmbedding(row.embedding);
-      if (embedding.length !== queryEmbedding.length) continue;
-      const similarity = cosineSimilarity(queryEmbedding, embedding);
-      if (similarity < retrieval.minRelevance) continue;
-      candidates.push({
-        chunkId: row.chunk_id,
-        content: row.content,
-        source: row.source,
-        timestamp: row.timestamp,
-        distance: 1 - similarity,
-        embedding,
-      });
-    }
-
-    if (candidates.length === 0) return [];
-
-    candidates.sort((a, b) => a.distance - b.distance);
-    const poolSize = Math.min(limit * retrieval.candidatePoolMultiplier, candidates.length);
-    const pool = candidates.slice(0, poolSize);
-    const selected = mmrSelect(queryEmbedding, pool, limit, retrieval.mmrLambda);
-
-    return selected.map((item) => ({
-      content: item.content,
-      source: item.source,
-      timestamp: item.timestamp,
-      relevance: distanceToRelevance(item.distance),
-    }));
-  }
-
-  close(): void {
-    this.db.close();
-  }
-}
-
-const stores = new Map<string, VecStore>();
-
-export function getVecStore(dbPath: string): VecStore {
-  let store = stores.get(dbPath);
-  if (!store) {
-    store = new VecStore(dbPath);
-    stores.set(dbPath, store);
-  }
-  return store;
-}
-
-/** @internal test hook */
-export function resetVecStoreForTests(): void {
-  for (const store of stores.values()) store.close();
-  stores.clear();
-}

package/src/sidecar/sidecarManager.ts

@@ -1,134 +0,0 @@
-// Agent 侧：connect-or-create、spawn lock、execa 生命周期
-import { readFileSync, unlinkSync, writeFileSync } from "node:fs";
-
-import { execa } from "execa";
-
-import { SIDECAR_SPAWN_LOCK_FILE } from "../constants/paths.js";
-import {
-  SIDECAR_FORCE_KILL_DELAY_MS,
-  SIDECAR_SPAWN_LOCK_STALE_MS,
-  SIDECAR_START_TIMEOUT_MS,
-} from "../constants/timing.js";
-import { ensureDirSync, joinPath, pathDirname, pathExists } from "../utils/fs.js";
-import { nowMs } from "../utils/time.js";
-import { ping } from "./client.js";
-import { resolveSidecarEntry } from "./paths.js";
-import { canConnect, waitUntilReady } from "./utils.js";
-
-export { resolveSidecarEntry } from "./paths.js";
-
-const START_TIMEOUT_MS = SIDECAR_START_TIMEOUT_MS;
-
-let instance: SidecarManager | undefined;
-
-export type SidecarOpts = {
-  entry?: string;
-  socketPath: string;
-  dbPath: string;
-};
-
-/** 上层唯一入口：确保 sidecar 在跑（attach 或 spawn） */
-export async function ensureSidecarRunning(opts: SidecarOpts): Promise<void> {
-  const resolved = { ...opts, entry: opts.entry ?? resolveSidecarEntry() };
-  ensureDirSync(pathDirname(resolved.socketPath));
-
-  if (await canConnect(resolved.socketPath)) return;
-
-  if (!acquireSpawnLock(resolved.socketPath)) {
-    await waitUntilReady(() => canConnect(resolved.socketPath), START_TIMEOUT_MS);
-    return;
-  }
-
-  try {
-    if (await canConnect(resolved.socketPath)) return;
-    await getInstance().spawn(resolved);
-  } finally {
-    releaseSpawnLock(resolved.socketPath);
-  }
-}
-
-export async function stopSidecar(): Promise<void> {
-  await getInstance().stop();
-}
-
-function getInstance(): SidecarManager {
-  instance ??= new SidecarManager();
-  return instance;
-}
-
-class SidecarManager {
-  private child?: ReturnType<typeof execa>;
-
-  async spawn(opts: Required<SidecarOpts>): Promise<void> {
-    if (await ping(opts.socketPath)) return;
-
-    this.child = execa(
-      process.execPath,
-      [opts.entry, "--socket", opts.socketPath, "--db", opts.dbPath],
-      {
-        stdio: "ignore",
-        cleanup: true,
-        forceKillAfterDelay: SIDECAR_FORCE_KILL_DELAY_MS,
-      },
-    );
-
-    this.child.catch(() => {});
-
-    await waitUntilReady(() => ping(opts.socketPath), START_TIMEOUT_MS);
-  }
-
-  async stop(): Promise<void> {
-    this.child?.kill("SIGTERM");
-    await this.child;
-    this.child = undefined;
-  }
-}
-
-function spawnLockPath(socketPath: string): string {
-  return joinPath(pathDirname(socketPath), SIDECAR_SPAWN_LOCK_FILE);
-}
-
-function acquireSpawnLock(socketPath: string): boolean {
-  const lockPath = spawnLockPath(socketPath);
-  for (let i = 0; i < 5; i++) {
-    try {
-      writeFileSync(lockPath, `${process.pid}\n${nowMs()}\n`, { flag: "wx" });
-      return true;
-    } catch (error) {
-      if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
-      if (isLockStale(lockPath)) {
-        try {
-          unlinkSync(lockPath);
-        } catch {}
-        continue;
-      }
-      return false;
-    }
-  }
-  return false;
-}
-
-function isLockStale(lockPath: string): boolean {
-  if (!pathExists(lockPath)) return false;
-  try {
-    const [pidLine = "", tsLine = "0"] = readFileSync(lockPath, "utf8").trim().split("\n");
-    const pid = Number.parseInt(pidLine, 10);
-    const ts = Number.parseInt(tsLine, 10);
-    if (Number.isFinite(pid)) {
-      try {
-        process.kill(pid, 0);
-      } catch {
-        return true;
-      }
-    }
-    return !Number.isFinite(ts) || nowMs() - ts > SIDECAR_SPAWN_LOCK_STALE_MS;
-  } catch {
-    return true;
-  }
-}
-
-function releaseSpawnLock(socketPath: string): void {
-  try {
-    unlinkSync(spawnLockPath(socketPath));
-  } catch {}
-}

package/src/sidecar/utils.ts

@@ -1,28 +0,0 @@
-import net from "node:net";
-
-export function waitUntilReady(predicate: () => Promise<boolean>, timeout: number): Promise<void> {
-    return new Promise((resolve, reject) => {
-      const timer = setTimeout(() => {
-        reject(new Error(`Timeout waiting for predicate after ${timeout}ms`));
-      }, timeout);
-      const interval = setInterval(async () => {
-        if (await predicate()) {
-          clearInterval(interval);
-          clearTimeout(timer);
-          resolve();
-        }
-      }, 100);
-    });
-  }
-
-export function canConnect(socketPath: string): Promise<boolean> {
-  return new Promise((resolve) => {
-    const socket = net.connect(socketPath);
-    socket.on("connect", () => {
-      resolve(true);
-    });
-    socket.on("error", () => {
-      resolve(false);
-    });
-  });
-}

package/src/sidecar/warmup.ts

@@ -1,50 +0,0 @@
-import { fetchIndexStats, ping, query } from "./client.js";
-import { SIDECAR_WARMUP_QUERY_TIMEOUT_MS } from "../constants/timing.js";
-import { debugMemory } from "../utils/debugLog.js";
-import { nowMs } from "../utils/time.js";
-
-export type WarmSidecarOptions = {
-  queryTimeoutMs?: number;
-};
-
-/** Pre-open sidecar DB and optionally smoke-test embed + vec paths. Fail-silent. */
-export async function warmSidecar(
-  socketPath: string,
-  options: WarmSidecarOptions = {},
-): Promise<void> {
-  const startedAt = nowMs();
-
-  if (!(await ping(socketPath))) {
-    debugMemory("sidecar", "warm", { ok: false, reason: "ping" });
-    return;
-  }
-
-  const statsStartedAt = nowMs();
-  const statsResult = await fetchIndexStats(socketPath);
-  const statsMs = nowMs() - statsStartedAt;
-
-  if ("error" in statsResult) {
-    debugMemory("sidecar", "warm", { ok: false, stats_ms: statsMs, reason: "stats" });
-    return;
-  }
-
-  let queryMs = 0;
-  if (statsResult.stats.chunk_count > 0) {
-    const queryStartedAt = nowMs();
-    const queryTimeoutMs = options.queryTimeoutMs ?? SIDECAR_WARMUP_QUERY_TIMEOUT_MS;
-    try {
-      await query(socketPath, ".", queryTimeoutMs);
-    } catch {
-      // warm query is best-effort
-    }
-    queryMs = nowMs() - queryStartedAt;
-  }
-
-  debugMemory("sidecar", "warm", {
-    ok: true,
-    stats_ms: statsMs,
-    query_ms: queryMs,
-    chunks: statsResult.stats.chunk_count,
-    total_ms: nowMs() - startedAt,
-  });
-}

package/src/store/backend.ts

@@ -1,66 +0,0 @@
-import lockfile from "proper-lockfile";
-
-import {
-  MEMORY_LOCK_MAX_TIMEOUT_MS,
-  MEMORY_LOCK_MIN_TIMEOUT_MS,
-  MEMORY_LOCK_RETRIES,
-} from "../constants/timing.js";
-import {
-  ensureDir,
-  ensureFile,
-  joinPath,
-  listDir,
-  pathDirname,
-  readText,
-  removeFile,
-  writeText,
-} from "../utils/fs.js";
-
-import { isAutoOverflowFile } from "./paths.js";
-
-export class MarkdownMemoryBackend {
-  constructor(private readonly memoryFile: string) {}
-
-  async ensureAgentDir(): Promise<void> {
-    await ensureDir(pathDirname(this.memoryFile));
-  }
-
-  async readText(path: string): Promise<string> {
-    return readText(path);
-  }
-
-  async writeText(path: string, content: string): Promise<void> {
-    await writeText(path, content);
-  }
-
-  async listAutoFiles(agentDir: string): Promise<string[]> {
-    const names = await listDir(agentDir);
-    return names.filter(isAutoOverflowFile).sort();
-  }
-
-  autoFilePath(agentDir: string, fileName: string): string {
-    return joinPath(agentDir, fileName);
-  }
-
-  async deleteAutoFile(path: string): Promise<void> {
-    await removeFile(path);
-  }
-
-  async withMemoryLock<T>(fn: () => Promise<T>): Promise<T> {
-    await this.ensureAgentDir();
-    await ensureFile(this.memoryFile);
-
-    const release = await lockfile.lock(this.memoryFile, {
-      retries: {
-        retries: MEMORY_LOCK_RETRIES,
-        minTimeout: MEMORY_LOCK_MIN_TIMEOUT_MS,
-        maxTimeout: MEMORY_LOCK_MAX_TIMEOUT_MS,
-      },
-    });
-    try {
-      return await fn();
-    } finally {
-      await release();
-    }
-  }
-}

package/src/store/index.ts

@@ -1,5 +0,0 @@
-export { MemoryStore, createMemoryStore } from "./memoryStore.js";
-export { MarkdownMemoryBackend } from "./backend.js";
-export { getAgentPaths, resolveAgentDir } from "./paths.js";
-export * from "./types.js";
-export * from "../constants/memory.js";

package/src/store/indexChunks.ts

@@ -1,89 +0,0 @@
-import type { IndexDocument } from "../sidecar/protocol.js";
-import type { ChunkingConfig } from "../config/chunking.js";
-import { CHUNKING_DISABLED_MAX_CHARS } from "../constants/chunking.js";
-import { pathBasename } from "../utils/fs.js";
-import type { MemorySection, ParsedEntry } from "./types.js";
-
-function formatChunkContent(section: MemorySection, body: string): string {
-  return `[${section}] ${body.trim()}`;
-}
-
-/**
- * Split long text on paragraph / line / sentence boundaries, then hard-wrap at maxChars.
- */
-export function splitTextByMaxChars(text: string, maxChars: number): string[] {
-  const trimmed = text.trim();
-  if (!trimmed) return [];
-  if (maxChars <= 0 || trimmed.length <= maxChars) return [trimmed];
-
-  const parts: string[] = [];
-  let remaining = trimmed;
-
-  while (remaining.length > maxChars) {
-    let cut = maxChars;
-    const minCut = Math.floor(maxChars * 0.35);
-
-    const paragraph = remaining.lastIndexOf("\n\n", cut);
-    if (paragraph >= minCut) {
-      cut = paragraph;
-    } else {
-      const line = remaining.lastIndexOf("\n", cut);
-      if (line >= minCut) {
-        cut = line;
-      } else {
-        const sentence = remaining.lastIndexOf(". ", cut);
-        if (sentence >= minCut) {
-          cut = sentence + 1;
-        } else {
-          const space = remaining.lastIndexOf(" ", cut);
-          if (space >= minCut) {
-            cut = space;
-          }
-        }
-      }
-    }
-
-    const part = remaining.slice(0, cut).trim();
-    if (part) parts.push(part);
-    remaining = remaining.slice(cut).trim();
-    if (!part && remaining.length > 0) {
-      parts.push(remaining.slice(0, maxChars).trim());
-      remaining = remaining.slice(maxChars).trim();
-    }
-  }
-
-  if (remaining) parts.push(remaining);
-  return parts.length > 0 ? parts : [trimmed];
-}
-
-export function buildIndexDocuments(
-  entries: ParsedEntry[],
-  chunking: ChunkingConfig,
-): IndexDocument[] {
-  const docs: IndexDocument[] = [];
-  const maxChars =
-    chunking.maxChars <= CHUNKING_DISABLED_MAX_CHARS
-      ? CHUNKING_DISABLED_MAX_CHARS
-      : chunking.maxChars;
-
-  for (const entry of entries) {
-    const bodies =
-      maxChars <= CHUNKING_DISABLED_MAX_CHARS
-        ? [entry.content.trim()].filter(Boolean)
-        : splitTextByMaxChars(entry.content, maxChars);
-
-    if (bodies.length === 0) continue;
-
-    bodies.forEach((body, index) => {
-      const id = bodies.length === 1 ? entry.id : `${entry.id}#${index}`;
-      docs.push({
-        id,
-        content: formatChunkContent(entry.section, body),
-        source: pathBasename(entry.sourceFile),
-        timestamp: entry.timestamp,
-      });
-    });
-  }
-
-  return docs;
-}

package/src/store/markdown/format.ts

@@ -1,16 +0,0 @@
-import type { MemorySection, StoreMemoryEntry } from "../types.js";
-
-export function formatEntryLine(entry: StoreMemoryEntry): string {
-  const prefix = entry.userAuthored ? "[user] " : "";
-  const meta = `<!-- id:${entry.id}${entry.userAuthored ? " user" : ""} ts:${entry.timestamp} -->`;
-  return `- ${prefix}${entry.content.trim()} ${meta}`;
-}
-
-export function formatSectionHeader(section: MemorySection): string {
-  return `## ${section}`;
-}
-
-export function countLines(content: string): number {
-  if (!content) return 0;
-  return content.split("\n").length;
-}

package/src/store/markdown/parse.ts

@@ -1,63 +0,0 @@
-import { pathBasename } from "../../utils/fs.js";
-import { epochTimestamp } from "../../utils/time.js";
-
-import { OVERFLOW_POINTER_RE } from "../../constants/memory.js";
-import { MEMORY_SECTIONS, type MemorySection, type ParsedEntry } from "../types.js";
-
-const SECTION_RE = /^##\s+(Preferences|Conventions|Findings|Todos)\s*$/;
-const ENTRY_META_RE = /<!--\s*id:([^\s]+)(?:\s+user)?(?:\s+ts:([^\s]+))?\s*-->/;
-const USER_PREFIX_RE = /^\[user\]\s+/;
-
-export function parseMemoryMarkdown(content: string, sourceFile: string): ParsedEntry[] {
-  const entries: ParsedEntry[] = [];
-  let section: MemorySection | undefined;
-  const lines = content.split("\n");
-
-  for (let i = 0; i < lines.length; i++) {
-    const line = lines[i]!;
-    const sectionMatch = line.match(SECTION_RE);
-    if (sectionMatch) {
-      section = sectionMatch[1] as MemorySection;
-      continue;
-    }
-
-    if (!line.startsWith("- ") || !section) continue;
-
-    const pointerMatch = line.match(OVERFLOW_POINTER_RE);
-    if (pointerMatch) continue;
-
-    const metaMatch = line.match(ENTRY_META_RE);
-    const id = metaMatch?.[1] ?? `${pathBasename(sourceFile)}:${i + 1}`;
-    const timestamp = metaMatch?.[2] ?? epochTimestamp();
-    let body = line.slice(2).replace(ENTRY_META_RE, "").trim();
-    const userAuthored = USER_PREFIX_RE.test(body);
-    if (userAuthored) body = body.replace(USER_PREFIX_RE, "").trim();
-
-    if (!body) continue;
-
-    entries.push({
-      id,
-      section,
-      content: body,
-      userAuthored: userAuthored || undefined,
-      timestamp,
-      sourceFile,
-      line: i + 1,
-    });
-  }
-
-  return entries;
-}
-
-export function listOverflowPointers(content: string): string[] {
-  const files: string[] = [];
-  for (const line of content.split("\n")) {
-    const match = line.match(OVERFLOW_POINTER_RE);
-    if (match?.[1]) files.push(match[1]);
-  }
-  return files;
-}
-
-export function isKnownSection(name: string): name is MemorySection {
-  return (MEMORY_SECTIONS as readonly string[]).includes(name);
-}