@chendpoc/pi-memory 0.1.13 → 0.2.1

package/src/sidecar/client.ts

@@ -1,166 +1,114 @@
-import { randomBytes } from "node:crypto";
-import http from "node:http";
+// Agent 侧：发送请求到 sidecar
+import { randomUUID } from "node:crypto";
+import net from "node:net";
 
 import {
-  classifyHTTP,
-  classifyTransportError,
-  ErrTransport,
-  type ErrorClass,
-} from "../errclass.js";
-import type {
-  HealthPayload,
-  QueryIntent,
-  QueryRequest,
-  ReloadResponse,
-  ResponseEnvelope,
-} from "../types.js";
+  SIDECAR_PING_TIMEOUT_MS,
+  SIDECAR_QUERY_TIMEOUT_MS,
+  SIDECAR_REINDEX_TIMEOUT_MS,
+} from "../constants/timing.js";
+import { JsonlFramer, parseJsonlLine, serializeJsonlFrame } from "../ipc/jsonlFramer.js";
+import { isErrorResponse, type IndexDocument, type IndexStats, type SidecarResponse } from "./protocol.js";
 
-function requestId(): string {
-  return `req-${randomBytes(6).toString("hex")}`;
-}
+export function sidecarRequest<T extends SidecarResponse>(
+  socketPath: string,
+  frame: Record<string, unknown>,
+  timeoutMs = SIDECAR_QUERY_TIMEOUT_MS,
+): Promise<T> {
+  return new Promise((resolve, reject) => {
+    const socket = net.connect(socketPath);
+    const framer = new JsonlFramer();
 
-export class SidecarClient {
-  constructor(
-    private readonly socketPath: string,
-    private readonly timeoutMs: number,
-  ) {}
+    const timer = setTimeout(() => {
+      socket.destroy();
+      reject(new Error("Sidecar request timed out"));
+    }, timeoutMs);
 
-  async health(signal?: AbortSignal): Promise<HealthPayload> {
-    return this.do<HealthPayload>("GET", "/health", undefined, signal);
-  }
+    socket.on("connect", () => {
+      socket.write(serializeJsonlFrame(frame));
+    });
+
+    socket.on("data", (chunk) => {
+      for (const line of framer.push(chunk.toString())) {
+        clearTimeout(timer);
+        socket.end();
+
+        let response: SidecarResponse;
+        try {
+          response = parseJsonlLine<SidecarResponse>(line);
+        } catch {
+          reject(new Error("Invalid JSON response from sidecar"));
+          return;
+        }
+
+        if (isErrorResponse(response)) {
+          reject(new Error(response.error));
+          return;
+        }
 
-  async query(
-    intent: QueryIntent,
-    signal?: AbortSignal,
-  ): Promise<{
-    env: ResponseEnvelope | null;
-    errorClass: ErrorClass;
-    transportError?: Error;
-  }> {
-    const rid = requestId();
-    const body: QueryRequest = { intent, request_id: rid };
-    try {
-      const { status, data } = await this.doRaw<ResponseEnvelope>(
-        "POST",
-        "/query",
-        body,
-        signal,
-        rid,
-      );
-      return {
-        env: data,
-        errorClass: classifyHTTP(status, data),
-      };
-    } catch (err) {
-      if (err instanceof Error && err.message.includes("transport")) {
-        return {
-          env: null,
-          errorClass: classifyTransportError(err),
-          transportError: err,
-        };
+        resolve(response as T);
+        return;
       }
-      return {
-        env: null,
-        errorClass: "unavailable",
-        transportError: err instanceof Error ? err : new Error(String(err)),
-      };
-    }
-  }
+    });
 
-  async reload(signal?: AbortSignal): Promise<ReloadResponse> {
-    return this.do<ReloadResponse>("POST", "/bundle/reload", {}, signal);
-  }
+    socket.on("error", reject);
+  });
+}
 
-  private async do<T>(
-    method: string,
-    path: string,
-    body: unknown,
-    signal?: AbortSignal,
-    requestIdHeader?: string,
-  ): Promise<T> {
-    const { data } = await this.doRaw<T>(
-      method,
-      path,
-      body,
-      signal,
-      requestIdHeader,
+export async function ping(socketPath: string): Promise<boolean> {
+  try {
+    const res = await sidecarRequest<Extract<SidecarResponse, { type: "pong" }>>(
+      socketPath,
+      { type: "ping" },
+      SIDECAR_PING_TIMEOUT_MS,
     );
-    return data;
+    return res.type === "pong";
+  } catch {
+    return false;
   }
+}
 
-  private doRaw<T>(
-    method: string,
-    path: string,
-    body: unknown,
-    signal?: AbortSignal,
-    requestIdHeader?: string,
-  ): Promise<{ status: number; data: T }> {
-    return new Promise((resolve, reject) => {
-      const payload = body != null ? JSON.stringify(body) : undefined;
-      const req = http.request(
-        {
-          socketPath: this.socketPath,
-          path,
-          method,
-          headers: {
-            "X-Request-ID": requestIdHeader ?? requestId(),
-            ...(payload
-              ? {
-                  "Content-Type": "application/json",
-                  "Content-Length": Buffer.byteLength(payload),
-                }
-              : {}),
-          },
-          timeout: this.timeoutMs,
-        },
-        (res) => {
-          const chunks: Buffer[] = [];
-          res.on("data", (c) => chunks.push(c));
-          res.on("end", () => {
-            const text = Buffer.concat(chunks).toString("utf8");
-            if (!text) {
-              reject(new Error(`${ErrTransport.message}: empty body`));
-              return;
-            }
-            try {
-              resolve({
-                status: res.statusCode ?? 0,
-                data: JSON.parse(text) as T,
-              });
-            } catch (e) {
-              reject(
-                new Error(
-                  `${ErrTransport.message}: decode: ${e instanceof Error ? e.message : e}`,
-                ),
-              );
-            }
-          });
-        },
-      );
+export async function fetchIndexStats(
+  socketPath: string,
+): Promise<{ stats: IndexStats } | { error: string }> {
+  try {
+    const res = await sidecarRequest<Extract<SidecarResponse, { type: "stats_ok" }>>(
+      socketPath,
+      { type: "stats" },
+      SIDECAR_PING_TIMEOUT_MS,
+    );
+    if (res.type !== "stats_ok") return { error: "unexpected sidecar response" };
+    const { type: _type, ...stats } = res;
+    return { stats };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return { error: message };
+  }
+}
 
-      req.on("timeout", () => {
-        req.destroy(new Error(`${ErrTransport.message}: timeout`));
-      });
-      req.on("error", (err) => {
-        reject(new Error(`${ErrTransport.message}: ${err.message}`));
-      });
+export async function query(
+  socketPath: string,
+  queryText: string,
+  timeoutMs = SIDECAR_QUERY_TIMEOUT_MS,
+) {
+  const request_id = randomUUID();
+  return sidecarRequest<Extract<SidecarResponse, { type: "result" }>>(
+    socketPath,
+    { type: "query", request_id, query: queryText },
+    timeoutMs,
+  );
+}
 
-      if (signal) {
-        if (signal.aborted) {
-          req.destroy(new Error(`${ErrTransport.message}: aborted`));
-          return;
-        }
-        signal.addEventListener(
-          "abort",
-          () => {
-            req.destroy(new Error(`${ErrTransport.message}: aborted`));
-          },
-          { once: true },
-        );
-      }
+export type ReindexResult = Extract<SidecarResponse, { type: "reindex_ok" }>;
 
-      if (payload) req.write(payload);
-      req.end();
-    });
-  }
+export async function reindex(
+  socketPath: string,
+  documents: IndexDocument[] = [],
+): Promise<ReindexResult> {
+  const request_id = randomUUID();
+  return sidecarRequest<ReindexResult>(
+    socketPath,
+    { type: "reindex", request_id, documents },
+    SIDECAR_REINDEX_TIMEOUT_MS,
+  );
 }

package/src/sidecar/paths.ts

@@ -0,0 +1,31 @@
+import { fileURLToPath } from "node:url";
+
+import { SIDECAR_DB_FILE, SIDECAR_SOCKET_FILE } from "../constants/paths.js";
+import { canRead, joinPath, pathDirname } from "../utils/fs.js";
+
+export type SidecarPaths = {
+  socketPath: string;
+  dbPath: string;
+};
+
+export function resolveSidecarPaths(agentDir: string): SidecarPaths {
+  return {
+    socketPath: joinPath(agentDir, SIDECAR_SOCKET_FILE),
+    dbPath: joinPath(agentDir, SIDECAR_DB_FILE),
+  };
+}
+
+/** Resolve compiled sidecar entry (dist/sidecar/server/process.js). */
+export function resolveSidecarEntry(): string {
+  const here = pathDirname(fileURLToPath(import.meta.url));
+  const candidates = [
+    joinPath(here, "server/process.js"),
+    joinPath(here, "../../dist/sidecar/server/process.js"),
+  ];
+
+  for (const entry of candidates) {
+    if (canRead(entry)) return entry;
+  }
+
+  throw new Error("Sidecar entry not found; run `pnpm build` first");
+}

package/src/sidecar/protocol.ts

@@ -0,0 +1,38 @@
+export type MemoryEntry = {
+  content: string;
+  relevance: number;
+  timestamp: string;
+  source: string;
+};
+
+export type IndexDocument = {
+  id: string;
+  content: string;
+  source: string;
+  timestamp: string;
+};
+
+export type SidecarRequest =
+  | { type: "ping" }
+  | { type: "stats" }
+  | { type: "query"; request_id: string; query: string }
+  | { type: "reindex"; request_id: string; documents?: IndexDocument[] };
+
+export type IndexStats = {
+  index_generation: number;
+  chunk_count: number;
+  embedding_provider?: string;
+  embedding_model?: string;
+  embedding_dim?: number;
+};
+
+export type SidecarResponse =
+  | { type: "pong" }
+  | { type: "stats_ok" } & IndexStats
+  | { type: "result"; request_id: string; results: MemoryEntry[] }
+  | { type: "reindex_ok"; request_id: string; indexed: number; index_generation: number }
+  | { type: "error"; request_id?: string; error: string };
+
+export function isErrorResponse(res: SidecarResponse): res is Extract<SidecarResponse, { type: "error" }> {
+  return res.type === "error";
+}

package/src/sidecar/reindexBridge.ts

@@ -0,0 +1,52 @@
+import debounce from "lodash/debounce.js";
+
+import { DEFAULT_REINDEX_DEBOUNCE_MS } from "../constants/timing.js";
+
+import { sidecarQueryCache } from "../preflight/queryCache.js";
+import { ensureSidecarRunning, type SidecarOpts } from "./sidecarManager.js";
+import { reindex } from "./client.js";
+import type { IndexDocument } from "./protocol.js";
+
+export type ReindexScheduler = {
+  schedule(): void;
+  runNow(): Promise<void>;
+};
+
+export function createReindexScheduler(opts: {
+  sidecar: SidecarOpts;
+  agentDir: string;
+  getDocuments: () => Promise<IndexDocument[]>;
+  debounceMs?: number;
+}): ReindexScheduler {
+  let running = false;
+  let pending = false;
+
+  const runNow = async (): Promise<void> => {
+    if (running) {
+      pending = true;
+      return;
+    }
+
+    running = true;
+    try {
+      await ensureSidecarRunning(opts.sidecar);
+      const documents = await opts.getDocuments();
+      const result = await reindex(opts.sidecar.socketPath, documents);
+      sidecarQueryCache.onReindexComplete(opts.agentDir, result.index_generation);
+    } catch {
+      // fail-silent; preflight falls back to MEMORY.md
+    } finally {
+      running = false;
+      if (pending) {
+        pending = false;
+        await runNow();
+      }
+    }
+  };
+
+  const schedule = debounce(() => {
+    void runNow();
+  }, opts.debounceMs ?? DEFAULT_REINDEX_DEBOUNCE_MS);
+
+  return { schedule, runNow };
+}

package/src/sidecar/server/process.ts

@@ -0,0 +1,27 @@
+// Sidecar 子进程入口
+import { loadEnv } from "../../config/loadEnv.js";
+import { createSidecarServer } from "./server.js";
+
+loadEnv();
+
+function readArg(flag: string): string {
+  const idx = process.argv.indexOf(flag);
+  const value = idx === -1 ? undefined : process.argv[idx + 1];
+  if (!value) {
+    throw new Error(`Missing required argument: ${flag}`);
+  }
+  return value;
+}
+
+const socketPath = readArg("--socket");
+const dbPath = readArg("--db");
+
+const { shutdown } = createSidecarServer({ socketPath, dbPath });
+
+function onSignal(): void {
+  shutdown();
+  process.exit(0);
+}
+
+process.on("SIGTERM", onSignal);
+process.on("SIGINT", onSignal);

package/src/sidecar/server/query.ts

@@ -0,0 +1,16 @@
+import type { MemoryEntry, SidecarResponse } from "../protocol.js";
+import { getVecStore } from "./vec/store.js";
+
+export type QueryContext = {
+  dbPath: string;
+};
+
+export async function handleQuery(
+  requestId: string,
+  queryText: string,
+  ctx: QueryContext,
+): Promise<Extract<SidecarResponse, { type: "result" }>> {
+  const store = getVecStore(ctx.dbPath);
+  const results: MemoryEntry[] = await store.query(queryText);
+  return { type: "result", request_id: requestId, results };
+}

package/src/sidecar/server/reindex.ts

@@ -0,0 +1,21 @@
+import type { IndexDocument, SidecarResponse } from "../protocol.js";
+import { getVecStore } from "./vec/store.js";
+
+export type ReindexContext = {
+  dbPath: string;
+};
+
+export async function handleReindex(
+  requestId: string,
+  ctx: ReindexContext,
+  documents: IndexDocument[] = [],
+): Promise<Extract<SidecarResponse, { type: "reindex_ok" }>> {
+  const store = getVecStore(ctx.dbPath);
+  const outcome = await store.reindex(documents);
+  return {
+    type: "reindex_ok",
+    request_id: requestId,
+    indexed: outcome.indexed,
+    index_generation: outcome.indexGeneration,
+  };
+}

package/src/sidecar/server/server.ts

@@ -0,0 +1,105 @@
+import { createServer, type Server, type Socket } from "node:net";
+import { writeFileSync } from "node:fs";
+
+import { SIDECAR_PID_SUFFIX } from "../../constants/paths.js";
+import { JsonlFramer, parseJsonlLine, serializeJsonlFrame } from "../../ipc/jsonlFramer.js";
+import { ensureDirSync, pathDirname } from "../../utils/fs.js";
+import { cleanupSocketFiles, removeSocketFile, secureSocketPath } from "../../utils/socket.js";
+import type { SidecarRequest, SidecarResponse } from "../protocol.js";
+import { handleQuery } from "./query.js";
+import { handleReindex } from "./reindex.js";
+import { handleStats } from "./stats.js";
+
+export type SidecarServerOpts = {
+  socketPath: string;
+  dbPath: string;
+};
+
+export type SidecarServer = {
+  server: Server;
+  shutdown: () => void;
+};
+
+function writeResponse(socket: Socket, response: SidecarResponse): void {
+  socket.write(serializeJsonlFrame(response));
+}
+
+function writeError(socket: Socket, error: string, requestId?: string): void {
+  writeResponse(socket, { type: "error", request_id: requestId, error });
+}
+
+async function dispatchFrame(
+  frame: SidecarRequest,
+  socket: Socket,
+  ctx: SidecarServerOpts,
+): Promise<void> {
+  switch (frame.type) {
+    case "ping":
+      writeResponse(socket, { type: "pong" });
+      return;
+    case "stats":
+      writeResponse(socket, handleStats({ dbPath: ctx.dbPath }));
+      return;
+    case "query":
+      writeResponse(socket, await handleQuery(frame.request_id, frame.query, { dbPath: ctx.dbPath }));
+      return;
+    case "reindex":
+      writeResponse(
+        socket,
+        await handleReindex(frame.request_id, { dbPath: ctx.dbPath }, frame.documents),
+      );
+      return;
+    default: {
+      const unknown = frame as { type?: string; request_id?: string };
+      writeError(socket, `unknown frame type: ${unknown.type ?? "undefined"}`, unknown.request_id);
+    }
+  }
+}
+
+export function createSidecarServer(opts: SidecarServerOpts): SidecarServer {
+  const pidPath = opts.socketPath + SIDECAR_PID_SUFFIX;
+
+  ensureDirSync(pathDirname(opts.socketPath));
+  removeSocketFile(opts.socketPath);
+
+  const server = createServer((socket) => {
+    const framer = new JsonlFramer();
+
+    socket.on("data", (chunk) => {
+      for (const line of framer.push(chunk.toString())) {
+        let frame: SidecarRequest;
+        try {
+          frame = parseJsonlLine<SidecarRequest>(line);
+        } catch {
+          writeError(socket, "invalid JSON frame");
+          continue;
+        }
+
+        void dispatchFrame(frame, socket, opts).catch((error: unknown) => {
+          const message = error instanceof Error ? error.message : String(error);
+          const requestId = typeof frame === "object" && frame && "request_id" in frame
+            ? String(frame.request_id)
+            : undefined;
+          writeError(socket, message, requestId);
+        });
+      }
+    });
+  });
+
+  function shutdown(): void {
+    server.close();
+    cleanupSocketFiles(opts.socketPath, pidPath);
+  }
+
+  server.listen(opts.socketPath, () => {
+    writeFileSync(pidPath, String(process.pid));
+    secureSocketPath(opts.socketPath);
+  });
+
+  server.on("error", (error) => {
+    console.error("Sidecar server error:", error);
+    process.exit(1);
+  });
+
+  return { server, shutdown };
+}

package/src/sidecar/server/stats.ts

@@ -0,0 +1,23 @@
+import type { SidecarResponse } from "../protocol.js";
+import { getVecStore } from "./vec/store.js";
+
+export type StatsContext = {
+  dbPath: string;
+};
+
+export function handleStats(ctx: StatsContext): Extract<SidecarResponse, { type: "stats_ok" }> {
+  const store = getVecStore(ctx.dbPath);
+  const meta = store.getStoredEmbeddingMeta();
+  return {
+    type: "stats_ok",
+    index_generation: store.getIndexGeneration(),
+    chunk_count: store.getChunkCount(),
+    ...(meta
+      ? {
+          embedding_provider: meta.provider,
+          embedding_model: meta.model,
+          embedding_dim: meta.dim,
+        }
+      : {}),
+  };
+}

package/src/sidecar/server/vec/embedder.ts

@@ -0,0 +1 @@
+export { getEmbedder, resetEmbedderForTests, createEmbedder, type Embedder } from "../../../adapters/embed/factory.js";

package/src/sidecar/server/vec/mmr.ts

@@ -0,0 +1,55 @@
+import { MMR_LAMBDA } from "../../../constants/retrieval.js";
+
+export type ScoredCandidate = {
+  chunkId: string;
+  content: string;
+  source: string;
+  timestamp: string;
+  distance: number;
+  embedding: Float32Array;
+};
+
+export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
+  let dot = 0;
+  for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
+  return dot;
+}
+
+/** MMR re-ranking: λ defaults from retrieval config (see constants/retrieval.ts). */
+export function mmrSelect(
+  queryEmbedding: Float32Array,
+  candidates: ScoredCandidate[],
+  limit: number,
+  lambda = MMR_LAMBDA,
+): ScoredCandidate[] {
+  if (candidates.length === 0) return [];
+
+  const selected: ScoredCandidate[] = [];
+  const remaining = [...candidates];
+
+  while (selected.length < limit && remaining.length > 0) {
+    let bestIdx = 0;
+    let bestScore = -Infinity;
+
+    for (let i = 0; i < remaining.length; i++) {
+      const candidate = remaining[i]!;
+      const relevance = cosineSimilarity(queryEmbedding, candidate.embedding);
+      const maxSim = selected.length === 0
+        ? 0
+        : Math.max(...selected.map((s) => cosineSimilarity(candidate.embedding, s.embedding)));
+      const score = lambda * relevance - (1 - lambda) * maxSim;
+      if (score > bestScore) {
+        bestScore = score;
+        bestIdx = i;
+      }
+    }
+
+    selected.push(remaining.splice(bestIdx, 1)[0]!);
+  }
+
+  return selected;
+}
+
+export function distanceToRelevance(distance: number): number {
+  return Math.max(0, Math.min(1, 1 - distance));
+}