@vellumai/assistant 0.6.4 → 0.6.5

package/src/memory/pkb/pkb-reconcile.test.ts

@@ -0,0 +1,251 @@
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+mock.module("../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+// Capture enqueued PKB re-index jobs.
+const enqueuedJobs: Array<{
+  pkbRoot: string;
+  absPath: string;
+  memoryScopeId: string;
+}> = [];
+
+mock.module("../jobs/embed-pkb-file.js", () => ({
+  enqueuePkbIndexJob: (input: {
+    pkbRoot: string;
+    absPath: string;
+    memoryScopeId: string;
+  }) => {
+    enqueuedJobs.push(input);
+    return "job-id";
+  },
+}));
+
+// Capture calls into the fake Qdrant client.
+let scrollPoints: Array<{ id: string; payload: Record<string, unknown> }> = [];
+const deleteCalls: Array<{ path: string; memoryScopeId: string }> = [];
+
+mock.module("../qdrant-client.js", () => ({
+  getQdrantClient: () => ({
+    scrollByTargetType: async (
+      _targetType: string,
+      _options?: { memoryScopeId?: string },
+    ) => scrollPoints,
+    deleteByTargetTypeAndPath: async (
+      _targetType: string,
+      path: string,
+      memoryScopeId: string,
+    ) => {
+      deleteCalls.push({ path, memoryScopeId });
+    },
+  }),
+}));
+
+// Circuit breaker — pass-through.
+mock.module("../qdrant-circuit-breaker.js", () => ({
+  withQdrantBreaker: async <T,>(fn: () => Promise<T>) => fn(),
+}));
+
+// indexPkbFile is not invoked from the reconcile path (we enqueue jobs
+// instead), but the pkb-index module imports the embedding pipeline which
+// pulls in a config. Stub it so module import doesn't explode.
+mock.module("../job-utils.js", () => ({
+  embedAndUpsert: async () => {},
+}));
+mock.module("../../config/loader.js", () => ({
+  getConfig: () => ({ __stub: true }),
+}));
+
+import { reconcilePkbIndex } from "./pkb-reconcile.js";
+
+function pkbPoint(
+  path: string,
+  contentHash: string,
+  chunkIndex = 0,
+): { id: string; payload: Record<string, unknown> } {
+  return {
+    id: `${path}#${chunkIndex}`,
+    payload: {
+      target_type: "pkb_file",
+      target_id: `${path}#${chunkIndex}`,
+      path,
+      chunk_index: chunkIndex,
+      content_hash: contentHash,
+      memory_scope_id: "default",
+    },
+  };
+}
+
+// sha256(content).slice(0, 16) — precomputing isn't necessary; reconcile only
+// compares hashes for equality, so we read the real hash after scanning.
+import { scanPkbFiles } from "./pkb-index.js";
+
+async function seedPkbAndHash(
+  root: string,
+  files: Array<{ path: string; content: string }>,
+): Promise<Map<string, string>> {
+  for (const f of files) {
+    await writeFile(join(root, f.path), f.content);
+  }
+  const entries = await scanPkbFiles(root);
+  const byPath = new Map<string, string>();
+  for (const e of entries ?? []) byPath.set(e.path, e.contentHash);
+  return byPath;
+}
+
+describe("reconcilePkbIndex", () => {
+  beforeEach(() => {
+    enqueuedJobs.length = 0;
+    deleteCalls.length = 0;
+    scrollPoints = [];
+  });
+
+  test("fresh install: enqueues every on-disk file, deletes nothing", async () => {
+    const root = await mkdtemp(join(tmpdir(), "pkb-reconcile-fresh-"));
+    await seedPkbAndHash(root, [
+      { path: "a.md", content: "# A" },
+      { path: "b.md", content: "# B" },
+    ]);
+
+    // Qdrant empty.
+    scrollPoints = [];
+
+    const result = await reconcilePkbIndex(root, "default");
+
+    expect(result.enqueued).toBe(2);
+    expect(result.deleted).toBe(0);
+    expect(enqueuedJobs).toHaveLength(2);
+    const paths = new Set(enqueuedJobs.map((j) => j.absPath));
+    expect(paths.has(join(root, "a.md"))).toBe(true);
+    expect(paths.has(join(root, "b.md"))).toBe(true);
+    for (const job of enqueuedJobs) {
+      expect(job.pkbRoot).toBe(root);
+      expect(job.memoryScopeId).toBe("default");
+    }
+    expect(deleteCalls).toHaveLength(0);
+  });
+
+  test("steady state: matching hashes → no work enqueued or deleted", async () => {
+    const root = await mkdtemp(join(tmpdir(), "pkb-reconcile-steady-"));
+    const hashes = await seedPkbAndHash(root, [
+      { path: "a.md", content: "# A" },
+      { path: "b.md", content: "# B" },
+    ]);
+
+    scrollPoints = [
+      pkbPoint("a.md", hashes.get("a.md")!),
+      pkbPoint("b.md", hashes.get("b.md")!),
+    ];
+
+    const result = await reconcilePkbIndex(root, "default");
+
+    expect(result.enqueued).toBe(0);
+    expect(result.deleted).toBe(0);
+    expect(enqueuedJobs).toHaveLength(0);
+    expect(deleteCalls).toHaveLength(0);
+  });
+
+  test("modified file: hash drift enqueues exactly one re-index", async () => {
+    const root = await mkdtemp(join(tmpdir(), "pkb-reconcile-modified-"));
+    const hashes = await seedPkbAndHash(root, [
+      { path: "a.md", content: "# A updated" },
+      { path: "b.md", content: "# B" },
+    ]);
+
+    // Qdrant has a stale hash for a.md but matches for b.md.
+    scrollPoints = [
+      pkbPoint("a.md", "0000000000000000"),
+      pkbPoint("b.md", hashes.get("b.md")!),
+    ];
+
+    const result = await reconcilePkbIndex(root, "default");
+
+    expect(result.enqueued).toBe(1);
+    expect(result.deleted).toBe(0);
+    expect(enqueuedJobs).toHaveLength(1);
+    expect(enqueuedJobs[0].absPath).toBe(join(root, "a.md"));
+    expect(deleteCalls).toHaveLength(0);
+  });
+
+  test("stale indexed path not on disk: deletes that path, enqueues nothing", async () => {
+    const root = await mkdtemp(join(tmpdir(), "pkb-reconcile-stale-"));
+    const hashes = await seedPkbAndHash(root, [
+      { path: "a.md", content: "# A" },
+    ]);
+
+    // Qdrant has a.md (matching) plus a phantom gone.md.
+    scrollPoints = [
+      pkbPoint("a.md", hashes.get("a.md")!),
+      pkbPoint("gone.md", "dead00dead00dead"),
+    ];
+
+    const result = await reconcilePkbIndex(root, "default");
+
+    expect(result.enqueued).toBe(0);
+    expect(result.deleted).toBe(1);
+    expect(deleteCalls).toEqual([
+      { path: "gone.md", memoryScopeId: "default" },
+    ]);
+    expect(enqueuedJobs).toHaveLength(0);
+  });
+
+  test("missing pkbRoot: indexed points are preserved (no wholesale deletion)", async () => {
+    const parent = await mkdtemp(join(tmpdir(), "pkb-reconcile-missing-"));
+    const missing = join(parent, "does-not-exist");
+
+    // Qdrant has points for paths that would look "stale" if we treated a
+    // missing root as an empty disk view — the regression this test guards
+    // against is those being wholesale-deleted.
+    scrollPoints = [
+      pkbPoint("a.md", "aaaaaaaaaaaaaaaa"),
+      pkbPoint("b.md", "bbbbbbbbbbbbbbbb"),
+      pkbPoint("notes/c.md", "cccccccccccccccc"),
+    ];
+
+    const result = await reconcilePkbIndex(missing, "default");
+
+    expect(result.enqueued).toBe(0);
+    expect(result.deleted).toBe(0);
+    expect(enqueuedJobs).toHaveLength(0);
+    expect(deleteCalls).toHaveLength(0);
+  });
+
+  test("pkbRoot existed then was removed: indexed points are preserved", async () => {
+    const root = await mkdtemp(join(tmpdir(), "pkb-reconcile-vanished-"));
+    await writeFile(join(root, "a.md"), "# A");
+    await rm(root, { recursive: true, force: true });
+
+    scrollPoints = [pkbPoint("a.md", "aaaaaaaaaaaaaaaa")];
+
+    const result = await reconcilePkbIndex(root, "default");
+
+    expect(result.enqueued).toBe(0);
+    expect(result.deleted).toBe(0);
+    expect(deleteCalls).toHaveLength(0);
+  });
+
+  test("collapses multiple chunk points for the same path", async () => {
+    const root = await mkdtemp(join(tmpdir(), "pkb-reconcile-chunks-"));
+    const hashes = await seedPkbAndHash(root, [
+      { path: "a.md", content: "# A" },
+    ]);
+
+    // Two chunk points for a.md with identical hashes.
+    scrollPoints = [
+      pkbPoint("a.md", hashes.get("a.md")!, 0),
+      pkbPoint("a.md", hashes.get("a.md")!, 1),
+    ];
+
+    const result = await reconcilePkbIndex(root, "default");
+
+    expect(result.enqueued).toBe(0);
+    expect(result.deleted).toBe(0);
+  });
+});

package/src/memory/pkb/pkb-reconcile.ts

@@ -0,0 +1,148 @@
+/**
+ * PKB (Personal Knowledge Base) startup reconciliation.
+ *
+ * Diffs the on-disk PKB tree against the points currently indexed in Qdrant
+ * and brings the two into alignment by enqueueing re-index jobs for
+ * changed/new files and deleting Qdrant points for files that no longer
+ * exist on disk.
+ *
+ * Safe to run on every daemon boot — idempotent. Intended to be invoked
+ * fire-and-forget from daemon startup so the filesystem view stays
+ * authoritative without blocking the first turn on scan latency.
+ */
+
+import { join } from "node:path";
+
+import { getLogger } from "../../util/logger.js";
+import { enqueuePkbIndexJob } from "../jobs/embed-pkb-file.js";
+import { getQdrantClient } from "../qdrant-client.js";
+import { deletePkbFilePoints, scanPkbFiles } from "./pkb-index.js";
+import { PKB_TARGET_TYPE } from "./types.js";
+
+const log = getLogger("pkb-reconcile");
+
+export interface ReconcilePkbIndexResult {
+  enqueued: number;
+  deleted: number;
+}
+
+/**
+ * Reconcile the PKB Qdrant index against the on-disk tree at `pkbRoot`.
+ *
+ * For each on-disk file whose `contentHash` differs from the indexed hash
+ * (or that is missing from the index entirely) enqueue a re-index job. For
+ * each indexed path no longer present on disk, delete its Qdrant points.
+ *
+ * Returns the number of jobs enqueued and the number of paths deleted.
+ */
+export async function reconcilePkbIndex(
+  pkbRoot: string,
+  memoryScopeId: string,
+): Promise<ReconcilePkbIndexResult> {
+  // Build the on-disk view keyed by relative path. `scanPkbFiles` emits one
+  // entry per chunk — every chunk of the same file shares the same
+  // contentHash, so collapsing to a per-path map is lossless for our purposes.
+  //
+  // `scanPkbFiles` returns `null` when `pkbRoot` itself is missing (vs. `[]`
+  // for an existing-but-empty directory). Treating a missing root as "empty"
+  // would cause reconciliation to delete every indexed PKB point in Qdrant,
+  // which is catastrophic if the directory is only transiently gone (fs
+  // hiccup, workspace dir mis-set, user moved the workspace). Bail out with
+  // a warning instead of touching Qdrant in that case.
+  const diskEntries = await scanPkbFiles(pkbRoot);
+  if (diskEntries === null) {
+    log.warn(
+      { pkbRoot, memoryScopeId },
+      "PKB root directory missing — skipping reconciliation to avoid wiping the index",
+    );
+    return { enqueued: 0, deleted: 0 };
+  }
+  const diskByPath = new Map<string, { contentHash: string }>();
+  for (const entry of diskEntries) {
+    if (!diskByPath.has(entry.path)) {
+      diskByPath.set(entry.path, { contentHash: entry.contentHash });
+    }
+  }
+
+  // Build the indexed view keyed by relative path. Qdrant stores one point
+  // per (path, chunk_index); every chunk for a given file SHOULD share a
+  // single content_hash, but an interrupted/partial index run can leave
+  // chunks of the same file with differing hashes. Track whether any two
+  // chunks disagree (`mixed`) so we can force a re-index — otherwise stale
+  // chunks remain searchable as long as the first chunk's hash happened to
+  // match the current disk content.
+  const qdrant = getQdrantClient();
+  const points = await qdrant.scrollByTargetType(PKB_TARGET_TYPE, {
+    memoryScopeId,
+  });
+  const indexedByPath = new Map<
+    string,
+    { contentHash: string; mixed: boolean }
+  >();
+  for (const { payload } of points) {
+    const path = typeof payload.path === "string" ? payload.path : undefined;
+    const contentHash =
+      typeof payload.content_hash === "string"
+        ? payload.content_hash
+        : undefined;
+    if (!path || !contentHash) continue;
+    const existing = indexedByPath.get(path);
+    if (!existing) {
+      indexedByPath.set(path, { contentHash, mixed: false });
+    } else if (existing.contentHash !== contentHash) {
+      existing.mixed = true;
+    }
+  }
+
+  let enqueued = 0;
+  let deleted = 0;
+
+  // Re-index files that are new, have changed on disk, or have mixed
+  // per-chunk hashes in the index (partial/interrupted prior indexing).
+  for (const [relPath, disk] of diskByPath) {
+    const indexed = indexedByPath.get(relPath);
+    if (
+      !indexed ||
+      indexed.mixed ||
+      indexed.contentHash !== disk.contentHash
+    ) {
+      enqueuePkbIndexJob({
+        pkbRoot,
+        absPath: join(pkbRoot, relPath),
+        memoryScopeId,
+      });
+      enqueued++;
+    }
+  }
+
+  // Remove indexed paths that no longer exist on disk.
+  for (const relPath of indexedByPath.keys()) {
+    if (!diskByPath.has(relPath)) {
+      try {
+        await deletePkbFilePoints(relPath, memoryScopeId);
+        deleted++;
+      } catch (err) {
+        log.warn(
+          { err, relPath },
+          "Failed to delete stale PKB points — continuing reconciliation",
+        );
+      }
+    }
+  }
+
+  if (enqueued > 0 || deleted > 0) {
+    log.info(
+      {
+        pkbRoot,
+        memoryScopeId,
+        enqueued,
+        deleted,
+        diskCount: diskByPath.size,
+        indexedCount: indexedByPath.size,
+      },
+      "PKB index reconciled against filesystem",
+    );
+  }
+
+  return { enqueued, deleted };
+}