@vellumai/assistant 0.8.6 → 0.8.7

package/src/memory/v3/__tests__/validate.test.ts

@@ -1,277 +0,0 @@
-/**
- * Tests for `assistant/src/memory/v3/validate.ts`.
- *
- * Coverage matrix — one fixture per defect category plus a clean-tree control:
- *   - clean tree → every list empty, every count 0.
- *   - danglingChildRefs → a `node:` ref and a `page:` ref to absent targets.
- *   - orphanPages → a concept page on disk not wired into the tree; synthetic
- *     page-index entries (none here) and reachable pages excluded.
- *   - cycles → A → B → A back-edge detected during the full descent.
- *   - staleIndex → a parent node whose mtime predates a `node:` child's mtime.
- *   - unknownEdgeTargets → a page `edges:` entry pointing at a missing slug.
- *
- * Tests use temp workspaces under `os.tmpdir()`; they never touch `~/.vellum/`.
- * mtimes are pinned with `utimes` so the freshness check is deterministic and
- * independent of write ordering / filesystem timestamp granularity.
- */
-
-import { mkdtempSync, rmSync } from "node:fs";
-import { utimes } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { afterEach, beforeEach, describe, expect, test } from "bun:test";
-
-import { invalidateEdgeIndex } from "../../v2/edge-index.js";
-import { invalidatePageIndex } from "../../v2/page-index.js";
-import { writePage } from "../../v2/page-store.js";
-import type { ConceptPage } from "../../v2/types.js";
-import { invalidateTreeIndex } from "../tree-index.js";
-import { getTreeDir, ROOT_NODE_ID, writeNode } from "../tree-store.js";
-import type { TreeNode } from "../types.js";
-import { validateTree } from "../validate.js";
-
-let workspaceDir: string;
-
-beforeEach(() => {
-  workspaceDir = mkdtempSync(join(tmpdir(), "vellum-tree-validate-test-"));
-});
-
-afterEach(() => {
-  invalidateTreeIndex();
-  invalidatePageIndex();
-  invalidateEdgeIndex();
-  rmSync(workspaceDir, { recursive: true, force: true });
-});
-
-function node(id: string, children: string[], body = `body ${id}`): TreeNode {
-  return { id, frontmatter: { children }, body };
-}
-
-function page(slug: string, edges: string[] = []): ConceptPage {
-  return {
-    slug,
-    frontmatter: { edges, ref_files: [], ref_urls: [] },
-    body: `body ${slug}`,
-  };
-}
-
-/** Pin a node file's mtime (and atime) to an explicit epoch-ms value. */
-async function setNodeMtime(id: string, mtimeMs: number): Promise<void> {
-  const path = join(getTreeDir(workspaceDir), `${id}.md`);
-  const t = new Date(mtimeMs);
-  await utimes(path, t, t);
-}
-
-/**
- * Invalidate every cached index after seeding so the first `validateTree` of a
- * test body sees the on-disk fixture rather than a stale cache.
- */
-function resetCaches(): void {
-  invalidateTreeIndex();
-  invalidatePageIndex();
-  invalidateEdgeIndex();
-}
-
-describe("validateTree — clean tree", () => {
-  test("returns an empty report for a well-formed tree", async () => {
-    // _root → node:people → page:alice ; all refs resolve, alice reachable.
-    await writeNode(workspaceDir, node(ROOT_NODE_ID, ["node:people"]));
-    await writeNode(workspaceDir, node("people", ["page:alice"]));
-    await writePage(workspaceDir, page("alice"));
-    // Parent newest so the freshness check never fires on a clean tree.
-    await setNodeMtime("people", 1_000);
-    await setNodeMtime(ROOT_NODE_ID, 2_000);
-    resetCaches();
-
-    const report = await validateTree(workspaceDir);
-
-    expect(report.danglingChildRefs).toEqual([]);
-    expect(report.danglingChildRefCount).toBe(0);
-    expect(report.orphanPages).toEqual([]);
-    expect(report.orphanPageCount).toBe(0);
-    expect(report.cycles).toEqual([]);
-    expect(report.cycleCount).toBe(0);
-    expect(report.staleIndex).toEqual([]);
-    expect(report.staleIndexCount).toBe(0);
-    expect(report.unknownEdgeTargets).toEqual([]);
-    expect(report.unknownEdgeTargetCount).toBe(0);
-  });
-});
-
-describe("validateTree — danglingChildRefs", () => {
-  test("flags node: and page: refs whose targets are missing", async () => {
-    await writeNode(
-      workspaceDir,
-      node(ROOT_NODE_ID, ["node:ghost", "page:missing-page"]),
-    );
-    resetCaches();
-
-    const report = await validateTree(workspaceDir);
-
-    expect(report.danglingChildRefs).toEqual([
-      { node: ROOT_NODE_ID, ref: "ghost", kind: "node" },
-      { node: ROOT_NODE_ID, ref: "missing-page", kind: "page" },
-    ]);
-    expect(report.danglingChildRefCount).toBe(2);
-  });
-
-  test("does not flag refs whose targets exist", async () => {
-    await writeNode(workspaceDir, node(ROOT_NODE_ID, ["node:child"]));
-    await writeNode(workspaceDir, node("child", ["page:alice"]));
-    await writePage(workspaceDir, page("alice"));
-    resetCaches();
-
-    const report = await validateTree(workspaceDir);
-
-    expect(report.danglingChildRefs).toEqual([]);
-  });
-});
-
-describe("validateTree — orphanPages", () => {
-  test("flags concept pages not reachable from the root", async () => {
-    await writeNode(workspaceDir, node(ROOT_NODE_ID, ["page:reached"]));
-    await writePage(workspaceDir, page("reached"));
-    await writePage(workspaceDir, page("orphan"));
-    resetCaches();
-
-    const report = await validateTree(workspaceDir);
-
-    expect(report.orphanPages).toEqual(["orphan"]);
-    expect(report.orphanPageCount).toBe(1);
-  });
-
-  test("a page hanging off an unreachable node is still an orphan", async () => {
-    // `floating` is not referenced by _root, so its page child is unreachable.
-    await writeNode(workspaceDir, node(ROOT_NODE_ID, []));
-    await writeNode(workspaceDir, node("floating", ["page:detached"]));
-    await writePage(workspaceDir, page("detached"));
-    resetCaches();
-
-    const report = await validateTree(workspaceDir);
-
-    expect(report.orphanPages).toEqual(["detached"]);
-  });
-});
-
-describe("validateTree — cycles", () => {
-  test("detects an A → B → A node cycle as a back-edge", async () => {
-    // _root → node:a → node:b → node:a (cycle closes on the b → a edge).
-    await writeNode(workspaceDir, node(ROOT_NODE_ID, ["node:a"]));
-    await writeNode(workspaceDir, node("a", ["node:b"]));
-    await writeNode(workspaceDir, node("b", ["node:a"]));
-    resetCaches();
-
-    const report = await validateTree(workspaceDir);
-
-    expect(report.cycles).toEqual([{ from: "b", to: "a" }]);
-    expect(report.cycleCount).toBe(1);
-  });
-
-  test("a shared DAG sub-node (two parents, no cycle) is not a cycle", async () => {
-    await writeNode(workspaceDir, node(ROOT_NODE_ID, ["node:p1", "node:p2"]));
-    await writeNode(workspaceDir, node("p1", ["node:shared"]));
-    await writeNode(workspaceDir, node("p2", ["node:shared"]));
-    await writeNode(workspaceDir, node("shared", []));
-    resetCaches();
-
-    const report = await validateTree(workspaceDir);
-
-    expect(report.cycles).toEqual([]);
-  });
-
-  test("detects a cycle in a component unreachable from the root", async () => {
-    // _root is a leaf with no edges; the x ↔ y cycle lives in a disconnected
-    // component the root can never reach. The sweep over unvisited nodes must
-    // still surface it.
-    await writeNode(workspaceDir, node(ROOT_NODE_ID, []));
-    await writeNode(workspaceDir, node("x", ["node:y"]));
-    await writeNode(workspaceDir, node("y", ["node:x"]));
-    resetCaches();
-
-    const report = await validateTree(workspaceDir);
-
-    expect(report.cycleCount).toBe(1);
-    // The back-edge closes on whichever of the pair the sweep enters second.
-    const [{ from, to }] = report.cycles;
-    expect(new Set([from, to])).toEqual(new Set(["x", "y"]));
-  });
-
-  test("a page off an unreachable cyclic node stays an orphan", async () => {
-    // The disconnected-component sweep covers x/y for cycle detection but must
-    // not mark them root-reachable — `detached` therefore remains an orphan.
-    await writeNode(workspaceDir, node(ROOT_NODE_ID, []));
-    await writeNode(workspaceDir, node("x", ["node:y", "page:detached"]));
-    await writeNode(workspaceDir, node("y", ["node:x"]));
-    await writePage(workspaceDir, page("detached"));
-    resetCaches();
-
-    const report = await validateTree(workspaceDir);
-
-    expect(report.cycleCount).toBe(1);
-    expect(report.orphanPages).toEqual(["detached"]);
-  });
-});
-
-describe("validateTree — staleIndex", () => {
-  test("flags a node whose mtime predates a node: child's mtime", async () => {
-    await writeNode(workspaceDir, node(ROOT_NODE_ID, ["node:child"]));
-    await writeNode(workspaceDir, node("child", []));
-    // Parent older than child → stale.
-    await setNodeMtime(ROOT_NODE_ID, 1_000);
-    await setNodeMtime("child", 5_000);
-    resetCaches();
-
-    const report = await validateTree(workspaceDir);
-
-    expect(report.staleIndex).toEqual([
-      {
-        node: ROOT_NODE_ID,
-        child: "child",
-        nodeMtimeMs: 1_000,
-        childMtimeMs: 5_000,
-      },
-    ]);
-    expect(report.staleIndexCount).toBe(1);
-  });
-
-  test("a parent newer than its child is not stale", async () => {
-    await writeNode(workspaceDir, node(ROOT_NODE_ID, ["node:child"]));
-    await writeNode(workspaceDir, node("child", []));
-    await setNodeMtime("child", 1_000);
-    await setNodeMtime(ROOT_NODE_ID, 5_000);
-    resetCaches();
-
-    const report = await validateTree(workspaceDir);
-
-    expect(report.staleIndex).toEqual([]);
-  });
-});
-
-describe("validateTree — unknownEdgeTargets", () => {
-  test("flags a page edge pointing at a missing slug", async () => {
-    await writeNode(workspaceDir, node(ROOT_NODE_ID, ["page:alice"]));
-    await writePage(workspaceDir, page("alice", ["nonexistent"]));
-    resetCaches();
-
-    const report = await validateTree(workspaceDir);
-
-    expect(report.unknownEdgeTargets).toEqual([
-      { from: "alice", to: "nonexistent" },
-    ]);
-    expect(report.unknownEdgeTargetCount).toBe(1);
-  });
-
-  test("an edge to an existing page is not flagged", async () => {
-    await writeNode(
-      workspaceDir,
-      node(ROOT_NODE_ID, ["page:alice", "page:bob"]),
-    );
-    await writePage(workspaceDir, page("alice", ["bob"]));
-    await writePage(workspaceDir, page("bob"));
-    resetCaches();
-
-    const report = await validateTree(workspaceDir);
-
-    expect(report.unknownEdgeTargets).toEqual([]);
-  });
-});

package/src/memory/v3/auto-edges.ts

@@ -1,223 +0,0 @@
-/**
- * Memory v3 — learned weighted auto-edge store.
- *
- * Read/write helpers over `memory_v3_auto_edges` (migration 263) — the
- * **learned** association graph, a distinct class from the curated `edges:`
- * frontmatter graph. Each row is a weighted directed pair `source → target`
- * that the edge-learning job accrues from *used* co-activations and decays over
- * wall-clock time.
- *
- * Three primitives:
- *   - {@link reinforce} — bump a pair's weight, but only for a *used*
- *     co-activation (we reinforce usefulness, not mere retrieval).
- *   - {@link decay} — multiplicatively decay all weights toward zero on a
- *     half-life schedule, so a pair that stops being reinforced fades. This is
- *     the rich-get-richer counterweight: weight is a leaky integrator, not a
- *     monotone counter.
- *   - {@link aboveThreshold} — project the learned graph to the
- *     `ReadonlyMap<source, ReadonlySet<target>>` adjacency that edge
- *     expansion's `extraAdjacency` seam consumes (only pairs at/above a weight
- *     threshold traverse).
- *
- * The decay model mirrors v2's injection-events EMA: `λ = ln 2 / halfLife`, and
- * a pair decays by `exp(-λ × elapsed)` since its `last_reinforced_at`.
- */
-
-import { getLogger } from "../../util/logger.js";
-import type { DrizzleDb } from "../db-connection.js";
-import { getSqliteFrom } from "../db-connection.js";
-
-const log = getLogger("memory-v3-auto-edges");
-
-/** Weight added to a pair per *used* co-activation reinforcement. */
-export const REINFORCE_INCREMENT = 1;
-
-/** Weights below this after decay are pruned rather than kept as dead rows. */
-export const PRUNE_FLOOR = 0.01;
-
-/** A learned auto-edge, as read back from the table. */
-export interface AutoEdgeRow {
-  sourceSlug: string;
-  targetSlug: string;
-  weight: number;
-  lastReinforcedAt: number;
-}
-
-/**
- * Reinforce the directed pair `source → target`: bump its weight by
- * {@link REINFORCE_INCREMENT} and stamp `last_reinforced_at = now`. **Only call
- * this for a *used* co-activation** — the edge graph encodes which associations
- * actually proved load-bearing for a turn, not which pages merely surfaced
- * together. (The caller decides usedness from the co-activation's `used` flag;
- * this primitive is unconditional so it stays composable.)
- *
- * UPSERT on the `(source, target)` primary key: a new pair starts at the
- * increment; an existing pair accrues on top of its current weight (after the
- * latest decay pass, since decay rewrites weight in place).
- *
- * Best-effort: a failed write must never abort the edge-learning job.
- */
-export function reinforce(
-  database: DrizzleDb,
-  source: string,
-  target: string,
-  now: number,
-): void {
-  try {
-    const raw = getSqliteFrom(database);
-    raw
-      .prepare(
-        `INSERT INTO memory_v3_auto_edges
-           (source_slug, target_slug, weight, last_reinforced_at)
-           VALUES (?, ?, ?, ?)
-         ON CONFLICT(source_slug, target_slug) DO UPDATE SET
-           weight = weight + ?,
-           last_reinforced_at = ?`,
-      )
-      .run(source, target, REINFORCE_INCREMENT, now, REINFORCE_INCREMENT, now);
-  } catch (err) {
-    log.warn(
-      { err, source, target },
-      "failed to reinforce auto-edge; continuing",
-    );
-  }
-}
-
-/**
- * Multiplicatively decay every auto-edge weight toward zero on a half-life
- * schedule: `weight ← weight × exp(-λ × (now − last_reinforced_at))`, with
- * `λ = ln 2 / halfLifeMs`. A pair last reinforced one half-life ago halves; two
- * half-lives ago quarters; and so on. `last_reinforced_at` advances to `now`
- * so successive decay passes don't double-count the same elapsed interval.
- *
- * Pairs whose decayed weight falls below {@link PRUNE_FLOOR} are deleted so the
- * learned graph doesn't accumulate a long tail of effectively-dead edges.
- *
- * Returns the number of rows pruned, for the job's structured log.
- */
-export function decay(
-  database: DrizzleDb,
-  now: number,
-  halfLifeMs: number,
-): number {
-  if (halfLifeMs <= 0) return 0;
-  const lambda = Math.LN2 / halfLifeMs;
-  try {
-    const raw = getSqliteFrom(database);
-    const rows = raw
-      .query(
-        `SELECT source_slug, target_slug, weight, last_reinforced_at
-           FROM memory_v3_auto_edges`,
-      )
-      .all() as Array<{
-      source_slug: string;
-      target_slug: string;
-      weight: number;
-      last_reinforced_at: number;
-    }>;
-    if (rows.length === 0) return 0;
-
-    const update = raw.prepare(
-      `UPDATE memory_v3_auto_edges
-         SET weight = ?, last_reinforced_at = ?
-         WHERE source_slug = ? AND target_slug = ?`,
-    );
-    const prune = raw.prepare(
-      `DELETE FROM memory_v3_auto_edges
-         WHERE source_slug = ? AND target_slug = ?`,
-    );
-
-    let pruned = 0;
-    const apply = raw.transaction(() => {
-      for (const row of rows) {
-        // Future timestamps (clock skew) would amplify rather than decay — clamp
-        // elapsed at 0 so decay only ever shrinks weight.
-        const elapsed = Math.max(0, now - row.last_reinforced_at);
-        const decayed = row.weight * Math.exp(-lambda * elapsed);
-        if (decayed < PRUNE_FLOOR) {
-          prune.run(row.source_slug, row.target_slug);
-          pruned += 1;
-        } else {
-          update.run(decayed, now, row.source_slug, row.target_slug);
-        }
-      }
-    });
-    apply();
-    return pruned;
-  } catch (err) {
-    log.warn({ err }, "failed to decay auto-edges; continuing");
-    return 0;
-  }
-}
-
-/**
- * Project the learned graph to the `extraAdjacency` shape edge expansion
- * consumes: `source → Set<target>` for every pair whose weight is at or above
- * `threshold`. Edge expansion thresholds nothing itself — it merges whatever
- * adjacency it's handed — so this read is where the weight cutoff is applied.
- *
- * Returns an empty map on any read failure so the caller (a best-effort read
- * lane) degrades to "no learned edges" rather than aborting retrieval.
- */
-export function aboveThreshold(
-  database: DrizzleDb,
-  threshold: number,
-): Map<string, Set<string>> {
-  const adjacency = new Map<string, Set<string>>();
-  try {
-    const raw = getSqliteFrom(database);
-    const rows = raw
-      .query(
-        `SELECT source_slug, target_slug
-           FROM memory_v3_auto_edges
-           WHERE weight >= ?
-           ORDER BY source_slug ASC, target_slug ASC`,
-      )
-      .all(threshold) as Array<{ source_slug: string; target_slug: string }>;
-    for (const row of rows) {
-      let targets = adjacency.get(row.source_slug);
-      if (!targets) {
-        targets = new Set<string>();
-        adjacency.set(row.source_slug, targets);
-      }
-      targets.add(row.target_slug);
-    }
-  } catch (err) {
-    log.warn({ err, threshold }, "failed to read auto-edges; continuing");
-  }
-  return adjacency;
-}
-
-/**
- * Read the top-weight auto-edges, heaviest first, capped at `limit`. The
- * edge-learning job surfaces these as advisory promotion candidates for the
- * assistant to ratify into curated `edges:` during consolidation.
- */
-export function topByWeight(database: DrizzleDb, limit: number): AutoEdgeRow[] {
-  if (limit <= 0) return [];
-  try {
-    const raw = getSqliteFrom(database);
-    const rows = raw
-      .query(
-        `SELECT source_slug, target_slug, weight, last_reinforced_at
-           FROM memory_v3_auto_edges
-           ORDER BY weight DESC, source_slug ASC, target_slug ASC
-           LIMIT ?`,
-      )
-      .all(limit) as Array<{
-      source_slug: string;
-      target_slug: string;
-      weight: number;
-      last_reinforced_at: number;
-    }>;
-    return rows.map((r) => ({
-      sourceSlug: r.source_slug,
-      targetSlug: r.target_slug,
-      weight: r.weight,
-      lastReinforcedAt: r.last_reinforced_at,
-    }));
-  } catch (err) {
-    log.warn({ err, limit }, "failed to read top auto-edges; continuing");
-    return [];
-  }
-}

package/src/memory/v3/coactivation-store.ts

@@ -1,124 +0,0 @@
-/**
- * Memory v3 — co-activation store.
- *
- * Best-effort read/write helpers over `memory_v3_coactivation` (migration
- * 262). Each row is a pass-1 → pass-N co-activation pair observed during a
- * single v3 retrieval loop: a `target_slug` first surfaced on a later descent
- * pass was co-selected alongside a `source_slug` that surfaced on pass 1,
- * `pass_gap = passOf(target) − passOf(source)`.
- *
- * This is the raw gradient signal — edge-learning reconciles it into curated
- * graph edge weights later. Writes are off the retrieval critical path: a
- * failed insert here must never abort the turn on top of a successful
- * retrieval the caller already depends on.
- */
-
-import { getLogger } from "../../util/logger.js";
-import type { DrizzleDb } from "../db-connection.js";
-import { getSqliteFrom } from "../db-connection.js";
-
-const log = getLogger("memory-v3-coactivation");
-
-/** One co-activation pair to persist. */
-export interface CoactivationRow {
-  conversationId: string;
-  turn: number;
-  sourceSlug: string;
-  targetSlug: string;
-  passGap: number;
-  /** Usefulness flag. 0 at emit time; reconciled later by edge-learning. */
-  used: number;
-  createdAt: number;
-}
-
-/** A persisted co-activation row, as read back from the table. */
-export interface PersistedCoactivationRow {
-  id: number;
-  conversationId: string;
-  turn: number;
-  sourceSlug: string;
-  targetSlug: string;
-  passGap: number;
-  used: number;
-  createdAt: number;
-}
-
-/**
- * Append co-activation rows. Best-effort — a SQLite write must never abort the
- * agent turn on top of a successful retrieval the rest of the caller depends
- * on. Mirrors {@link recordInjectionEvents}.
- */
-export function recordCoactivations(
-  database: DrizzleDb,
-  rows: readonly CoactivationRow[],
-): void {
-  if (rows.length === 0) return;
-  try {
-    const raw = getSqliteFrom(database);
-    const insert = raw.prepare(
-      `INSERT INTO memory_v3_coactivation
-        (conversation_id, turn, source_slug, target_slug, pass_gap, used, created_at)
-        VALUES (?, ?, ?, ?, ?, ?, ?)`,
-    );
-    const append = raw.transaction((items: readonly CoactivationRow[]) => {
-      for (const r of items) {
-        insert.run(
-          r.conversationId,
-          r.turn,
-          r.sourceSlug,
-          r.targetSlug,
-          r.passGap,
-          r.used,
-          r.createdAt,
-        );
-      }
-    });
-    append(rows);
-  } catch (err) {
-    log.warn(
-      { err, rowCount: rows.length },
-      "failed to record co-activations; continuing",
-    );
-  }
-}
-
-/**
- * Read co-activation rows, oldest first. When `since` is provided, only rows
- * with `created_at >= since` are returned.
- */
-export function readCoactivations(
-  database: DrizzleDb,
-  since?: number,
-): PersistedCoactivationRow[] {
-  const raw = getSqliteFrom(database);
-  const where = since !== undefined ? `WHERE created_at >= ?` : ``;
-  const params = since !== undefined ? [since] : [];
-  const rows = raw
-    .query(
-      `SELECT id, conversation_id, turn, source_slug, target_slug,
-              pass_gap, used, created_at
-        FROM memory_v3_coactivation
-        ${where}
-        ORDER BY created_at ASC, id ASC`,
-    )
-    .all(...params) as Array<{
-    id: number;
-    conversation_id: string;
-    turn: number;
-    source_slug: string;
-    target_slug: string;
-    pass_gap: number;
-    used: number;
-    created_at: number;
-  }>;
-  return rows.map((r) => ({
-    id: r.id,
-    conversationId: r.conversation_id,
-    turn: r.turn,
-    sourceSlug: r.source_slug,
-    targetSlug: r.target_slug,
-    passGap: r.pass_gap,
-    used: r.used,
-    createdAt: r.created_at,
-  }));
-}