@vellumai/assistant 0.11.3 → 0.11.4-dev.202608190019.b94dbf2

package/src/plugins/defaults/memory/src/memory-v3-routes.ts

@@ -19,6 +19,7 @@ import { z } from "zod";
 
 import { getConfig } from "../../../../config/loader.js";
 import type { AssistantConfig } from "../../../../config/types.js";
+import { getTelemetrySqlite } from "../../../../persistence/db-connection.js";
 import {
   ACTOR_PRINCIPALS,
   type RoutePolicy,
@@ -26,6 +27,7 @@ import {
 import type { RouteDefinition } from "../../../../runtime/routes/types.js";
 import { getLogger } from "../logging.js";
 import { backfillAllSections } from "../v3/maintain-job.js";
+import { MEMORY_V3_INJECTION_GATE_CHECK_NAME } from "../v3/orchestrate.js";
 import { invalidateLanes } from "../v3/shadow-plugin.js";
 
 const log = getLogger("memory-v3-routes");
@@ -101,6 +103,211 @@ const WRITE_POLICY: RoutePolicy = {
   allowedPrincipalTypes: ACTOR_PRINCIPALS,
 };
 
+// ---------------------------------------------------------------------------
+// gate-stats
+// ---------------------------------------------------------------------------
+
+const DEFAULT_LOOKBACK_DAYS = 30;
+const MAX_LOOKBACK_DAYS = 90;
+const MS_PER_DAY = 24 * 60 * 60 * 1000;
+
+/** Corpus size buckets for gate-stats aggregation (in concept pages). */
+const CORPUS_BUCKETS = [
+  { label: "0–9", min: 0, max: 9 },
+  { label: "10–49", min: 10, max: 49 },
+  { label: "50–199", min: 50, max: 199 },
+  { label: "200+", min: 200, max: Infinity },
+] as const;
+
+type BucketLabel = (typeof CORPUS_BUCKETS)[number]["label"];
+
+interface BucketAcc {
+  total: number;
+  scored: number;
+  passed: number;
+  /** Passes that came from scored runs — the numerator for scoredPassRate. */
+  scoredPassed: number;
+  reasons: Record<string, number>;
+}
+
+function corpusBucketLabel(pageCount: number): BucketLabel {
+  for (const b of CORPUS_BUCKETS) {
+    if (pageCount <= b.max) {
+      return b.label;
+    }
+  }
+  return "200+";
+}
+
+interface GatePayloadDetail {
+  pass?: boolean;
+  reason?: string;
+  scored?: boolean;
+  real_concept_page_count?: number;
+}
+
+const GateStatsBucketSchema = z.object({
+  pageCountRange: z
+    .string()
+    .describe("Concept page count range (e.g. '10–49')"),
+  total: z
+    .number()
+    .describe("All gate runs in this bucket (scored + pass-open)"),
+  scored: z
+    .number()
+    .describe("Runs that weighed scores (dense lane available)"),
+  passed: z.number().describe("Runs that passed the gate"),
+  scoredPassRate: z
+    .number()
+    .nullable()
+    .describe(
+      "scoredPasses / scored (contested decisions only); null when no scored runs in bucket",
+    ),
+  reasons: z
+    .record(z.string(), z.number())
+    .describe("Gate reason code → run count (dense_pass, fail_no_signal, …)"),
+});
+
+export const GateStatsResponseSchema = z.object({
+  lookbackDays: z.number().describe("Lookback window applied"),
+  totalRuns: z.number().describe("Total gate runs found in the window"),
+  buckets: z
+    .array(GateStatsBucketSchema)
+    .describe("Stats by concept page count range, lean→large"),
+  unknownPageCount: z
+    .object({
+      total: z.number(),
+      passed: z.number(),
+      reasons: z.record(z.string(), z.number()),
+    })
+    .describe("Runs with no real_concept_page_count in telemetry detail"),
+});
+
+export type GateStatsResponse = z.infer<typeof GateStatsResponseSchema>;
+
+/**
+ * Aggregate memory v3 injection gate runs from the telemetry outbox for the
+ * given lookback window. Groups runs by concept page count bucket and computes
+ * pass rates and reason distributions per bucket.
+ *
+ * Pass-open shortcuts (dense disabled/unavailable, gate threw) are included in
+ * `total` but not in `scored`, so `scoredPassRate` reflects only contested gate
+ * decisions — the signal that actually reveals whether the gate thresholds are
+ * calibrated correctly.
+ */
+export function handleMemoryV3GateStats(
+  lookbackDays: number = DEFAULT_LOOKBACK_DAYS,
+  db: ReturnType<typeof getTelemetrySqlite> = getTelemetrySqlite(),
+): GateStatsResponse {
+  const days = Math.min(
+    Math.max(1, Math.floor(lookbackDays)),
+    MAX_LOOKBACK_DAYS,
+  );
+  const since = Date.now() - days * MS_PER_DAY;
+
+  const emptyBuckets = (): GateStatsResponse["buckets"] =>
+    CORPUS_BUCKETS.map((b) => ({
+      pageCountRange: b.label,
+      total: 0,
+      scored: 0,
+      passed: 0,
+      scoredPassRate: null,
+      reasons: {},
+    }));
+  if (!db) {
+    return {
+      lookbackDays: days,
+      totalRuns: 0,
+      buckets: emptyBuckets(),
+      unknownPageCount: { total: 0, passed: 0, reasons: {} },
+    };
+  }
+
+  const rows = db
+    .query(
+      /*sql*/ `
+      SELECT payload FROM telemetry_events
+      WHERE name = 'watchdog'
+        AND created_at >= ?
+        AND json_extract(payload, '$.check_name') = ?
+    `,
+    )
+    .all(since, MEMORY_V3_INJECTION_GATE_CHECK_NAME) as Array<{
+    payload: string;
+  }>;
+
+  const bucketMap = new Map<BucketLabel, BucketAcc>(
+    CORPUS_BUCKETS.map((b) => [
+      b.label,
+      { total: 0, scored: 0, passed: 0, scoredPassed: 0, reasons: {} },
+    ]),
+  );
+  const unknown: BucketAcc = {
+    total: 0,
+    scored: 0,
+    passed: 0,
+    scoredPassed: 0,
+    reasons: {},
+  };
+
+  for (const row of rows) {
+    let detail: GatePayloadDetail;
+    try {
+      const parsed = JSON.parse(row.payload) as { detail?: GatePayloadDetail };
+      detail = parsed.detail ?? {};
+    } catch {
+      continue;
+    }
+
+    const pageCount =
+      typeof detail.real_concept_page_count === "number"
+        ? detail.real_concept_page_count
+        : null;
+
+    const acc =
+      pageCount !== null
+        ? (bucketMap.get(corpusBucketLabel(pageCount)) ?? unknown)
+        : unknown;
+
+    acc.total += 1;
+    if (detail.scored === true) {
+      acc.scored += 1;
+      if (detail.pass === true) {
+        acc.scoredPassed += 1;
+      }
+    }
+    if (detail.pass === true) {
+      acc.passed += 1;
+    }
+    const reason =
+      typeof detail.reason === "string" ? detail.reason : "unknown";
+    acc.reasons[reason] = (acc.reasons[reason] ?? 0) + 1;
+  }
+
+  const buckets = CORPUS_BUCKETS.map((b) => {
+    const acc = bucketMap.get(b.label)!;
+    return {
+      pageCountRange: b.label,
+      total: acc.total,
+      scored: acc.scored,
+      passed: acc.passed,
+      scoredPassRate: acc.scored > 0 ? acc.scoredPassed / acc.scored : null,
+      reasons: acc.reasons,
+    };
+  });
+
+  return {
+    lookbackDays: days,
+    totalRuns: rows.length,
+    buckets,
+    unknownPageCount: {
+      total: unknown.total,
+      passed: unknown.passed,
+      reasons: unknown.reasons,
+    },
+  };
+}
+
 export const ROUTES: RouteDefinition[] = [
   {
     operationId: "memory_v3_rebuild_index",

package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts

@@ -197,6 +197,7 @@ const { memoryV2ConsolidateJob, CONSOLIDATION_FAILURE_CHECKPOINT_KEY } =
 const { CUTOFF_PLACEHOLDER, CONSOLIDATION_PROMPT } =
   await import("../prompts/consolidation.js");
 const { invalidatePageIndex } = await import("../page-index.js");
+const { formatRememberEntry } = await import("../../buffer-format.js");
 
 // The handler only reads `config.memory.enabled`, `config.memory.v2.enabled`,
 // `config.memory.v2.consolidation_prompt_path`, and
@@ -437,6 +438,38 @@ describe("memoryV2ConsolidateJob — chunked cutoff (consolidation_max_entries_p
     expect(outcome.cutoff).not.toBe("[meeting-notes]");
   });
 
+  test("a fact whose body holds a canonical entry line counts once and never supplies the cutoff", async () => {
+    // Written through the real writer, so this covers the round trip rather
+    // than a hand-built buffer: the body line carries the exact shape an entry
+    // opening has. Counting it would both inflate the per-run budget and let
+    // the cutoff be drawn from inside a fact, deferring entries against a
+    // timestamp the user never filed anything at.
+    writeFileSync(
+      bufferPath(),
+      formatRememberEntry(
+        "Alice's rollout plan:\n- [Apr 28, 9:00 AM] cutover",
+        new Date(2026, 3, 27, 9, 0),
+      ) +
+        formatRememberEntry(
+          "Bob takes his coffee black.",
+          new Date(2026, 3, 27, 9, 1),
+        ),
+    );
+
+    const outcome = await memoryV2ConsolidateJob(
+      makeJob(),
+      configWithMaxEntries(2),
+    );
+
+    expect(outcome.kind).toBe("invoked");
+    if (outcome.kind !== "invoked") {
+      throw new Error("unreachable");
+    }
+    // Two facts, cap 2 → no chunking. Three would chunk and defer one.
+    expect(outcome.deferredEntries).toBe(0);
+    expect(outcome.cutoff).not.toBe("Apr 28, 9:00 AM");
+  });
+
   test("multi-line entries under the cap → full-buffer cutoff even when raw line count exceeds it", async () => {
     writeFileSync(
       bufferPath(),
@@ -582,6 +615,32 @@ describe("memoryV2ConsolidateJob — non-empty buffer", () => {
     }
   });
 
+  test("threads page-index dangling links into the prompt's repair step", async () => {
+    const conceptsDir = join(tmpWorkspace, "memory", "concepts");
+    const sourcePath = join(conceptsDir, "atl-1290.md");
+    // A `links:` entry naming a page that does not exist (the LUM-3316 shape).
+    writeFileSync(
+      sourcePath,
+      "---\nlinks:\n  - atl-1291\n---\n# ATL-1290\n\nSee [[atl-1291]].\n",
+    );
+    invalidatePageIndex();
+    try {
+      const result = await memoryV2ConsolidateJob(makeJob(), CONFIG);
+      expect(result.kind).toBe("invoked");
+      const prompt = runnerLastArgs?.prompt as string;
+      expect(prompt).toContain("resolve dangling links");
+      expect(prompt).toContain(
+        "- `memory/concepts/atl-1290.md` → `atl-1291` (frontmatter `links:` entry)",
+      );
+      expect(prompt).toContain(
+        "- `memory/concepts/atl-1290.md` → `atl-1291` (inline `[[wikilink]]` in the body)",
+      );
+    } finally {
+      rmSync(sourcePath, { force: true });
+      invalidatePageIndex();
+    }
+  });
+
   test("dispatches with skipPromptIndexing so the kickoff prompt is not indexed", async () => {
     // The prompt is a static instruction manual — indexing it would write
     // near-identical memory segments, embeddings, and a lexical entry on
@@ -847,6 +906,94 @@ describe("memoryV2ConsolidateJob — progress check and follow-up coalescing", (
     expect(enqueuedJobs).toHaveLength(0);
   });
 
+  test("a run that leaves dangling links reports the count and still enqueues follow-ups", async () => {
+    const conceptsDir = join(tmpWorkspace, "memory", "concepts");
+    runnerImpl = async () => {
+      // The agent files the buffer into a page whose links: names a page it
+      // never spawned. Writing through the file tools would invalidate the
+      // index via the tool hook; the stub does so by hand.
+      writeFileSync(
+        join(conceptsDir, "atl-1290.md"),
+        "---\nlinks:\n  - atl-1291\n---\n# ATL-1290\n",
+      );
+      writeFileSync(bufferPath(), "");
+      invalidatePageIndex();
+      return { conversationId: "conv-1", ok: true };
+    };
+    try {
+      const result = await memoryV2ConsolidateJob(makeJob(), CONFIG);
+
+      expect(result.kind).toBe("invoked");
+      if (result.kind !== "invoked") {
+        throw new Error("unreachable");
+      }
+      expect(result.noProgress).toBe(false);
+      expect(result.danglingLinks).toBe(1);
+      // Surfaced, not gating: the page that was written still gets indexed.
+      expect(result.followUpJobIds).toEqual(["job-1"]);
+      expect(enqueuedJobs.map((j) => j.type)).toEqual(["memory_v2_reembed"]);
+    } finally {
+      rmSync(join(conceptsDir, "atl-1290.md"), { force: true });
+      invalidatePageIndex();
+    }
+  });
+
+  test("a run that spawns the missing target clears the pre-run dangling link", async () => {
+    const conceptsDir = join(tmpWorkspace, "memory", "concepts");
+    writeFileSync(
+      join(conceptsDir, "atl-1290.md"),
+      "---\nlinks:\n  - atl-1291\n---\n# ATL-1290\n",
+    );
+    invalidatePageIndex();
+    runnerImpl = async () => {
+      writeFileSync(join(conceptsDir, "atl-1291.md"), "# ATL-1291\n");
+      writeFileSync(bufferPath(), "");
+      invalidatePageIndex();
+      return { conversationId: "conv-1", ok: true };
+    };
+    try {
+      const result = await memoryV2ConsolidateJob(makeJob(), CONFIG);
+
+      expect(result.kind).toBe("invoked");
+      if (result.kind !== "invoked") {
+        throw new Error("unreachable");
+      }
+      // The pre-run index fed the repair step...
+      expect(runnerLastArgs?.prompt as string).toContain(
+        "resolve dangling links",
+      );
+      // ...and the post-run count reflects the repair.
+      expect(result.danglingLinks).toBe(0);
+    } finally {
+      rmSync(join(conceptsDir, "atl-1290.md"), { force: true });
+      rmSync(join(conceptsDir, "atl-1291.md"), { force: true });
+      invalidatePageIndex();
+    }
+  });
+
+  test("a no-progress run still reports its dangling-link count", async () => {
+    const conceptsDir = join(tmpWorkspace, "memory", "concepts");
+    writeFileSync(
+      join(conceptsDir, "atl-1290.md"),
+      "---\nlinks:\n  - atl-1291\n---\n# ATL-1290\n",
+    );
+    invalidatePageIndex();
+    runnerImpl = async () => ({ conversationId: "conv-1", ok: true });
+    try {
+      const result = await memoryV2ConsolidateJob(makeJob(), CONFIG);
+
+      expect(result.kind).toBe("invoked");
+      if (result.kind !== "invoked") {
+        throw new Error("unreachable");
+      }
+      expect(result.noProgress).toBe(true);
+      expect(result.danglingLinks).toBe(1);
+    } finally {
+      rmSync(join(conceptsDir, "atl-1290.md"), { force: true });
+      invalidatePageIndex();
+    }
+  });
+
   test("a pending reembed suppresses its duplicate; v3 maintain still enqueues", async () => {
     v3FlagOn = true;
     pendingJobTypes.add("memory_v2_reembed");

package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts

@@ -26,6 +26,7 @@ import {
   CONSOLIDATION_PROMPT_V3,
   CORE_PAGES_CONSOLIDATION_SECTION,
   renderConsolidationPrompt,
+  renderDanglingLinksSection,
   renderParseFailuresSection,
 } from "../prompts/consolidation.js";
 
@@ -59,6 +60,13 @@ const GATED_SECTIONS: Array<{
     ]),
     marker: "repair unreadable pages",
   },
+  {
+    name: "dangling-link repair step (gate: index-reported dangling links)",
+    section: renderDanglingLinksSection([
+      { from: "example-page", to: "example-target", kind: "links" },
+    ]),
+    marker: "resolve dangling links",
+  },
 ];
 
 /** All `ConsolidationPromptOptions` gates forced off — the v2-only rendering. */

package/src/plugins/defaults/memory/substrate/__tests__/edge-index.test.ts

@@ -15,7 +15,6 @@ import {
   getReachable,
   invalidateEdgeIndex,
   totalEdgeCount,
-  validateEdgeTargets,
 } from "../edge-index.js";
 import { deletePage, writePage } from "../page-store.js";
 import type { ConceptPage } from "../types.js";
@@ -228,35 +227,6 @@ describe("getReachable", () => {
   });
 });
 
-// ---------------------------------------------------------------------------
-// validateEdgeTargets
-// ---------------------------------------------------------------------------
-
-describe("validateEdgeTargets", () => {
-  test("ok=true when every outgoing target has a known slug", async () => {
-    await writePage(workspaceDir, makePage("alice", ["bob"]));
-    await writePage(workspaceDir, makePage("bob"));
-
-    const idx = await getEdgeIndex(workspaceDir);
-    const result = validateEdgeTargets(idx, new Set(["alice", "bob"]));
-    expect(result).toEqual({ ok: true, missing: [] });
-  });
-
-  test("flags outgoing targets that don't correspond to a known slug", async () => {
-    await writePage(workspaceDir, makePage("alice", ["ghost", "bob"]));
-    await writePage(workspaceDir, makePage("bob", ["phantom"]));
-
-    const idx = await getEdgeIndex(workspaceDir);
-    const result = validateEdgeTargets(idx, new Set(["alice", "bob"]));
-
-    expect(result.ok).toBe(false);
-    expect(result.missing).toEqual([
-      { from: "alice", to: "ghost" },
-      { from: "bob", to: "phantom" },
-    ]);
-  });
-});
-
 // ---------------------------------------------------------------------------
 // totalEdgeCount
 // ---------------------------------------------------------------------------

package/src/plugins/defaults/memory/substrate/__tests__/ingest.test.ts

@@ -12,7 +12,8 @@
  *   - Batch over the cap: RangeError.
  *   - In-batch duplicate slug: later occurrence `invalid`.
  *   - Non-blocking warnings: missing `source`, unparseable `origin_date`,
- *     frontmatter `slug` differing from the storage slug.
+ *     frontmatter `slug` differing from the storage slug, link targets
+ *     neither on disk nor in the batch.
  *   - Under-lock collision snapshot: a page committed between validation and
  *     the write phase is skipped, and `listPages` runs after `tryAcquireLock`.
  *   - Partial write failure: the reindex fan-out still runs for committed
@@ -363,6 +364,77 @@ describe("ingestPages", () => {
     expect(await listPages(workspace)).toEqual(["people/alice", "people/bob"]);
   });
 
+  test("link targets neither on disk nor in the batch warn without blocking the write", async () => {
+    // `people/bob` already exists on disk; `projects/garden` arrives in the
+    // same batch; `atl-1291` and `ghost` exist nowhere.
+    await ingestPages(workspace, [
+      { slug: "people/bob", content: validPage("Bob.") },
+    ]);
+    enqueuedJobs.length = 0;
+
+    const summary = await ingestPages(workspace, [
+      {
+        slug: "people/alice",
+        content:
+          "---\nsource: import:chatgpt\norigin_date: 2025-03-14\n" +
+          "links:\n  - people/bob\n  - projects/garden\n  - atl-1291\n" +
+          "edges: [ghost]\n---\n" +
+          "Alice mentions [[projects/garden]] and [[people/bob]] and [[skills/deploy]].\n",
+      },
+      { slug: "projects/garden", content: validPage("Garden.") },
+    ]);
+
+    expect(summary.written).toBe(2);
+    expect(summary.invalid).toBe(0);
+    const [alice, garden] = summary.results;
+    expect(alice.action).toBe("written");
+    expect(alice.warnings).toHaveLength(1);
+    // Every unresolved target on one line, sorted; targets that resolve on
+    // disk, in the batch, or under a synthetic prefix stay silent.
+    expect(alice.warnings[0]).toContain(
+      "link targets not on disk or in this batch: atl-1291, ghost",
+    );
+    expect(alice.warnings[0]).not.toContain("people/bob");
+    expect(alice.warnings[0]).not.toContain("projects/garden");
+    expect(alice.warnings[0]).not.toContain("skills/deploy");
+    expect(garden.warnings).toEqual([]);
+    // Written regardless, and reindexed.
+    expect(await readPage(workspace, "people/alice")).not.toBeNull();
+    expect(enqueuedJobs.map((j) => j.type).sort()).toEqual([
+      "memory_v2_reembed",
+      "memory_v3_maintain",
+    ]);
+  });
+
+  test("dangling-link warnings are reported on dry-run and skipped for pages the batch will not write", async () => {
+    await ingestPages(workspace, [
+      { slug: "people/bob", content: validPage("Bob.") },
+    ]);
+
+    const summary = await ingestPages(
+      workspace,
+      [
+        {
+          slug: "people/alice",
+          content: validPage("Alice mentions [[nowhere]]."),
+        },
+        // Exists on disk and overwrite is off, so this content is never
+        // written; its dangling link is not this batch's business.
+        {
+          slug: "people/bob",
+          content: validPage("Bob mentions [[also-nowhere]]."),
+        },
+      ],
+      { dryRun: true },
+    );
+
+    expect(summary.results[0].action).toBe("written");
+    expect(summary.results[0].warnings.join(" ")).toContain("nowhere");
+    expect(summary.results[1].action).toBe("skipped_exists");
+    expect(summary.results[1].warnings).toEqual([]);
+    expect(await readPage(workspace, "people/alice")).toBeNull();
+  });
+
   test("a pending reindex job of the same type suppresses its duplicate enqueue", async () => {
     pendingJobTypes = new Set(["memory_v2_reembed"]);
     await ingestPages(workspace, [

package/src/plugins/defaults/memory/substrate/__tests__/page-index.test.ts

@@ -56,6 +56,7 @@ const {
 } = await import("../page-index.js");
 const { writePage } = await import("../page-store.js");
 const { invalidateEdgeIndex } = await import("../edge-index.js");
+const { LINK_SEPARATOR } = await import("../page-links.js");
 
 let workspaceDir: string;
 
@@ -272,6 +273,42 @@ describe("getPageIndex", () => {
     const alice = idx.bySlug.get("alice")!;
     const bob = idx.bySlug.get("bob")!;
     expect(alice.edges).toEqual([bob.id]);
+    // Dropped from the numeric edges, reported on the index.
+    expect(idx.danglingLinks).toEqual([
+      { from: "alice", to: "ghost", kind: "edges" },
+    ]);
+  });
+
+  test("danglingLinks covers links:, body wikilinks, and edges:, resolved against pages and synthetic rows", async () => {
+    skillState.entries = [{ id: "browser", content: "Drive a browser." }];
+    const alice = makePage("alice", {
+      summary: "A",
+      edges: ["bob"],
+      body: "Alice uses [[skills/browser]] and [[bob|Bob]] but also [[nobody#sec]].",
+    });
+    alice.frontmatter.links = [
+      `bob${LINK_SEPARATOR}peer`,
+      `atl-1291${LINK_SEPARATOR}ticket`,
+    ];
+    await writePage(workspaceDir, alice);
+    await writePage(workspaceDir, makePage("bob", { summary: "B" }));
+
+    const idx = await getPageIndex(workspaceDir);
+    expect(idx.danglingLinks).toEqual([
+      { from: "alice", to: "atl-1291", kind: "links" },
+      { from: "alice", to: "nobody", kind: "wikilink" },
+    ]);
+  });
+
+  test("danglingLinks is empty when every reference resolves", async () => {
+    await writePage(
+      workspaceDir,
+      makePage("alice", { summary: "A", edges: ["bob"], body: "See [[bob]]." }),
+    );
+    await writePage(workspaceDir, makePage("bob", { summary: "B" }));
+
+    const idx = await getPageIndex(workspaceDir);
+    expect(idx.danglingLinks).toEqual([]);
   });
 
   test("exposes frontmatter leaves in the index entry", async () => {