@vellumai/assistant 0.11.3 → 0.11.4-dev.202608190019.b94dbf2

package/src/plugins/defaults/memory/substrate/consolidation-job.ts

@@ -45,7 +45,11 @@
  *   6. Verify the run drained the buffer. `runResult.ok` only means the
  *      background run completed — the trim itself is delegated to the agent.
  *      A run that completes without shrinking the buffer is reported as
- *      `invoked` with `noProgress: true` and enqueues no follow-ups.
+ *      `invoked` with `noProgress: true` and enqueues no follow-ups. The
+ *      post-run page index is also read for `danglingLinks` (structural
+ *      references with no target page): reported on the outcome and in the
+ *      log, and fed into the NEXT pass's prompt as a repair step like
+ *      `parseFailures`. They never gate follow-ups.
  *   7. On progress, enqueue `memory_v2_reembed` (re-index any pages the agent
  *      touched). Tracking touched pages via mtime would be more precise but
  *      is fragile across filesystems; the embedder's content-hash cache makes
@@ -91,7 +95,10 @@ import {
   type MemoryJobType,
 } from "../../../../persistence/jobs-store.js";
 import { runBackgroundJob } from "../../../../runtime/background-job-runner.js";
-import { formatBufferTimestamp } from "../graph/tool-handlers.js";
+import {
+  formatBufferTimestamp,
+  matchBufferEntryStart,
+} from "../buffer-format.js";
 import { getLogger } from "../logging.js";
 import { getWorkspaceDir } from "../paths.js";
 import {
@@ -101,6 +108,7 @@ import {
   tryAcquireLock,
 } from "./consolidation-lock.js";
 import { getPageIndex, type PageParseFailure } from "./page-index.js";
+import type { DanglingLink } from "./page-links.js";
 import { resolveConsolidationPrompt } from "./prompts/consolidation.js";
 import { resolveSubstrateTuning } from "./tuning.js";
 
@@ -293,8 +301,17 @@ export type ConsolidationOutcome =
        * follow-ups were enqueued.
        */
       noProgress: boolean;
+      /**
+       * Structural references (`links:`, `[[wikilinks]]`, `edges:`) in the
+       * post-run corpus whose target page does not exist; `null` when the
+       * post-run index could not be read.
+       */
+      danglingLinks: number | null;
     };
 
+/** Dangling links named in the post-run warn line before the count takes over. */
+const MAX_LOGGED_DANGLING_LINKS = 20;
+
 export async function memoryV2ConsolidateJob(
   _job: MemoryJob,
   config: AssistantConfig,
@@ -416,16 +433,20 @@ export async function memoryV2ConsolidateJob(
     // v2-only install must keep producing `summary:`-bearing fragment pages.
     const memoryV3Live = isMemoryV3Live(config);
     // Pages the index build dropped (malformed frontmatter) or degraded
-    // (unterminated fence) — rendered into the prompt's repair step so the
-    // agent fixes them this pass. Best-effort: prompt assembly must never
-    // fail because the index build did.
+    // (unterminated fence), and structural references with no target page,
+    // rendered into the prompt's repair steps so the agent fixes them this
+    // pass. Best-effort: prompt assembly must never fail because the index
+    // build did.
     let parseFailures: PageParseFailure[] = [];
+    let danglingLinks: DanglingLink[] = [];
     try {
-      parseFailures = (await getPageIndex(getWorkspaceDir())).parseFailures;
+      const index = await getPageIndex(getWorkspaceDir());
+      parseFailures = index.parseFailures;
+      danglingLinks = index.danglingLinks;
     } catch (err) {
       log.warn(
         { err },
-        "consolidation: page-index read failed; omitting the repair section",
+        "consolidation: page-index read failed; omitting the repair sections",
       );
     }
     const prompt = resolveConsolidationPrompt(
@@ -435,6 +456,7 @@ export async function memoryV2ConsolidateJob(
         includeCorePagesSection: memoryV3Live,
         articleShape: memoryV3Live ? "v3" : "v2",
         parseFailures,
+        danglingLinks,
       },
     );
 
@@ -493,6 +515,13 @@ export async function memoryV2ConsolidateJob(
     const bufferLinesAfter = countBufferLines(bufferPath);
     const noProgress = bufferLinesAfter >= bufferLinesBefore;
 
+    // The agent's file-tool writes invalidate the page index, so this read
+    // sees the post-run corpus.
+    const danglingAfter = await readDanglingLinks(
+      runResult.conversationId,
+      danglingLinks.length,
+    );
+
     // Failure-state bookkeeping. A skipped run (`skipReason`) never invoked
     // the agent, so it neither clears nor records. A completed run that made
     // no progress behaves like a failure for scheduling — the size trigger
@@ -523,6 +552,7 @@ export async function memoryV2ConsolidateJob(
         deferredEntries,
         followUpJobIds: [],
         noProgress: true,
+        danglingLinks: danglingAfter,
       };
     }
 
@@ -574,12 +604,47 @@ export async function memoryV2ConsolidateJob(
       deferredEntries,
       followUpJobIds,
       noProgress: false,
+      danglingLinks: danglingAfter,
     };
   } finally {
     releaseLock(lockPath);
   }
 }
 
+/**
+ * Post-run dangling-link count; warns (with a capped sample and the pre-run
+ * count) when any remain. `null` when the index cannot be read.
+ */
+async function readDanglingLinks(
+  conversationId: string,
+  danglingBefore: number,
+): Promise<number | null> {
+  let dangling: DanglingLink[];
+  try {
+    dangling = (await getPageIndex(getWorkspaceDir())).danglingLinks;
+  } catch (err) {
+    log.warn(
+      { err, conversationId },
+      "consolidation: post-run page-index read failed; dangling links unknown",
+    );
+    return null;
+  }
+  if (dangling.length > 0) {
+    log.warn(
+      {
+        conversationId,
+        danglingBefore,
+        danglingAfter: dangling.length,
+        sample: dangling
+          .slice(0, MAX_LOGGED_DANGLING_LINKS)
+          .map((d) => `${d.from} -> ${d.to} (${d.kind})`),
+      },
+      "consolidation left structural links whose target page does not exist; the next pass renders them as a repair step",
+    );
+  }
+  return dangling.length;
+}
+
 /**
  * Read `memory/buffer.md`. Missing file → empty string so the skip-on-empty
  * branch doesn't have to distinguish "no file" from "blank file".
@@ -597,32 +662,36 @@ function readBufferContent(bufferPath: string): string {
 
 /**
  * Extract the bracketed timestamp from a `buffer.md` entry line
- * (`- [Mon D, h:mm AM/PM] …`, see `formatBufferEntry`). Returned verbatim so
- * it can serve directly as a consolidation cutoff — both sides of the agent's
- * "timestamp ≥ cutoff" comparison then share the exact `formatBufferTimestamp`
- * shape.
+ * (`- [Mon D, h:mm AM/PM] …`, see {@link formatRememberEntry}). Returned
+ * verbatim so it can serve directly as a consolidation cutoff: both sides of
+ * the agent's "timestamp >= cutoff" comparison then share the exact
+ * {@link formatBufferTimestamp} shape.
  *
- * The bracket contents must match that shape exactly: a remembered fact's
- * continuation lines can themselves start with `- [` (markdown checklists
- * `- [ ] …`, wikilink bullets `- [[…]]`), and counting those as entries
- * would inflate the per-run budget — or worse, hand the agent a garbage
- * cutoff like a blank string. Returns `null` for anything that isn't a real
- * timestamped entry start.
+ * Recognition is delegated to the shared matcher, so a remembered fact's
+ * continuation lines never register as entries. That matters here beyond
+ * tidiness: counting a fact's `- [ ] …` checklist lines or an indented
+ * entry-shaped body line as entries would inflate the per-run budget, or hand
+ * the agent a cutoff drawn from the middle of a fact.
  */
 function extractBufferEntryTimestamp(line: string): string | null {
-  const match = /^\s*-\s*\[([A-Z][a-z]{2} \d{1,2}, \d{1,2}:\d{2} [AP]M)\]/.exec(
-    line,
-  );
-  return match ? match[1] : null;
+  return matchBufferEntryStart(line)?.timestamp ?? null;
 }
 
 /**
  * Count non-empty lines in `memory/buffer.md`. Used by the scheduler to
  * implement the size-based consolidation trigger. Missing file → 0.
  *
- * Each entry is one line (`- [Mon D, h:mm AM/PM] …\n`), so non-empty-line
- * count == entry count for a well-formed buffer; blank lines and trailing
- * newlines don't inflate the count.
+ * Lines, deliberately, not entries. A multiline `remember()` fact is one
+ * entry spread over several lines, so the two counts diverge and each answers
+ * a different question. This trigger and the injected-Buffer cap that reuses
+ * it (`capBufferSection` in `static-context.ts`) both care about how much
+ * context the buffer costs, which scales with lines. The per-run budget
+ * `consolidation_max_entries_per_run` cares about how many facts the agent
+ * must file, so it counts entry-start lines instead.
+ *
+ * Do not "fix" this to count entries: a single 200-line fact is a context
+ * problem the trigger should fire on, even though it is one entry. Blank
+ * lines and trailing newlines don't inflate the count.
  */
 export function countBufferLines(bufferPath: string): number {
   return countNonEmptyLines(readBufferContent(bufferPath));

package/src/plugins/defaults/memory/substrate/edge-index.ts

@@ -166,37 +166,6 @@ export function getReachable(
   return result;
 }
 
-export interface EdgeValidationResult {
-  ok: boolean;
-  /**
-   * Outgoing-edge targets that don't correspond to any known slug. Sorted by
-   * `(from, to)` for deterministic output.
-   */
-  missing: Array<{ from: string; to: string }>;
-}
-
-/**
- * Validate every outgoing edge target against `knownSlugs`. Returns a sorted
- * list of `(from, to)` pairs whose `to` slug has no corresponding concept
- * page on disk.
- */
-export function validateEdgeTargets(
-  index: EdgeIndex,
-  knownSlugs: ReadonlySet<string>,
-): EdgeValidationResult {
-  const missing: Array<{ from: string; to: string }> = [];
-  const sources = [...index.outgoing.keys()].sort();
-  for (const from of sources) {
-    const targets = [...(index.outgoing.get(from) ?? [])].sort();
-    for (const to of targets) {
-      if (!knownSlugs.has(to)) {
-        missing.push({ from, to });
-      }
-    }
-  }
-  return { ok: missing.length === 0, missing };
-}
-
 /**
  * Total count of directed edges in the index — i.e. the sum of every page's
  * outgoing-edge fanout. Each (from, to) pair counts once.

package/src/plugins/defaults/memory/substrate/ingest.ts

@@ -29,7 +29,9 @@ import {
   tryAcquireLock,
 } from "./consolidation-lock.js";
 import { parseOriginDate } from "./page-index.js";
+import { findDanglingLinks } from "./page-links.js";
 import {
+  isValidSlug,
   listPages,
   parsePageContent,
   validateSlug,
@@ -127,6 +129,14 @@ const REINDEX_JOB_TYPES: readonly MemoryJobType[] = [
  *      consolidation lock, so a page committed by another writer between
  *      validation and the write phase is still skipped rather than
  *      overwritten.
+ *   5. One more non-blocking warning, once the on-disk snapshot exists: a
+ *      `links:`, `[[wikilink]]`, or `edges:` target that is neither on disk
+ *      nor a valid page in this batch. Retrieval drops such a reference, so
+ *      the caller either stages the missing page or turns the reference into
+ *      prose. Warned, not rejected: a page is not worth losing over one link,
+ *      and a multi-batch import legitimately references a later batch.
+ *      Targets under the synthetic capability prefixes are not checked (their
+ *      catalog is not on disk).
  *
  * With `dryRun` the summary is returned without touching disk or the lock;
  * the dry-run collision snapshot is lock-free and therefore advisory.
@@ -250,6 +260,7 @@ export async function ingestPages(
     // concurrent writer can change what a later real run would do.
     const existingSlugs = new Set(await listPages(workspaceDir));
     classifyCollisions(pending, existingSlugs, overwrite);
+    warnDanglingLinks(pending, existingSlugs);
     return summarize(results, dryRun);
   }
 
@@ -264,6 +275,7 @@ export async function ingestPages(
     // writer after validation still classifies as a collision.
     const existingSlugs = new Set(await listPages(workspaceDir));
     const toWrite = classifyCollisions(pending, existingSlugs, overwrite);
+    warnDanglingLinks(pending, existingSlugs);
     const summary = summarize(results, dryRun);
 
     for (const page of toWrite) {
@@ -319,6 +331,53 @@ function classifyCollisions(
   return toWrite;
 }
 
+/**
+ * Append a dangling-link warning to every page about to be written whose
+ * structural references name a slug that is neither on disk nor a valid page
+ * in this batch. Runs after collision classification so pages the batch will
+ * not write (`skipped_exists`) are not warned about.
+ */
+function warnDanglingLinks(
+  pending: readonly PendingPage[],
+  existingSlugs: ReadonlySet<string>,
+): void {
+  const knownSlugs = new Set(existingSlugs);
+  for (const { page } of pending) {
+    knownSlugs.add(page.slug);
+  }
+  const toWrite = pending.filter(({ result }) => result.action === "written");
+  const dangling = findDanglingLinks(
+    toWrite.map(({ page }) => page),
+    knownSlugs,
+    isValidSlug,
+  ).filter(
+    (d) => !RESERVED_SLUG_PREFIXES.some((prefix) => d.to.startsWith(prefix)),
+  );
+  if (dangling.length === 0) {
+    return;
+  }
+  const targetsBySlug = new Map<string, string[]>();
+  for (const { from, to } of dangling) {
+    const targets = targetsBySlug.get(from) ?? [];
+    if (!targets.includes(to)) {
+      targets.push(to);
+    }
+    targetsBySlug.set(from, targets);
+  }
+  for (const { page, result } of toWrite) {
+    const targets = targetsBySlug.get(page.slug);
+    if (targets === undefined) {
+      continue;
+    }
+    result.warnings.push(
+      `link target${targets.length === 1 ? "" : "s"} not on disk or in ` +
+        `this batch: ${targets.join(", ")}; stage the missing page or ` +
+        `turn the reference into prose (retrieval drops a link whose ` +
+        `target page does not exist)`,
+    );
+  }
+}
+
 function summarize(
   results: IngestPageResult[],
   dryRun: boolean,

package/src/plugins/defaults/memory/substrate/page-index.ts

@@ -20,7 +20,14 @@
  */
 
 import { getLogger } from "../logging.js";
-import { getPageMtimeMs, listPages, readPage } from "./page-store.js";
+import { type DanglingLink, findDanglingLinks } from "./page-links.js";
+import {
+  getPageMtimeMs,
+  isValidSlug,
+  listPages,
+  readPage,
+} from "./page-store.js";
+import type { ConceptPage } from "./types.js";
 
 // Dynamic import for `./skill-store.js` happens inside `getPageIndex` so that
 // modules that only need `invalidatePageIndex` (page-store.ts,
@@ -103,8 +110,9 @@ export interface PageParseFailure {
  * ASCII so IDs are deterministic across rebuilds with the same input. The
  * `bySlug` and `byId` maps are convenience lookups; `rendered` is the prompt
  * block consumed by the router. `parseFailures` lists the pages this build
- * dropped or degraded, so consumers (the consolidation prompt) can surface
- * them for repair.
+ * dropped or degraded and `danglingLinks` the structural references whose
+ * target has no entry, so consumers (the consolidation prompt, the validate
+ * route) can surface them for repair.
  */
 export interface PageIndex {
   entries: PageIndexEntry[];
@@ -112,6 +120,8 @@ export interface PageIndex {
   byId: Map<number, PageIndexEntry>;
   rendered: string;
   parseFailures: PageParseFailure[];
+  /** Resolved against every `bySlug` key, synthetic rows included. */
+  danglingLinks: DanglingLink[];
 }
 
 interface CachedIndex {
@@ -187,6 +197,7 @@ export async function getPageIndex(workspaceDir: string): Promise<PageIndex> {
 
   const drafts: DraftEntry[] = [];
   const parseFailures: PageParseFailure[] = [];
+  const linkSources: ConceptPage[] = [];
   for (let i = 0; i < settled.length; i++) {
     const result = settled[i];
     const slug = slugs[i];
@@ -232,6 +243,7 @@ export async function getPageIndex(workspaceDir: string): Promise<PageIndex> {
       modifiedAt: mtimeMs,
       freshAt: resolveFreshAt(page.frontmatter.origin_date, mtimeMs),
     });
+    linkSources.push(page);
   }
 
   for (const entry of skillEntries) {
@@ -276,8 +288,8 @@ export async function getPageIndex(workspaceDir: string): Promise<PageIndex> {
     return entry;
   });
 
-  // Edges whose target slug isn't in the index are dropped silently — the
-  // frontmatter sweep is responsible for surfacing those as warnings.
+  // Edges whose target slug isn't in the index are not resolved here; they are
+  // reported in `danglingLinks` below instead.
   for (let i = 0; i < entries.length; i++) {
     const draft = drafts[i];
     const resolved: number[] = [];
@@ -297,8 +309,21 @@ export async function getPageIndex(workspaceDir: string): Promise<PageIndex> {
     a.slug < b.slug ? -1 : a.slug > b.slug ? 1 : 0,
   );
 
+  const danglingLinks = findDanglingLinks(
+    linkSources,
+    new Set(bySlug.keys()),
+    isValidSlug,
+  );
+
   const rendered = renderIndex(entries);
-  const index: PageIndex = { entries, bySlug, byId, rendered, parseFailures };
+  const index: PageIndex = {
+    entries,
+    bySlug,
+    byId,
+    rendered,
+    parseFailures,
+    danglingLinks,
+  };
   cache = { workspaceDir, index };
   return index;
 }
@@ -525,9 +550,10 @@ function buildLocalPageIndex(
     bySlug: localBySlug,
     byId: localById,
     rendered: renderIndex(localEntries),
-    // Parse failures are a property of the workspace build, not of a carved
-    // batch — the full index is the reporting surface.
+    // Parse failures and dangling links are properties of the workspace
+    // build, not of a carved batch: the full index is the reporting surface.
     parseFailures: [],
+    danglingLinks: [],
   };
 }
 
@@ -626,5 +652,6 @@ function emptyPageIndex(): PageIndex {
     byId: new Map(),
     rendered: "",
     parseFailures: [],
+    danglingLinks: [],
   };
 }

package/src/plugins/defaults/memory/substrate/page-links.ts

@@ -0,0 +1,133 @@
+// SUBSTRATE (v2+v3).
+/**
+ * Memory substrate: the single owner of concept-page link syntax.
+ *
+ * A `links:` frontmatter entry is `"<target-slug> <sep> <why>"` with the
+ * separator {@link LINK_SEPARATOR}, split on its first occurrence (no
+ * separator: bare slug). An inline `[[wikilink]]` may carry `|display` and/or
+ * `#anchor`, neither part of the target. `edges:` entries are bare slugs.
+ *
+ * Every reader of these shapes imports from here (the v3 edge lane, the
+ * pending-buffer hints, the page index, ingest); do not add a second wikilink
+ * matcher or `links:` splitter, for the same reason `buffer-format.ts` keeps
+ * one buffer-entry matcher.
+ *
+ * Leaf module: imports nothing from the rest of the substrate.
+ */
+
+/** Space, U+2014 EM DASH, space. */
+export const LINK_SEPARATOR = ` ${String.fromCodePoint(0x2014)} `;
+
+/** Global; group 1 is the raw inner text. Pair with {@link wikilinkTarget}. */
+export const WIKILINK_REGEX = /\[\[([^\]]+)\]\]/g;
+
+export function parseLinkEntry(entry: string): {
+  target: string;
+  description: string | undefined;
+} {
+  const sep = entry.indexOf(LINK_SEPARATOR);
+  if (sep === -1) {
+    return { target: entry.trim(), description: undefined };
+  }
+  return {
+    target: entry.slice(0, sep).trim(),
+    description: entry.slice(sep + LINK_SEPARATOR.length).trim() || undefined,
+  };
+}
+
+/** The slug a wikilink's inner text names; empty for `[[|label]]` / `[[#anchor]]`. */
+export function wikilinkTarget(inner: string): string {
+  let target = inner;
+  const pipe = target.indexOf("|");
+  if (pipe !== -1) {
+    target = target.slice(0, pipe);
+  }
+  const hash = target.indexOf("#");
+  if (hash !== -1) {
+    target = target.slice(0, hash);
+  }
+  return target.trim();
+}
+
+/** Every wikilink target in `text`, in order, empty targets dropped, duplicates kept. */
+export function extractWikilinkTargets(text: string): string[] {
+  const targets: string[] = [];
+  for (const match of text.matchAll(WIKILINK_REGEX)) {
+    const target = wikilinkTarget(match[1]);
+    if (target) {
+      targets.push(target);
+    }
+  }
+  return targets;
+}
+
+export type PageLinkKind = "links" | "wikilink" | "edges";
+
+/** A structural reference from page `from` to slug `to` that has no page behind it. */
+export interface DanglingLink {
+  from: string;
+  to: string;
+  kind: PageLinkKind;
+}
+
+/** The slice of a `ConceptPage` the finder reads (a parsed page satisfies it). */
+export interface PageLinkSource {
+  slug: string;
+  frontmatter: {
+    links?: readonly string[] | undefined;
+    edges?: readonly string[] | undefined;
+  };
+  /** Frontmatter-stripped body. */
+  body: string;
+}
+
+/**
+ * Structural references on `pages` whose target is not in `knownSlugs`.
+ * Only slug-shaped targets (`isSlug`, the page store's rule) are references
+ * at all: `[[ -f foo ]]` in a shell snippet is not a link to a page. Self
+ * references are ignored, each `(from, to, kind)` is reported once, and the
+ * result is sorted by `(from, to, kind)` so renderings are byte-stable.
+ */
+export function findDanglingLinks(
+  pages: readonly PageLinkSource[],
+  knownSlugs: ReadonlySet<string>,
+  isSlug: (target: string) => boolean,
+): DanglingLink[] {
+  const seen = new Set<string>();
+  const dangling: DanglingLink[] = [];
+  const report = (from: string, to: string, kind: PageLinkKind): void => {
+    if (to === from || knownSlugs.has(to) || !isSlug(to)) {
+      return;
+    }
+    const key = `${kind} ${from} ${to}`;
+    if (seen.has(key)) {
+      return;
+    }
+    seen.add(key);
+    dangling.push({ from, to, kind });
+  };
+
+  for (const page of pages) {
+    for (const entry of page.frontmatter.links ?? []) {
+      report(page.slug, parseLinkEntry(entry).target, "links");
+    }
+    for (const target of extractWikilinkTargets(page.body)) {
+      report(page.slug, target, "wikilink");
+    }
+    for (const target of page.frontmatter.edges ?? []) {
+      report(page.slug, target.trim(), "edges");
+    }
+  }
+
+  dangling.sort(
+    (a, b) =>
+      compareStrings(a.from, b.from) ||
+      compareStrings(a.to, b.to) ||
+      compareStrings(a.kind, b.kind),
+  );
+  return dangling;
+}
+
+function compareStrings(a: string, b: string): number {
+  return a < b ? -1 : a > b ? 1 : 0;
+}

package/src/plugins/defaults/memory/substrate/page-store.ts

@@ -153,6 +153,16 @@ export function validateSlug(slug: string): void {
   }
 }
 
+/** Boolean form of {@link validateSlug}. */
+export function isValidSlug(slug: string): boolean {
+  try {
+    validateSlug(slug);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 // ---------------------------------------------------------------------------
 // Path helpers
 // ---------------------------------------------------------------------------