@vellumai/assistant 0.8.9-staging.2 → 0.8.9-staging.4

package/src/plugins/defaults/memory-v3-shadow/capabilities.ts

@@ -1,23 +1,10 @@
 /**
- * Memory v3 — synthetic "capabilities" leaf (skills + `assistant` CLI commands).
+ * Memory v3 — synthetic capability rows (skills + `assistant` CLI commands).
  *
- * v2 surfaces the assistant's invokable capabilities — installed/catalog skills
- * and top-level CLI subcommands — by seeding them as synthetic concept-collection
- * rows (`skills/<id>`, `cli-commands/<name>`) into the unified router pool, then
- * rendering the router-selected ones under `### Skills You Can Use` / `### CLI
- * Commands You Can Use`. They are NOT always-injected — they are router-selected
- * per turn, same as concept pages.
- *
- * v3 reproduces that behavior with one always-on leaf:
- *  - The leaf is synthesized in code (not a `leaves/*.md` data file) and injected
- *    ONLY into the live lane tree (see `shadow-plugin.ts`). Because the classifier
- *    builds its own tree without this injection, concept pages are never routed
- *    INTO the capabilities leaf.
- *  - Its members are the synthetic skill/CLI slugs, so the BM25 needle indexes
- *    them for free (the needle corpus is `tree.byPage`).
- *  - It is added to the always-on core set, so L1 always opens it and the per-leaf
- *    L2 selects the relevant subset each turn — the semantic equivalent of v2's
- *    "always in the pool, selected per turn".
+ * The assistant's invokable capabilities — installed/catalog skills and
+ * top-level CLI subcommands — are seeded as synthetic concept-collection rows
+ * (`skills/<id>`, `cli-commands/<name>`). The section-lane retrieval pipeline
+ * surfaces the relevant ones per turn, same as concept pages.
  *
  * Page summaries for synthetic slugs already resolve through the existing
  * `pageSummary` lane (the v2 page index appends skill/CLI rows with a summary),
@@ -33,56 +20,13 @@ import {
   getSkillCapability,
   isSkillSlug,
 } from "../../../memory/v2/skill-store.js";
-import type { LeafNode, LeafPath, LeafTree, Slug } from "./types.js";
-
-/** Path of the always-on synthetic leaf that owns skill + CLI capability rows. */
-export const CAPABILITIES_LEAF_PATH: LeafPath = "capabilities";
-
-/** L1/needle label for the capabilities leaf. */
-export const CAPABILITIES_LEAF_DESCRIPTION =
-  "Tools the assistant can invoke: installed and available skills, and the " +
-  "top-level `assistant` CLI subcommands — what the assistant can DO and how " +
-  "to reach for each capability.";
+import type { Slug } from "./types.js";
 
 /** True iff the slug is a synthetic skill or CLI-command capability row. */
 export function isCapabilitySlug(slug: Slug): boolean {
   return isSkillSlug(slug) || isCliCommandSlug(slug);
 }
 
-/**
- * Inject the synthetic capabilities leaf into a live lane tree: register the leaf
- * node with `syntheticSlugs` as members, add the leaf to each member's `byPage`
- * entry (UNION — never drops existing leaves), and mark the leaf always-on by
- * adding its path to `core`. Mutates `tree` and `core` in place. Idempotent.
- *
- * Must run BEFORE the needle is built so the synthetic members land in the needle
- * corpus (`tree.byPage`).
- */
-export function injectCapabilitiesLeaf(
-  tree: LeafTree,
-  core: Set<LeafPath>,
-  syntheticSlugs: Slug[],
-): void {
-  const node: LeafNode = {
-    path: CAPABILITIES_LEAF_PATH,
-    frontmatter: { path: CAPABILITIES_LEAF_PATH, in_core: true },
-    description: CAPABILITIES_LEAF_DESCRIPTION,
-    members: [...syntheticSlugs],
-    domain: CAPABILITIES_LEAF_PATH,
-  };
-  tree.leaves.set(CAPABILITIES_LEAF_PATH, node);
-
-  for (const slug of syntheticSlugs) {
-    const existing = tree.byPage.get(slug) ?? [];
-    if (!existing.includes(CAPABILITIES_LEAF_PATH)) {
-      tree.byPage.set(slug, [...existing, CAPABILITIES_LEAF_PATH]);
-    }
-  }
-
-  // Always-on: L1 always opens it, L2 selects the relevant subset per turn.
-  core.add(CAPABILITIES_LEAF_PATH);
-}
-
 interface CapabilityEntry {
   id: string;
   content: string;
@@ -125,3 +69,23 @@ export function renderCapabilityContent(
   }
   return null;
 }
+
+/**
+ * Resolve a slug's frontmatter-stripped body for the section index: synthetic
+ * skill/CLI capability slugs have no on-disk page, so they contribute their
+ * rendered capability content (exactly what `page-content.ts` injects for them),
+ * while real pages fall through to `readDiskBody`. Shared by `initLanes`'
+ * `pageBody` and the full-backfill body reader so the capability-or-disk dispatch
+ * lives in one place.
+ *
+ * `readDiskBody` is injected (rather than imported) so this helper does not pull
+ * the page store into `capabilities.ts` — each caller supplies its own cached or
+ * direct disk reader.
+ */
+export async function capabilityOrDiskBody(
+  slug: Slug,
+  readDiskBody: (slug: Slug) => Promise<string>,
+): Promise<string> {
+  if (isCapabilitySlug(slug)) return renderCapabilityContent(slug) ?? "";
+  return readDiskBody(slug);
+}

package/src/plugins/defaults/memory-v3-shadow/dense.ts

@@ -0,0 +1,97 @@
+// ---------------------------------------------------------------------------
+// Memory v3 — dense retrieval lane (section-grain)
+// ---------------------------------------------------------------------------
+//
+// Read counterpart to `section-dense-store.ts`. Embeds the turn query and runs
+// a single cosine search against the `memory_v3_sections` collection, then
+// dedupes the matched section points down to the top-`k` distinct articles —
+// each carrying its best-scoring section ordinal. This is the dense lane of the
+// section-grain retrieval design: where the v2 dense lane matches whole pages,
+// this one matches the single most relevant section of a long article and hands
+// the orchestrator both the article and which section matched (so the selector
+// can show the matched section as the descriptor).
+//
+// Degrades safely: any embedding or Qdrant failure logs a warning and returns
+// `[]`. The orchestrator unions the other lanes (needle, edge) plus carry-
+// forward regardless, so a dense outage narrows recall but never breaks a turn.
+
+import type { AssistantConfig } from "../../../config/types.js";
+import { embedWithBackend } from "../../../memory/embedding-backend.js";
+import { getLogger } from "../../../util/logger.js";
+import {
+  getSectionDenseClient,
+  SECTION_COLLECTION,
+} from "./section-dense-store.js";
+import type { Slug } from "./types.js";
+
+const log = getLogger("memory-v3-dense-lane");
+
+/**
+ * Multiplier applied to `k` when fetching section points from Qdrant. Several
+ * sections can belong to the same article, so we oversample the section hits to
+ * leave room for the article-level dedupe to still yield `k` distinct articles.
+ */
+export const OVERSAMPLE = 6;
+
+/** A single dense-lane hit: an article plus the section ordinal that matched. */
+export interface DenseHit {
+  article: Slug;
+  section: number;
+}
+
+/**
+ * Run the dense lane: embed `query`, search the section collection for the top
+ * `k * OVERSAMPLE` section points, then dedupe to the top-`k` distinct articles
+ * — each with its best-scoring section ordinal. Section points are returned by
+ * Qdrant in descending score order, so the first time an article is seen is its
+ * best section; subsequent sections of the same article are ignored.
+ *
+ * Returns `[]` on any embedding or Qdrant failure (logged at warn level).
+ */
+export async function denseLane(
+  config: AssistantConfig,
+  query: string,
+  k: number,
+): Promise<DenseHit[]> {
+  if (k <= 0) return [];
+
+  let points: Array<{ payload?: unknown; score?: number }>;
+  try {
+    const { vectors } = await embedWithBackend(config, [query]);
+    const vector = vectors[0];
+    if (!vector || vector.length === 0) return [];
+
+    const result = await getSectionDenseClient(config).query(
+      SECTION_COLLECTION,
+      {
+        query: vector,
+        limit: k * OVERSAMPLE,
+        with_payload: true,
+      },
+    );
+    points = result.points;
+  } catch (err) {
+    log.warn({ err }, "memory v3 dense lane failed; degrading to no hits");
+    return [];
+  }
+
+  // Walk hits in score order, keeping the first (best) section per article and
+  // stopping once we have `k` distinct articles.
+  const seen = new Set<Slug>();
+  const hits: DenseHit[] = [];
+  for (const point of points) {
+    const payload = point.payload as
+      | { article?: unknown; ordinal?: unknown }
+      | null
+      | undefined;
+    const article = payload?.article;
+    const ordinal = payload?.ordinal;
+    if (typeof article !== "string" || typeof ordinal !== "number") continue;
+    if (seen.has(article)) continue;
+    seen.add(article);
+    hits.push({ article, section: ordinal });
+    if (hits.length >= k) break;
+  }
+
+  return hits;
+}

package/src/plugins/defaults/memory-v3-shadow/edge.ts

@@ -0,0 +1,252 @@
+import type { PageIndexEntry } from "../../../memory/v2/page-index.js";
+import { parseFrontmatterFields } from "../../../skills/frontmatter.js";
+import type { Slug } from "./types.js";
+
+/**
+ * Edge lane for memory-v3 retrieval: a directed article link-graph used to
+ * expand a turn's lexical/dense seeds outward to their first-class neighbours.
+ *
+ * The graph unions THREE outbound-edge sources per article:
+ *
+ *   (a) The optional `links:` frontmatter — a YAML list of
+ *       `"<target-slug> — <description>"` strings (split on the first
+ *       ` — `, space-emdash-space). This is the authored, first-class edge
+ *       source: the curated description is carried through expansion so the
+ *       orchestrator can use it directly as a select descriptor.
+ *   (b) Inline `[[wikilink]]` targets parsed from the body.
+ *   (c) `PageIndexEntry.edges` numeric ids resolved to slugs — the fallback
+ *       for pages with no frontmatter `links`.
+ *
+ * All targets are resolved against the corpus slug set; unknown/dangling
+ * targets are dropped. A curated description is stored per (source → target)
+ * edge only when the `links` source supplied one; wikilink/numeric edges carry
+ * no description (the orchestrator falls back to a section descriptor).
+ *
+ * The build is pure given a `pageRaw` reader callback (no hard-coded I/O), so
+ * it is built once at lane-init and rebuilt by the maintain job, and is
+ * trivially unit-testable with a stub reader.
+ */
+
+/** Default in-degree above which an article is treated as a hub and excluded
+ * from expansion (a hub neighbour is too generic to be a useful surface). */
+const DEFAULT_HUB_DEGREE = 30;
+
+const DEFAULT_SEED_COUNT = 18;
+const DEFAULT_PER_SEED = 6;
+const DEFAULT_CAP = 45;
+
+/** Matches the space-emdash-space separator in a `links:` entry. */
+const LINK_SEPARATOR = " — ";
+
+/** Matches `[[target]]` wikilinks; captures the raw target before any `|`
+ * display-text or `#section` anchor. */
+const WIKILINK_REGEX = /\[\[([^\]]+)\]\]/g;
+
+/**
+ * The directed link-graph. `adjacency` maps each source slug to its outbound
+ * edges (target slug → curated description, or `undefined` when the edge came
+ * from a wikilink/numeric source). `hubs` is the set of high-in-degree slugs
+ * excluded from expansion. `slugs` is the resolved corpus slug set.
+ */
+export interface EdgeGraph {
+  adjacency: Map<Slug, Map<Slug, string | undefined>>;
+  hubs: Set<Slug>;
+  slugs: Set<Slug>;
+}
+
+/** One article surfaced by {@link edgeExpand}, with the curated `links`
+ * description of the traversed edge when it carried one (else `undefined`). */
+export interface EdgeNeighbor {
+  article: Slug;
+  description?: string;
+}
+
+export interface EdgeExpandOptions {
+  /** Predicate gating which neighbours may be surfaced (e.g. liveness /
+   * not-already-selected). Neighbours failing `alive` are skipped. */
+  alive?: (slug: Slug) => boolean;
+  /** Only the top `seedCount` seeds (in input order) are expanded. */
+  seedCount?: number;
+  /** Up to `perSeed` neighbours are added per expanded seed. */
+  perSeed?: number;
+  /** Hard cap on the total number of distinct surfaced articles. */
+  cap?: number;
+}
+
+interface BuildEdgeGraphOptions {
+  /** In-degree above which an article is a hub. Defaults to
+   * {@link DEFAULT_HUB_DEGREE}. */
+  hubDegree?: number;
+}
+
+/**
+ * Split one `links:` entry (`"<target-slug> — <description>"`) into its target
+ * slug and curated description on the FIRST ` — ` (space-emdash-space).
+ * Entries with no separator are bare target slugs and carry no description.
+ */
+function parseLinkEntry(entry: string): {
+  target: Slug;
+  description: string | undefined;
+} {
+  const sep = entry.indexOf(LINK_SEPARATOR);
+  if (sep === -1) return { target: entry.trim(), description: undefined };
+  return {
+    target: entry.slice(0, sep).trim(),
+    description: entry.slice(sep + LINK_SEPARATOR.length).trim() || undefined,
+  };
+}
+
+/** Parse inline `[[wikilink]]` targets from a body. Strips `|display` and
+ * `#anchor` suffixes; returns trimmed target slugs (possibly with duplicates,
+ * deduped by the caller's map insertion). */
+function parseWikilinks(body: string): string[] {
+  const targets: string[] = [];
+  for (const match of body.matchAll(WIKILINK_REGEX)) {
+    let target = match[1];
+    const pipe = target.indexOf("|");
+    if (pipe !== -1) target = target.slice(0, pipe);
+    const hash = target.indexOf("#");
+    if (hash !== -1) target = target.slice(0, hash);
+    target = target.trim();
+    if (target) targets.push(target);
+  }
+  return targets;
+}
+
+/**
+ * Build the directed article link-graph from the page-index entries and a raw
+ * page reader. See the module docstring for the three unioned edge sources.
+ *
+ * Pure given `pageRaw`: the build performs no hard-coded I/O. A read that
+ * rejects drops that article's authored/wikilink edges but still keeps its
+ * numeric `PageIndexEntry.edges` fallback.
+ */
+export async function buildEdgeGraph(
+  articles: readonly PageIndexEntry[],
+  pageRaw: (slug: Slug) => Promise<string>,
+  opts: BuildEdgeGraphOptions = {},
+): Promise<EdgeGraph> {
+  const hubDegree = opts.hubDegree ?? DEFAULT_HUB_DEGREE;
+
+  const slugs = new Set<Slug>(articles.map((a) => a.slug));
+  const byId = new Map<number, Slug>(articles.map((a) => [a.id, a.slug]));
+
+  const raws = await Promise.all(
+    articles.map((a) =>
+      pageRaw(a.slug).then(
+        (text) => text,
+        () => null, // read failed — fall back to numeric edges only
+      ),
+    ),
+  );
+
+  const adjacency = new Map<Slug, Map<Slug, string | undefined>>();
+  const inDegree = new Map<Slug, number>();
+
+  const addEdge = (
+    source: Slug,
+    target: Slug,
+    description: string | undefined,
+  ): void => {
+    if (target === source) return; // no self-edges
+    if (!slugs.has(target)) return; // drop unknown/dangling targets
+    let out = adjacency.get(source);
+    if (!out) {
+      out = new Map();
+      adjacency.set(source, out);
+    }
+    if (!out.has(target)) {
+      out.set(target, description);
+      inDegree.set(target, (inDegree.get(target) ?? 0) + 1);
+    } else if (out.get(target) === undefined && description !== undefined) {
+      // A later source (the authored `links`) supplies a description for an
+      // edge first seen without one — upgrade it. In-degree already counted.
+      out.set(target, description);
+    }
+  };
+
+  for (let i = 0; i < articles.length; i++) {
+    const article = articles[i];
+    const source = article.slug;
+    const raw = raws[i];
+
+    // (a) authored `links:` frontmatter — primary, carries descriptions.
+    const parsed = raw !== null ? parseFrontmatterFields(raw) : null;
+    const links = parsed?.fields.links;
+    if (Array.isArray(links)) {
+      for (const entry of links) {
+        if (typeof entry !== "string") continue;
+        const { target, description } = parseLinkEntry(entry);
+        addEdge(source, target, description);
+      }
+    }
+
+    // (b) inline `[[wikilink]]` targets from the body. `parsed.body` strips
+    // the frontmatter; a page without frontmatter has `parsed === null`, so
+    // the whole raw text is the body.
+    const body = parsed ? parsed.body : raw;
+    if (body !== null) {
+      for (const target of parseWikilinks(body)) {
+        addEdge(source, target, undefined);
+      }
+    }
+
+    // (c) numeric page-index edges resolved to slugs — fallback.
+    for (const targetId of article.edges) {
+      const target = byId.get(targetId);
+      if (target) addEdge(source, target, undefined);
+    }
+  }
+
+  const hubs = new Set<Slug>();
+  for (const [slug, degree] of inDegree) {
+    if (degree > hubDegree) hubs.add(slug);
+  }
+
+  return { adjacency, hubs, slugs };
+}
+
+/**
+ * Expand `seeds` outward along the graph. For each of the top `seedCount`
+ * seeds (in input order), surface up to `perSeed` non-hub, `alive`-passing
+ * neighbours, capped at `cap` total distinct articles. Seeds themselves are
+ * not surfaced (they are already in the pool). Each surfaced article carries
+ * the curated `links` description of the traversed edge when it had one.
+ *
+ * Deterministic given the input seed order and the graph's insertion order.
+ */
+export function edgeExpand(
+  graph: EdgeGraph,
+  seeds: readonly Slug[],
+  opts: EdgeExpandOptions = {},
+): EdgeNeighbor[] {
+  const seedCount = opts.seedCount ?? DEFAULT_SEED_COUNT;
+  const perSeed = opts.perSeed ?? DEFAULT_PER_SEED;
+  const cap = opts.cap ?? DEFAULT_CAP;
+  const alive = opts.alive;
+
+  const seedSet = new Set<Slug>(seeds);
+  const surfaced = new Map<Slug, string | undefined>();
+
+  for (const seed of seeds.slice(0, seedCount)) {
+    if (surfaced.size >= cap) break;
+    const out = graph.adjacency.get(seed);
+    if (!out) continue;
+    let added = 0;
+    for (const [target, description] of out) {
+      if (added >= perSeed) break;
+      if (surfaced.size >= cap) break;
+      if (seedSet.has(target)) continue; // already in the pool
+      if (surfaced.has(target)) continue; // already surfaced via another seed
+      if (graph.hubs.has(target)) continue; // hubs excluded
+      if (alive && !alive(target)) continue;
+      surfaced.set(target, description);
+      added++;
+    }
+  }
+
+  return [...surfaced].map(([article, description]) => ({
+    article,
+    description,
+  }));
+}

package/src/plugins/defaults/memory-v3-shadow/injector.ts

@@ -22,7 +22,7 @@ import {
   type Injector,
   type TurnContext,
 } from "../../types.js";
-import { renderV3PageContent } from "./page-content.js";
+import { renderV3SectionContent } from "./page-content.js";
 import { renderMemoryBlock } from "./render-injection.js";
 import {
   MEMORY_V3_LIVE,
@@ -52,7 +52,8 @@ export const memoryV3Injector: Injector = {
       // `renderMemoryBlock` returns "" for an empty selection; inject nothing.
       const text = await renderMemoryBlock(
         result.finalInjection,
-        renderV3PageContent,
+        result.sectionBySlug,
+        renderV3SectionContent,
       );
       if (text.length === 0) return null;
       return {