@vellumai/assistant 0.8.5 → 0.8.7

package/src/memory/v3/capabilities.ts

@@ -0,0 +1,124 @@
+/**
+ * Memory v3 — synthetic "capabilities" leaf (skills + `assistant` CLI commands).
+ *
+ * v2 surfaces the assistant's invokable capabilities — installed/catalog skills
+ * and top-level CLI subcommands — by seeding them as synthetic concept-collection
+ * rows (`skills/<id>`, `cli-commands/<name>`) into the unified router pool, then
+ * rendering the router-selected ones under `### Skills You Can Use` / `### CLI
+ * Commands You Can Use`. They are NOT always-injected — they are router-selected
+ * per turn, same as concept pages.
+ *
+ * v3 reproduces that behavior with one always-on leaf:
+ *  - The leaf is synthesized in code (not a `leaves/*.md` data file) and injected
+ *    ONLY into the live lane tree (see `shadow-plugin.ts`). Because the classifier
+ *    builds its own tree without this injection, concept pages are never routed
+ *    INTO the capabilities leaf.
+ *  - Its members are the synthetic skill/CLI slugs, so the BM25 needle indexes
+ *    them for free (the needle corpus is `tree.byPage`).
+ *  - It is added to the always-on core set, so L1 always opens it and the per-leaf
+ *    L2 selects the relevant subset each turn — the semantic equivalent of v2's
+ *    "always in the pool, selected per turn".
+ *
+ * Page summaries for synthetic slugs already resolve through the existing
+ * `pageSummary` lane (the v2 page index appends skill/CLI rows with a summary),
+ * so only full-content rendering for the live `<memory>` block needs a synthetic
+ * fallback — see {@link renderCapabilityContent}.
+ */
+
+import {
+  getCliCommandCapability,
+  isCliCommandSlug,
+} from "../v2/cli-command-store.js";
+import { getSkillCapability, isSkillSlug } from "../v2/skill-store.js";
+import type { LeafNode, LeafPath, LeafTree, Slug } from "./types.js";
+
+/** Path of the always-on synthetic leaf that owns skill + CLI capability rows. */
+export const CAPABILITIES_LEAF_PATH: LeafPath = "capabilities";
+
+/** L1/needle label for the capabilities leaf. */
+export const CAPABILITIES_LEAF_DESCRIPTION =
+  "Tools the assistant can invoke: installed and available skills, and the " +
+  "top-level `assistant` CLI subcommands — what the assistant can DO and how " +
+  "to reach for each capability.";
+
+/** True iff the slug is a synthetic skill or CLI-command capability row. */
+export function isCapabilitySlug(slug: Slug): boolean {
+  return isSkillSlug(slug) || isCliCommandSlug(slug);
+}
+
+/**
+ * Inject the synthetic capabilities leaf into a live lane tree: register the leaf
+ * node with `syntheticSlugs` as members, add the leaf to each member's `byPage`
+ * entry (UNION — never drops existing leaves), and mark the leaf always-on by
+ * adding its path to `core`. Mutates `tree` and `core` in place. Idempotent.
+ *
+ * Must run BEFORE the needle is built so the synthetic members land in the needle
+ * corpus (`tree.byPage`).
+ */
+export function injectCapabilitiesLeaf(
+  tree: LeafTree,
+  core: Set<LeafPath>,
+  syntheticSlugs: Slug[],
+): void {
+  const node: LeafNode = {
+    path: CAPABILITIES_LEAF_PATH,
+    frontmatter: { path: CAPABILITIES_LEAF_PATH, in_core: true },
+    description: CAPABILITIES_LEAF_DESCRIPTION,
+    members: [...syntheticSlugs],
+    domain: CAPABILITIES_LEAF_PATH,
+  };
+  tree.leaves.set(CAPABILITIES_LEAF_PATH, node);
+
+  for (const slug of syntheticSlugs) {
+    const existing = tree.byPage.get(slug) ?? [];
+    if (!existing.includes(CAPABILITIES_LEAF_PATH)) {
+      tree.byPage.set(slug, [...existing, CAPABILITIES_LEAF_PATH]);
+    }
+  }
+
+  // Always-on: L1 always opens it, L2 selects the relevant subset per turn.
+  core.add(CAPABILITIES_LEAF_PATH);
+}
+
+interface CapabilityEntry {
+  id: string;
+  content: string;
+}
+
+export interface CapabilityResolvers {
+  skill: (idOrSlug: string) => CapabilityEntry | null;
+  cli: (idOrSlug: string) => CapabilityEntry | null;
+}
+
+const defaultResolvers: CapabilityResolvers = {
+  skill: getSkillCapability,
+  cli: getCliCommandCapability,
+};
+
+/**
+ * Render a synthetic skill/CLI slug's full capability content for the live
+ * `<memory>` block, mirroring {@link renderV3PageContent}'s `# header\n<content>`
+ * shape. Returns:
+ *  - the rendered block when the slug is a capability slug and resolves;
+ *  - `""` when it is a capability slug but the cache has no entry (degrade to a
+ *    blank section rather than throwing or falling through to an on-disk read
+ *    that would also miss);
+ *  - `null` when the slug is NOT a capability slug, so the caller falls through
+ *    to its normal on-disk page rendering.
+ *
+ * `resolvers` is injectable for tests; production uses the v2 store caches.
+ */
+export function renderCapabilityContent(
+  slug: Slug,
+  resolvers: CapabilityResolvers = defaultResolvers,
+): string | null {
+  if (isSkillSlug(slug)) {
+    const entry = resolvers.skill(slug);
+    return entry ? `# Skill: ${entry.id}\n${entry.content}` : "";
+  }
+  if (isCliCommandSlug(slug)) {
+    const entry = resolvers.cli(slug);
+    return entry ? `# CLI command: ${entry.id}\n${entry.content}` : "";
+  }
+  return null;
+}

package/src/memory/v3/core.ts

@@ -0,0 +1,26 @@
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { LeafPath } from "./types.js";
+
+/**
+ * Loads the always-on core leaf set from `<dataDir>/core.json`.
+ *
+ * The caller is responsible for resolving `dataDir` (including any
+ * workspace-local override — see `resolveDataDir` in `tree.ts`). A missing
+ * `core.json` yields an empty set (new workspaces may have no curated core).
+ */
+export async function loadCore(dataDir: string): Promise<Set<LeafPath>> {
+  const path = join(dataDir, "core.json");
+  let raw: string;
+  try {
+    raw = await readFile(path, "utf8");
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") {
+      return new Set();
+    }
+    throw err;
+  }
+  const parsed = JSON.parse(raw) as { alwaysOn?: LeafPath[] };
+  return new Set(parsed.alwaysOn ?? []);
+}

package/src/memory/v3/data/README.md

@@ -0,0 +1,84 @@
+# memory-v3 data artifacts
+
+This directory ships a **generic, synthetic stub** data set for memory-v3's
+topic-tree routing. It exists so the v3 loaders have something valid to read out
+of the box. It is **example-only**: the names, paths, and bodies here are
+neutral placeholders (`domain-a`, `topic-x`, `page-a`, …) with no real content.
+
+## Workspace override
+
+A maintainer's real per-instance content does **not** live in this repo. At
+runtime the loaders prefer a workspace override directory over this bundled
+default:
+
+```
+<workspace>/memory/v3/data/   ← preferred when present (real content)
+assistant/src/memory/v3/data/ ← this bundled stub (fallback)
+```
+
+If `<workspace>/memory/v3/data/` exists, the loaders read the leaf tree,
+assignments, and core list from there instead of from this directory. Otherwise
+they fall back to the stub shipped here.
+
+## File shapes
+
+### `leaves/**/*.md`
+
+Each leaf is a Markdown file whose path under `leaves/` mirrors its taxonomy
+path: `leaves/domain-a/topic-x.md` is the leaf `domain-a/topic-x`. Every file
+has YAML frontmatter followed by a label body.
+
+```markdown
+---
+path: domain-a/topic-x
+in_core: true
+---
+
+A recall-complete description (~100–300 words in real content) enumerating the
+entities, events, and register cues that route a turn to this leaf.
+```
+
+Frontmatter fields:
+
+| Field     | Type      | Meaning                                                                   |
+| --------- | --------- | ------------------------------------------------------------------------- |
+| `path`    | `string`  | The leaf's taxonomy path. Must match the file's location under `leaves/`. |
+| `in_core` | `boolean` | Whether this leaf is always loaded into context (see `core.json`).        |
+
+The body is the routing label — the prose used to decide whether a turn should
+route to this leaf.
+
+### `assignments.json`
+
+Maps each example slug to the list of leaf paths it belongs to. A slug may be
+multi-homed (assigned to more than one leaf). Every referenced leaf path **must**
+have a corresponding `.md` file under `leaves/`.
+
+```json
+{
+  "page-a": ["domain-a/topic-x"],
+  "page-b": ["domain-a/topic-x", "domain-a/topic-y"],
+  "page-c": ["domain-b/topic-z"]
+}
+```
+
+Shape: `Record<string /* slug */, string[] /* leaf paths */>`.
+
+### `core.json`
+
+The always-on core list — leaves that are loaded for every turn regardless of
+routing. Each entry **must** reference a leaf whose `.md` frontmatter has
+`in_core: true`.
+
+```json
+{ "alwaysOn": ["domain-a/topic-x"] }
+```
+
+Shape: `{ "alwaysOn": string[] /* leaf paths */ }`.
+
+## Consistency invariants
+
+- Every leaf path in `assignments.json` has a matching `.md` file under `leaves/`.
+- Every leaf path in `core.json`'s `alwaysOn` has `in_core: true` in its `.md`
+  frontmatter, and vice versa (an `in_core: true` leaf appears in `alwaysOn`).
+- A leaf file's `path` frontmatter matches its location under `leaves/`.

package/src/memory/v3/data/assignments.json

@@ -0,0 +1,5 @@
+{
+  "page-a": ["domain-a/topic-x"],
+  "page-b": ["domain-a/topic-x", "domain-a/topic-y"],
+  "page-c": ["domain-b/topic-z"]
+}

package/src/memory/v3/data/core.json

@@ -0,0 +1 @@
+{ "alwaysOn": ["domain-a/topic-x"] }

package/src/memory/v3/data/leaves/domain-a/topic-x.md

@@ -0,0 +1,9 @@
+---
+path: domain-a/topic-x
+in_core: true
+---
+
+Example leaf label for topic X in domain A. In a real workspace this paragraph
+is a recall-complete description (~100–300 words) enumerating the entities,
+events, and register cues that route a turn to this leaf. This bundled stub is
+intentionally generic; the maintainer's workspace supplies the real content.

package/src/memory/v3/data/leaves/domain-a/topic-y.md

@@ -0,0 +1,9 @@
+---
+path: domain-a/topic-y
+in_core: false
+---
+
+Example leaf label for topic Y in domain A. In a real workspace this paragraph
+is a recall-complete description (~100–300 words) enumerating the entities,
+events, and register cues that route a turn to this leaf. This bundled stub is
+intentionally generic; the maintainer's workspace supplies the real content.

package/src/memory/v3/data/leaves/domain-b/topic-z.md

@@ -0,0 +1,9 @@
+---
+path: domain-b/topic-z
+in_core: false
+---
+
+Example leaf label for topic Z in domain B. In a real workspace this paragraph
+is a recall-complete description (~100–300 words) enumerating the entities,
+events, and register cues that route a turn to this leaf. This bundled stub is
+intentionally generic; the maintainer's workspace supplies the real content.

package/src/memory/v3/maintain-job.ts

@@ -0,0 +1,314 @@
+/**
+ * Memory v3 — `memory_v3_maintain` job handler.
+ *
+ * A flag-gated, best-effort self-maintenance pass over the v3 topic tree and
+ * its page→leaf assignments. It runs three independent stages, in order:
+ *
+ *   1. **Classify-union** — `assignPages` over the delta: every page that is
+ *      still unassigned (empty `leaves:`) plus every page edited since the last
+ *      successful pass (the high-water mark below), UNIONing freshly classified
+ *      leaves into each page (never dropping existing picks). Re-touching
+ *      already-assigned-but-recently-edited pages lets the classifier enrich the
+ *      assistant's own in-consolidation labels with any leaves it missed.
+ *   2. **Prune** — drop any frontmatter `leaves:` entry that points at a leaf
+ *      path no longer present in the tree (dangling references left behind when
+ *      a leaf is renamed or removed). Read + rewrite only the affected pages.
+ *   3. **Needle rebuild** — `invalidateLanes()` so the next turn rebuilds the
+ *      leaf tree and BM25 needle from the freshly-updated assignments.
+ *
+ * Best-effort by construction: each stage is wrapped so a failure in one is
+ * logged and recorded in the outcome but does NOT abort the others. The job is
+ * a no-op (returns a disabled outcome) unless `memory-v3-shadow` OR
+ * `memory-v3-live` is enabled — the same flags that gate the v3 plugin.
+ *
+ * Dependency-injectable: `deps` lets tests substitute the tree loader,
+ * `assignPages`, and `invalidateLanes` without process-global module mocks.
+ */
+
+import { isAssistantFeatureFlagEnabled } from "../../config/assistant-feature-flags.js";
+import type { AssistantConfig } from "../../config/types.js";
+import { getLogger } from "../../util/logger.js";
+import { getWorkspaceDir } from "../../util/platform.js";
+import { getMemoryCheckpoint, setMemoryCheckpoint } from "../checkpoints.js";
+import type { MemoryJob } from "../jobs-store.js";
+import { getPageIndex } from "../v2/page-index.js";
+import { listPages, readPage, writePage } from "../v2/page-store.js";
+import { assignPages as realAssignPages } from "./assign.js";
+import { invalidateLanes as realInvalidateLanes } from "./shadow-plugin.js";
+import { loadLeafTree as realLoadLeafTree, resolveDataDir } from "./tree.js";
+import type { LeafPath, LeafTree, Slug } from "./types.js";
+
+const MEMORY_V3_SHADOW = "memory-v3-shadow" as const;
+const MEMORY_V3_LIVE = "memory-v3-live" as const;
+
+/**
+ * Durable checkpoint holding the epoch-ms high-water mark of the last successful
+ * classify-union pass. Pages whose mtime is past this mark are re-classified
+ * (enrichment); the mark is advanced only after a pass completes, captured after
+ * its writes so the pass's own frontmatter rewrites do not re-trigger it.
+ * Distinct from `memory_v3_maintain_last_run` (the enqueue-cadence checkpoint in
+ * `jobs-worker.ts`), which advances on every backstop enqueue rather than on an
+ * actual maintenance run.
+ */
+const MAINTAIN_ENRICH_HIGH_WATER_KEY =
+  "memory_v3_maintain:enriched_through_ms" as const;
+
+const log = getLogger("memory-v3-maintain");
+
+/** Injectable collaborators; defaults wire the real implementations. */
+export interface MaintainJobDeps {
+  /** Load the v3 leaf tree (page→leaf membership resolved from frontmatter). */
+  loadTree: () => Promise<LeafTree>;
+  /** Classify-union pass over pages. */
+  assignPages: typeof realAssignPages;
+  /**
+   * The slugs to classify this pass: unassigned pages plus pages edited since
+   * the last successful pass. See {@link computeClassifyTargets}.
+   */
+  selectClassifyTargets: () => Promise<Slug[]>;
+  /**
+   * Persist the enrichment high-water mark after a successful classify pass. The
+   * value is captured after the pass's writes (see the key docstring).
+   */
+  commitClassifyHighWater: (highWaterMs: number) => void;
+  /** Drop the memoized v3 lanes so the next turn rebuilds tree + needle. */
+  invalidateLanes: () => void;
+  /** Workspace root; pages live under `<workspaceDir>/memory/concepts/`. */
+  workspaceDir: string;
+}
+
+/** Per-stage outcome surfaced in the structured log line. */
+export interface MaintainOutcome {
+  /** True when both v3 flags were off and the job no-opped. */
+  disabled: boolean;
+  /** Pages newly assigned by the classify-union stage. */
+  assigned: number;
+  /** Pages whose dangling refs were pruned. */
+  pruned: number;
+  /** Dangling leaf references dropped across all pruned pages. */
+  prunedRefs: number;
+  /** Whether the needle/tree lanes were invalidated. */
+  invalidated: boolean;
+  /** Stages that threw (and were contained). */
+  failures: string[];
+}
+
+/**
+ * Load the v3 leaf tree the same way the live plugin does: resolve the data
+ * dir, build the page→leaf membership map from the page index frontmatter, and
+ * hand both to `loadLeafTree`.
+ */
+async function loadTreeFromWorkspace(workspaceDir: string): Promise<LeafTree> {
+  const pageIndex = await getPageIndex(workspaceDir);
+  const pageLeaves = new Map<Slug, LeafPath[]>();
+  for (const entry of pageIndex.entries) {
+    pageLeaves.set(entry.slug, entry.leaves);
+  }
+  return realLoadLeafTree(resolveDataDir(), pageLeaves);
+}
+
+/** Page-index projection the classify-target selector reads. */
+export interface ClassifyCandidate {
+  slug: Slug;
+  /** File mtime in epoch ms; 0 for synthetic skill/CLI rows (excluded). */
+  modifiedAt: number;
+  /** Leaf assignments from frontmatter; `[]` when unassigned. */
+  leaves: LeafPath[];
+}
+
+/**
+ * Pick the pages the classify-union stage should run this pass:
+ *
+ *  - **Unassigned** pages (empty `leaves:`) — always classified so a page is
+ *    never left permanently unrouted (the pre-enrichment behavior).
+ *  - **Recently edited** pages (mtime past `prevHighWaterMs`) — even when
+ *    already assigned, so the classifier can enrich the assistant's own
+ *    in-consolidation labels with leaves it missed. `assignPages` UNIONs, so a
+ *    page's existing picks are never dropped — only added to.
+ *
+ * Synthetic skill/CLI rows (`modifiedAt === 0`) are excluded: they are
+ * capability entries, not topic-tree pages. On the first run (`prevHighWaterMs`
+ * null) the recency arm is disabled, so a fresh or freshly-backfilled install
+ * does a single unassigned-only pass instead of re-classifying the whole corpus.
+ */
+export function computeClassifyTargets(
+  pages: ReadonlyArray<ClassifyCandidate>,
+  prevHighWaterMs: number | null,
+): Slug[] {
+  const targets: Slug[] = [];
+  for (const page of pages) {
+    if (page.modifiedAt <= 0) continue; // synthetic capability row
+    const unassigned = page.leaves.length === 0;
+    const recentlyEdited =
+      prevHighWaterMs !== null && page.modifiedAt > prevHighWaterMs;
+    if (unassigned || recentlyEdited) targets.push(page.slug);
+  }
+  return targets;
+}
+
+/** Read the persisted high-water mark, treating missing/garbage as first-run. */
+function readClassifyHighWater(): number | null {
+  const raw = getMemoryCheckpoint(MAINTAIN_ENRICH_HIGH_WATER_KEY);
+  if (raw === null) return null;
+  const parsed = Number.parseInt(raw, 10);
+  return Number.isNaN(parsed) ? null : parsed;
+}
+
+/** Default target selector: page index → {@link computeClassifyTargets}. */
+async function selectClassifyTargetsFromWorkspace(
+  workspaceDir: string,
+): Promise<Slug[]> {
+  const index = await getPageIndex(workspaceDir);
+  return computeClassifyTargets(index.entries, readClassifyHighWater());
+}
+
+function commitClassifyHighWater(highWaterMs: number): void {
+  setMemoryCheckpoint(MAINTAIN_ENRICH_HIGH_WATER_KEY, String(highWaterMs));
+}
+
+function defaultDeps(): MaintainJobDeps {
+  const workspaceDir = getWorkspaceDir();
+  return {
+    loadTree: () => loadTreeFromWorkspace(workspaceDir),
+    assignPages: realAssignPages,
+    selectClassifyTargets: () =>
+      selectClassifyTargetsFromWorkspace(workspaceDir),
+    commitClassifyHighWater,
+    invalidateLanes: realInvalidateLanes,
+    workspaceDir,
+  };
+}
+
+/**
+ * Prune dangling `leaves:` references: for every page, drop frontmatter leaf
+ * paths that are not present in `tree`. Read + rewrite only pages that actually
+ * change. Returns the number of pages rewritten and total refs dropped.
+ */
+async function prunePages(
+  tree: LeafTree,
+  workspaceDir: string,
+): Promise<{ pruned: number; prunedRefs: number }> {
+  const validLeaves = tree.leaves;
+  const slugs = await listPages(workspaceDir);
+  let pruned = 0;
+  let prunedRefs = 0;
+  for (const slug of slugs) {
+    const page = await readPage(workspaceDir, slug);
+    const leaves = page?.frontmatter.leaves;
+    if (!page || !leaves || leaves.length === 0) continue;
+    const kept = leaves.filter((leaf) => validLeaves.has(leaf));
+    if (kept.length === leaves.length) continue;
+    prunedRefs += leaves.length - kept.length;
+    pruned += 1;
+    await writePage(workspaceDir, {
+      ...page,
+      frontmatter: { ...page.frontmatter, leaves: kept },
+    });
+  }
+  return { pruned, prunedRefs };
+}
+
+/**
+ * Run the v3 self-maintenance pass. Each stage is independently contained: a
+ * thrown error is logged and recorded in `failures` without aborting the rest.
+ * No-ops when both v3 flags are off.
+ */
+export async function maintainJob(
+  _job: MemoryJob,
+  config: AssistantConfig,
+  deps: MaintainJobDeps = defaultDeps(),
+): Promise<MaintainOutcome> {
+  const outcome: MaintainOutcome = {
+    disabled: false,
+    assigned: 0,
+    pruned: 0,
+    prunedRefs: 0,
+    invalidated: false,
+    failures: [],
+  };
+
+  const enabled =
+    isAssistantFeatureFlagEnabled(MEMORY_V3_SHADOW, config) ||
+    isAssistantFeatureFlagEnabled(MEMORY_V3_LIVE, config);
+  if (!enabled) {
+    outcome.disabled = true;
+    return outcome;
+  }
+
+  // The tree is shared input for both the assign and prune stages. If it can't
+  // load, those two stages are skipped (recorded as a failure), but we still
+  // invalidate the lanes so a transient load error doesn't wedge the next turn.
+  let tree: LeafTree | null = null;
+  try {
+    tree = await deps.loadTree();
+  } catch (err) {
+    outcome.failures.push("load_tree");
+    log.warn(
+      { err: err instanceof Error ? err.message : String(err) },
+      "memory-v3 maintain: tree load failed (non-fatal)",
+    );
+  }
+
+  if (tree) {
+    // Stage 1: classify-union over the delta (unassigned ∪ recently-edited).
+    try {
+      const targets = await deps.selectClassifyTargets();
+      const results = await deps.assignPages({
+        tree,
+        workspaceDir: deps.workspaceDir,
+        slugs: targets,
+      });
+      outcome.assigned = results.filter(
+        (r) => !r.failed && r.after.length > r.before.length,
+      ).length;
+      // Advance the high-water mark only on success, captured AFTER the writes
+      // above so pages this pass just rewrote (mtime now bumped) sit at-or-below
+      // the mark and do not re-trigger themselves next pass.
+      deps.commitClassifyHighWater(Date.now());
+    } catch (err) {
+      outcome.failures.push("assign");
+      log.warn(
+        { err: err instanceof Error ? err.message : String(err) },
+        "memory-v3 maintain: classify-union failed (non-fatal)",
+      );
+    }
+
+    // Stage 2: prune dangling refs.
+    try {
+      const { pruned, prunedRefs } = await prunePages(tree, deps.workspaceDir);
+      outcome.pruned = pruned;
+      outcome.prunedRefs = prunedRefs;
+    } catch (err) {
+      outcome.failures.push("prune");
+      log.warn(
+        { err: err instanceof Error ? err.message : String(err) },
+        "memory-v3 maintain: prune failed (non-fatal)",
+      );
+    }
+  }
+
+  // Stage 3: rebuild tree + needle on the next turn.
+  try {
+    deps.invalidateLanes();
+    outcome.invalidated = true;
+  } catch (err) {
+    outcome.failures.push("invalidate");
+    log.warn(
+      { err: err instanceof Error ? err.message : String(err) },
+      "memory-v3 maintain: lane invalidation failed (non-fatal)",
+    );
+  }
+
+  log.info(
+    {
+      assigned: outcome.assigned,
+      pruned: outcome.pruned,
+      prunedRefs: outcome.prunedRefs,
+      invalidated: outcome.invalidated,
+      failures: outcome.failures,
+    },
+    "memory-v3 maintain pass complete",
+  );
+
+  return outcome;
+}