@vellumai/assistant 0.8.5 → 0.8.7

package/src/memory/v3/tree-index.ts

@@ -1,237 +0,0 @@
-/**
- * Memory v3 — Tree index (DAG build + cache).
- *
- * The v3 tree is a DAG *overlay* over the flat `memory/concepts/` pages: every
- * node carries an ordered `children` list whose entries are either
- * `"page:<slug>"` (a leaf concept page, canonical content untouched by v3) or
- * `"node:<id>"` (a sub-node in the tree). A page or node may be referenced by
- * more than one parent — hence DAG, not tree.
- *
- * This module scans every node on disk and materializes that edge list into
- * forward and reverse adjacency maps so downstream routing/validation can walk
- * the graph without re-reading the filesystem:
- *   - `childrenByNode` — node id → ordered child refs (forward edges).
- *   - `parentsByNode` — node id → set of parent node ids (reverse edges for
- *     `node:` children).
- *   - `pageParents` — page slug → set of parent node ids (reverse edges for
- *     `page:` children).
- *
- * The build is **structural only**: it never verifies that a referenced page
- * or node actually exists. Dangling refs are retained in the adjacency maps so
- * a later validation pass can report them. Root detection prefers the reserved
- * `_root` id; absent that it picks the single node with no parents (warning and
- * picking deterministically if the choice is ambiguous).
- *
- * The build is cached module-locally per `workspaceDir`, mirroring
- * `../v2/page-index.ts`. Callers must invalidate via `invalidateTreeIndex` when
- * tree nodes change — `tree-store.ts`'s `writeNode` / `deleteNode` already do.
- */
-
-import { getLogger } from "../../util/logger.js";
-import { listNodes, readNode, ROOT_NODE_ID } from "./tree-store.js";
-import type { TreeNode } from "./types.js";
-
-const log = getLogger("memory-v3-tree-index");
-
-/** Prefix marking a child ref that targets a leaf concept page. */
-const PAGE_REF_PREFIX = "page:";
-
-/** Prefix marking a child ref that targets a sub-node in the tree. */
-const NODE_REF_PREFIX = "node:";
-
-/**
- * A single parsed `children` entry. `kind` distinguishes a leaf concept page
- * (`"page"`) from a sub-node (`"node"`); `ref` is the bare slug or node id with
- * the `page:` / `node:` prefix stripped.
- */
-export interface ChildRef {
-  kind: "page" | "node";
-  ref: string;
-}
-
-/**
- * Snapshot of the v3 tree DAG for one workspace.
- *
- * `nodes` is every readable node keyed by id. The three adjacency maps are
- * derived from each node's `children`:
- *   - `childrenByNode` — forward edges, preserving `children` order.
- *   - `parentsByNode` — reverse edges restricted to `node:` children.
- *   - `pageParents` — reverse edges restricted to `page:` children, keyed by
- *     page slug.
- *
- * `root` is the entry-point node id (`_root` when present). Dangling refs (a
- * `node:`/`page:` target with no on-disk file) are retained throughout —
- * validation, not the index build, is responsible for surfacing them.
- */
-export interface TreeIndex {
-  nodes: Map<string, TreeNode>;
-  childrenByNode: Map<string, ReadonlyArray<ChildRef>>;
-  parentsByNode: Map<string, Set<string>>;
-  pageParents: Map<string, Set<string>>;
-  root: string;
-}
-
-interface CachedIndex {
-  workspaceDir: string;
-  index: TreeIndex;
-}
-
-let cache: CachedIndex | null = null;
-
-/**
- * Parse a raw `children` entry into a {@link ChildRef}. Returns `null` for any
- * entry that does not carry a recognized `page:` / `node:` prefix or whose ref
- * body is empty — those are malformed and dropped (with a warn) rather than
- * faithfully threaded through adjacency.
- */
-function parseChildRef(raw: string): ChildRef | null {
-  if (raw.startsWith(PAGE_REF_PREFIX)) {
-    const ref = raw.slice(PAGE_REF_PREFIX.length);
-    return ref.length > 0 ? { kind: "page", ref } : null;
-  }
-  if (raw.startsWith(NODE_REF_PREFIX)) {
-    const ref = raw.slice(NODE_REF_PREFIX.length);
-    return ref.length > 0 ? { kind: "node", ref } : null;
-  }
-  return null;
-}
-
-/** Append `parent` to the parent-set for `key`, creating the set on demand. */
-function addParent(
-  map: Map<string, Set<string>>,
-  key: string,
-  parent: string,
-): void {
-  let parents = map.get(key);
-  if (!parents) {
-    parents = new Set();
-    map.set(key, parents);
-  }
-  parents.add(parent);
-}
-
-/**
- * Pick the root node id from the materialized adjacency. Prefers the reserved
- * {@link ROOT_NODE_ID} when a node with that id exists. Otherwise the root is
- * the single node with no parents; if several nodes are parentless the choice
- * is ambiguous, so warn and pick the ASCII-smallest id for determinism. With no
- * nodes at all the root is `_root` (the well-known handle a migration authors
- * first), matching the empty-workspace contract.
- */
-function pickRoot(
-  nodes: Map<string, TreeNode>,
-  parentsByNode: Map<string, Set<string>>,
-): string {
-  if (nodes.has(ROOT_NODE_ID)) {
-    return ROOT_NODE_ID;
-  }
-
-  const parentless = [...nodes.keys()].filter(
-    (id) => !parentsByNode.has(id) || parentsByNode.get(id)!.size === 0,
-  );
-  parentless.sort();
-
-  if (parentless.length === 1) {
-    return parentless[0];
-  }
-  if (parentless.length === 0) {
-    return ROOT_NODE_ID;
-  }
-  log.warn(
-    { parentless },
-    "Ambiguous tree root — no '_root' node and multiple parentless nodes; picking ASCII-smallest deterministically",
-  );
-  return parentless[0];
-}
-
-/**
- * Return a `TreeIndex` for `workspaceDir`. Cached module-locally; the cache is
- * invalidated by `invalidateTreeIndex` (called by `tree-store.ts` hooks when
- * nodes change).
- *
- * Cold builds list every node and read them in parallel, dropping any whose
- * read rejects with a warn so one broken node never blocks the rest of the
- * index. Each readable node's `children` is parsed into {@link ChildRef}s and
- * threaded into forward (`childrenByNode`) and reverse (`parentsByNode` /
- * `pageParents`) adjacency. The build is structural only — referenced
- * pages/nodes are never verified to exist; dangling refs are retained for a
- * later validation pass.
- */
-export async function getTreeIndex(workspaceDir: string): Promise<TreeIndex> {
-  if (cache && cache.workspaceDir === workspaceDir) {
-    return cache.index;
-  }
-
-  const ids = await listNodes(workspaceDir);
-
-  // Read every node in parallel; nodes whose read rejects are dropped with a
-  // warn so a single broken node never blocks the rest of the index.
-  const settled = await Promise.allSettled(
-    ids.map((id) => readNode(workspaceDir, id)),
-  );
-
-  const nodes = new Map<string, TreeNode>();
-  const childrenByNode = new Map<string, ReadonlyArray<ChildRef>>();
-  const parentsByNode = new Map<string, Set<string>>();
-  const pageParents = new Map<string, Set<string>>();
-
-  for (let i = 0; i < settled.length; i++) {
-    const result = settled[i];
-    const id = ids[i];
-    if (result.status === "rejected") {
-      log.warn(
-        { id, err: result.reason },
-        "Dropping tree node from index — read failed",
-      );
-      continue;
-    }
-    const node = result.value;
-    // `readNode` returns null only on ENOENT; a node listed by `listNodes`
-    // that vanishes between list and read is a benign race — drop it silently.
-    if (!node) continue;
-    nodes.set(id, node);
-  }
-
-  // Build adjacency in a second pass so every node is registered first — that
-  // keeps a deterministic, list-order iteration independent of read timing.
-  for (const node of nodes.values()) {
-    const childRefs: ChildRef[] = [];
-    for (const raw of node.frontmatter.children) {
-      const parsed = parseChildRef(raw);
-      if (!parsed) {
-        log.warn(
-          { id: node.id, raw },
-          "Dropping malformed child ref — expected 'page:<slug>' or 'node:<id>'",
-        );
-        continue;
-      }
-      childRefs.push(parsed);
-      const reverse = parsed.kind === "node" ? parentsByNode : pageParents;
-      addParent(reverse, parsed.ref, node.id);
-    }
-    childrenByNode.set(node.id, childRefs);
-  }
-
-  const root = pickRoot(nodes, parentsByNode);
-
-  const index: TreeIndex = {
-    nodes,
-    childrenByNode,
-    parentsByNode,
-    pageParents,
-    root,
-  };
-  cache = { workspaceDir, index };
-  return index;
-}
-
-/**
- * Clear the cached index. Pass `workspaceDir` to scope invalidation to a
- * specific cache entry; omit it to clear unconditionally.
- */
-export function invalidateTreeIndex(workspaceDir?: string): void {
-  if (!cache) return;
-  if (workspaceDir === undefined || cache.workspaceDir === workspaceDir) {
-    cache = null;
-  }
-}

package/src/memory/v3/tree-store.ts

@@ -1,394 +0,0 @@
-/**
- * Memory v3 — Tree node store.
- *
- * Owns the on-disk read/write contract for `memory/tree/<id>.md`. Nodes may
- * live directly under `memory/tree/` or nested in subdirectories (e.g.
- * `memory/tree/people/colleagues.md`); the id encodes the relative path from
- * `tree/` minus the `.md` extension, using forward slashes as separators (so
- * `people/colleagues` is a valid id).
- *
- * The v3 tree is a DAG *overlay* over the existing flat `memory/concepts/`
- * pages — this module never touches `memory/concepts/`. Pages stay canonical
- * and shared; nodes reference pages and sub-nodes by `children` refs
- * (`page:<slug>` / `node:<id>`), which are the portable replacement for
- * filesystem symlinks.
- *
- * Each node is a YAML-frontmatter Markdown file: a `---`-delimited block
- * (`children`, optional `routing_hints` / `summary`) followed by the prose body
- * that is the node's full self-description. This module is the only v3
- * component that knows how to parse or render that format — every other v3
- * module routes through `readNode` / `writeNode` so the on-disk shape can
- * evolve without touching downstream callers.
- *
- * Writes are atomic (temp + rename) so a crash mid-write leaves either the old
- * file or the new file in place — never a half-written node. The id machinery
- * mirrors v2's page-store `slugify` / `validateSlug` so node ids and page slugs
- * share the same filesystem-safe shape.
- */
-
-import { randomUUID } from "node:crypto";
-import {
-  mkdir,
-  readdir,
-  readFile,
-  rename,
-  rm,
-  stat,
-  writeFile,
-} from "node:fs/promises";
-import { dirname, join, relative, sep } from "node:path";
-
-import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
-
-import { FRONTMATTER_REGEX } from "../../skills/frontmatter.js";
-import { invalidateTreeIndex } from "./tree-index.js";
-import { type TreeNode, TreeNodeFrontmatterSchema } from "./types.js";
-
-/** Filename suffix for tree nodes. */
-const NODE_EXTENSION = ".md";
-
-/** Cap individual id-segment length so we stay well under filesystem limits. */
-const MAX_ID_SEGMENT_LENGTH = 80;
-
-/** Cap the full id (including any folder separators) to a sane bound. */
-const MAX_ID_TOTAL_LENGTH = 200;
-
-/** Each path segment must match this — same shape `slugify` produces. */
-const ID_SEGMENT_REGEX = /^[a-z0-9](?:[a-z0-9-]*)$/;
-
-/**
- * Reserved id for the root of the v3 tree. The root node is the entry point a
- * future migration authors first; reserving the id keeps the well-known handle
- * stable across the codebase.
- */
-export const ROOT_NODE_ID = "_root";
-
-/**
- * Convert an arbitrary input string into a filesystem-safe id **segment**.
- *
- * Returns a single path segment (no `/`). Path-shaped ids are constructed by
- * the authoring migration writing files at full paths; this helper is for
- * turning free-form text (e.g. a node label) into one clean segment.
- *
- * Rules:
- *   - Lowercase ASCII letters, digits, and hyphens only.
- *   - Non-ASCII / non-alphanumeric characters (including `/`) collapse to hyphens.
- *   - Consecutive hyphens collapse to one; leading/trailing hyphens trimmed.
- *   - Truncated to {@link MAX_ID_SEGMENT_LENGTH} characters (with trailing
- *     hyphen re-trimmed after truncation).
- *   - Empty inputs (e.g. emoji-only) fall back to `node-<random>` so the caller
- *     always gets a non-empty, write-safe segment.
- */
-export function slugify(input: string): string {
-  let slug = input
-    .toLowerCase()
-    .normalize("NFKD")
-    .replace(/[^a-z0-9-]+/g, "-")
-    .replace(/-{2,}/g, "-")
-    .replace(/^-+|-+$/g, "");
-
-  if (slug.length > MAX_ID_SEGMENT_LENGTH) {
-    slug = slug.slice(0, MAX_ID_SEGMENT_LENGTH).replace(/-+$/, "");
-  }
-
-  if (!slug) {
-    slug = `node-${randomUUID().slice(0, 8)}`;
-  }
-
-  return slug;
-}
-
-/**
- * Validate a node id — possibly path-shaped — that is about to cross the
- * storage boundary. Throws on any malformed or unsafe value.
- *
- * The on-disk tree treats ids as relative paths under `memory/tree/`. A
- * malformed id (e.g. `..`, leading `/`, embedded null byte) could escape that
- * root via `path.join` if it slipped through, so we enforce shape here at every
- * read/write/delete entry point rather than relying on callers.
- *
- * The reserved {@link ROOT_NODE_ID} (`_root`) is accepted as a special case;
- * its leading underscore would otherwise fail {@link ID_SEGMENT_REGEX}.
- *
- * Rules:
- *   - Non-empty, ≤ {@link MAX_ID_TOTAL_LENGTH} chars.
- *   - Each `/`-separated segment matches {@link ID_SEGMENT_REGEX}
- *     (lowercase alphanum + hyphen, no leading hyphen, ≤80 chars).
- *   - No `..` segments, no empty segments (`a//b`), no leading or trailing `/`.
- *   - No `\` (Windows separator), no null bytes, no whitespace, no non-ASCII.
- */
-export function validateNodeId(id: string): void {
-  if (typeof id !== "string" || id.length === 0) {
-    throw new Error(`Invalid tree-node id: empty`);
-  }
-  if (id === ROOT_NODE_ID) {
-    return;
-  }
-  if (id.length > MAX_ID_TOTAL_LENGTH) {
-    throw new Error(
-      `Invalid tree-node id: length ${id.length} exceeds max ${MAX_ID_TOTAL_LENGTH}: ${id}`,
-    );
-  }
-  if (id.includes("\\")) {
-    throw new Error(`Invalid tree-node id: backslash not allowed: ${id}`);
-  }
-  if (id.includes("\0")) {
-    throw new Error(`Invalid tree-node id: null byte not allowed`);
-  }
-  if (/\s/.test(id)) {
-    throw new Error(`Invalid tree-node id: whitespace not allowed: ${id}`);
-  }
-  if (id.startsWith("/") || id.endsWith("/")) {
-    throw new Error(
-      `Invalid tree-node id: leading or trailing '/' not allowed: ${id}`,
-    );
-  }
-  const segments = id.split("/");
-  for (const segment of segments) {
-    if (segment.length === 0) {
-      throw new Error(`Invalid tree-node id: empty path segment: ${id}`);
-    }
-    if (segment === "..") {
-      throw new Error(`Invalid tree-node id: '..' segment not allowed: ${id}`);
-    }
-    if (segment.length > MAX_ID_SEGMENT_LENGTH) {
-      throw new Error(
-        `Invalid tree-node id: segment '${segment}' exceeds max ${MAX_ID_SEGMENT_LENGTH} chars: ${id}`,
-      );
-    }
-    if (!ID_SEGMENT_REGEX.test(segment)) {
-      throw new Error(
-        `Invalid tree-node id: segment '${segment}' must match [a-z0-9][a-z0-9-]*: ${id}`,
-      );
-    }
-  }
-}
-
-// ---------------------------------------------------------------------------
-// Path helpers
-// ---------------------------------------------------------------------------
-
-export function getTreeDir(workspaceDir: string): string {
-  return join(workspaceDir, "memory", "tree");
-}
-
-/**
- * Resolve the absolute path for a node id. Ids may contain `/` to indicate
- * folder hierarchy under `memory/tree/`; `path.join` handles those correctly
- * on POSIX, and `validateNodeId` (called at every public entry point) rejects
- * shapes that could escape the tree root.
- */
-function getNodePath(workspaceDir: string, id: string): string {
-  return join(getTreeDir(workspaceDir), `${id}${NODE_EXTENSION}`);
-}
-
-/**
- * Compute the id for a tree-node file, given the tree root and the absolute
- * file path. Returns the path-relative location with `.md` stripped and
- * platform separators normalized to `/`. Tolerant of paths that don't end in
- * `.md` so callers walking arbitrary content can use it defensively.
- */
-function idFromNodePath(treeRoot: string, filePath: string): string {
-  const rel = relative(treeRoot, filePath);
-  const withoutExt = rel.endsWith(NODE_EXTENSION)
-    ? rel.slice(0, -NODE_EXTENSION.length)
-    : rel;
-  return sep === "/" ? withoutExt : withoutExt.split(sep).join("/");
-}
-
-// ---------------------------------------------------------------------------
-// Frontmatter parse / render
-// ---------------------------------------------------------------------------
-
-/**
- * Split raw file contents into (frontmatter, body). If no frontmatter block is
- * present the entire input is treated as body and an empty frontmatter block is
- * returned (validated by `TreeNodeFrontmatterSchema` so any unexpected shape —
- * bad types, extra junk — surfaces as a parse error to the caller, not silent
- * dropped data).
- *
- * The schema's default guarantees `children` is always an array even on a
- * freshly created node with empty frontmatter.
- */
-function parseNodeContent(raw: string): {
-  frontmatter: TreeNode["frontmatter"];
-  body: string;
-} {
-  const match = raw.match(FRONTMATTER_REGEX);
-  if (!match) {
-    return {
-      frontmatter: TreeNodeFrontmatterSchema.parse({}),
-      body: raw,
-    };
-  }
-  const yamlBlock = match[1];
-  const body = raw.slice(match[0].length);
-  const parsed = parseYaml(yamlBlock) ?? {};
-  return {
-    frontmatter: TreeNodeFrontmatterSchema.parse(parsed),
-    body,
-  };
-}
-
-/**
- * Render a tree node back into the on-disk Markdown form. The output is always
- * frontmatter + body; even nodes with empty `children` keep the explicit YAML
- * key so callers see the canonical shape on round-trip.
- */
-export function renderNodeContent(node: TreeNode): string {
-  const frontmatter = TreeNodeFrontmatterSchema.parse(node.frontmatter);
-  const yamlBlock = stringifyYaml(frontmatter, { indent: 2 }).trimEnd();
-  return `---\n${yamlBlock}\n---\n${node.body}`;
-}
-
-// ---------------------------------------------------------------------------
-// Public API
-// ---------------------------------------------------------------------------
-
-/**
- * Read a single tree node. Returns `null` if the file does not exist.
- *
- * Any other read or parse failure (permission denied, malformed YAML,
- * frontmatter that fails schema validation) throws — unlike "missing", these
- * are programmer / data-corruption errors the caller needs to see.
- */
-export async function readNode(
-  workspaceDir: string,
-  id: string,
-): Promise<TreeNode | null> {
-  validateNodeId(id);
-  const path = getNodePath(workspaceDir, id);
-  let raw: string;
-  try {
-    raw = await readFile(path, "utf-8");
-  } catch (err) {
-    if ((err as NodeJS.ErrnoException).code === "ENOENT") {
-      return null;
-    }
-    throw err;
-  }
-  const { frontmatter, body } = parseNodeContent(raw);
-  return { id, frontmatter, body };
-}
-
-/**
- * File mtime for a tree node, in epoch ms. Returns 0 when the file is missing
- * or unreadable — callers treat 0 as "no mtime" (e.g. the validator's stale-
- * index check reads a missing node as the oldest possible mtime so it never
- * spuriously flags a parent against an absent child). Mirrors v2's
- * `getPageMtimeMs`.
- */
-export async function getNodeMtimeMs(
-  workspaceDir: string,
-  id: string,
-): Promise<number> {
-  validateNodeId(id);
-  try {
-    const s = await stat(getNodePath(workspaceDir, id));
-    return s.mtimeMs;
-  } catch {
-    return 0;
-  }
-}
-
-/**
- * Write a tree node atomically (temp file + rename). A crash between the temp
- * write and the rename leaves the prior file intact; a crash after the rename
- * leaves the new file. Readers therefore never observe a partial node.
- *
- * Parent directories are created on demand (`mkdir -p`) so nested-folder ids
- * like `people/colleagues` work without callers pre-creating the folder.
- */
-export async function writeNode(
-  workspaceDir: string,
-  node: TreeNode,
-): Promise<void> {
-  validateNodeId(node.id);
-  const path = getNodePath(workspaceDir, node.id);
-  const tmpPath = `${path}.tmp.${process.pid}.${randomUUID()}`;
-  const content = renderNodeContent(node);
-  try {
-    await mkdir(dirname(path), { recursive: true });
-    await writeFile(tmpPath, content, "utf-8");
-    await rename(tmpPath, path);
-  } catch (err) {
-    // Best-effort cleanup: if the rename failed (or the write succeeded but the
-    // rename did not), remove the orphan tmp file so we don't leak it into the
-    // tree/ directory where listNodes would then surface it.
-    await rm(tmpPath, { force: true }).catch(() => {});
-    throw err;
-  }
-  invalidateTreeIndex(workspaceDir);
-}
-
-/**
- * List every tree-node id present on disk, walking subdirectories.
- *
- * Ids are returned in path-relative form with forward slashes as separators
- * (e.g. `people/colleagues`) so callers can pass them straight back to
- * `readNode`.
- *
- * Hidden directories (segment starts with `.`), non-`.md` files, and atomic-
- * write temp files (`.tmp.<pid>.<uuid>`) are skipped. If the tree/ directory
- * does not yet exist (fresh workspace pre-migration), returns `[]`.
- */
-export async function listNodes(workspaceDir: string): Promise<string[]> {
-  const root = getTreeDir(workspaceDir);
-  const ids: string[] = [];
-  const queue: string[] = [root];
-
-  while (queue.length > 0) {
-    const dir = queue.shift()!;
-    let entries;
-    try {
-      entries = await readdir(dir, { withFileTypes: true });
-    } catch (err) {
-      if ((err as NodeJS.ErrnoException).code === "ENOENT") {
-        // Root missing → return []. Nested missing dir is impossible mid-walk
-        // (we only enqueue what readdir surfaced) but treat the same defensively.
-        if (dir === root) return [];
-        continue;
-      }
-      throw err;
-    }
-
-    for (const entry of entries) {
-      if (entry.name.startsWith(".")) continue;
-      const fullPath = join(dir, entry.name);
-      if (entry.isDirectory()) {
-        queue.push(fullPath);
-        continue;
-      }
-      if (!entry.isFile()) continue;
-      if (!entry.name.endsWith(NODE_EXTENSION)) continue;
-      // Skip orphaned temp files left behind by a crashed atomic write.
-      if (entry.name.includes(".tmp.")) continue;
-      ids.push(idFromNodePath(root, fullPath));
-    }
-  }
-
-  ids.sort();
-  return ids;
-}
-
-/**
- * Delete a tree node. Idempotent — missing files are not an error.
- *
- * Any other failure (permission denied, etc.) throws so the caller can react.
- */
-export async function deleteNode(
-  workspaceDir: string,
-  id: string,
-): Promise<void> {
-  validateNodeId(id);
-  const path = getNodePath(workspaceDir, id);
-  try {
-    await rm(path);
-  } catch (err) {
-    if ((err as NodeJS.ErrnoException).code === "ENOENT") {
-      return;
-    }
-    throw err;
-  }
-  invalidateTreeIndex(workspaceDir);
-}