@vellumai/assistant 0.6.6 → 0.7.0

package/src/memory/v2/consolidation-job.ts

@@ -0,0 +1,304 @@
+/**
+ * Memory v2 — `memory_v2_consolidate` job handler.
+ *
+ * The consolidation job is the centerpiece of v2: an hourly background pass
+ * that routes accumulated `memory/buffer.md` entries into concept pages,
+ * rewrites `memory/recent.md`, promotes new essentials/threads, and trims the
+ * buffer down to entries that arrived after the run started.
+ *
+ * Unlike `sweep`, consolidation runs as the assistant: `wakeAgentForOpportunity()`
+ * loads the standard system prompt (SOUL.md + IDENTITY.md + persona + memory/*
+ * autoloads) and the standard tool surface (read_file, write_file, edit_file,
+ * list_files, bash). The hint string carries the prompt body from §10 of the
+ * design doc with the cutoff timestamp templated in. Care, judgment, and the
+ * assistant's voice are the point — there is no "consolidator persona" to
+ * substitute in.
+ *
+ * Lifecycle:
+ *   1. Bail if the `memory-v2-enabled` feature flag is off (the worker may
+ *      have claimed a stale row at flag-flip time).
+ *   2. Acquire a single-process lock at `memory/.v2-state/consolidation.lock`
+ *      so two overlapping schedule windows can't fight over the same files.
+ *      The lock contains the holder's PID + timestamp so a crashed run leaves
+ *      a diagnosable trace.
+ *   3. Capture the cutoff timestamp at dispatch. Any buffer entry timestamped
+ *      at or after the cutoff arrived AFTER the run started — leave it for
+ *      the next pass.
+ *   4. Read `memory/buffer.md`. Bail if empty (no work to do, but the lock
+ *      and skip path still log so operators can confirm the schedule fired).
+ *   5. Bootstrap a background conversation (mirrors `runUpdateBulletinJobIfNeeded`)
+ *      and call `wakeAgentForOpportunity()` with the templated hint. The wake
+ *      reuses the assistant's full system prompt + tools.
+ *   6. On wake success, enqueue `memory_v2_rebuild_edges` (regenerate
+ *      frontmatter from `edges.json`) and `memory_v2_reembed` (re-index any
+ *      pages the agent touched). Tracking touched pages via mtime would be
+ *      more precise but is fragile across filesystems; the embedder's
+ *      content-hash cache makes a conservative full-reembed effectively free.
+ *      On wake failure no follow-ups are enqueued — the agent didn't run, so
+ *      there's nothing to regenerate or re-embed.
+ *   7. Release the lock.
+ *
+ * The handler never propagates a wake exception: it logs, cleans up the
+ * orphan conversation, releases the lock, and returns `wake_failed` so the
+ * next scheduled run can re-attempt. A thrown bootstrap error bubbles up and
+ * the jobs-worker treats it as a retryable failure.
+ */
+
+import {
+  closeSync,
+  mkdirSync,
+  openSync,
+  readFileSync,
+  unlinkSync,
+  writeSync,
+} from "node:fs";
+import { dirname, join } from "node:path";
+
+import { isAssistantFeatureFlagEnabled } from "../../config/assistant-feature-flags.js";
+import type { AssistantConfig } from "../../config/types.js";
+import { wakeAgentForOpportunity } from "../../runtime/agent-wake.js";
+import { getLogger } from "../../util/logger.js";
+import { getWorkspaceDir } from "../../util/platform.js";
+import { bootstrapConversation } from "../conversation-bootstrap.js";
+import { deleteConversation } from "../conversation-crud.js";
+import {
+  enqueueMemoryJob,
+  type MemoryJob,
+  type MemoryJobType,
+} from "../jobs-store.js";
+import { renderConsolidationPrompt } from "./prompts/consolidation.js";
+
+const log = getLogger("memory-v2-consolidate");
+
+/** Source string identifying this wake in `agent-wake` logs and surfaces. */
+const WAKE_SOURCE = "memory_v2_consolidation";
+
+/**
+ * Follow-up jobs to fan out after a successful consolidation. Both are stubs
+ * from PR 6 today; PR 21 will replace them with real handlers.
+ *
+ * Conservatively re-embeds every page rather than tracking which pages the
+ * agent touched: mtime-diffing is fragile across filesystems, and the
+ * embedder's content-hash cache makes unchanged pages effectively free.
+ */
+const FOLLOW_UP_JOB_TYPES: readonly MemoryJobType[] = [
+  "memory_v2_rebuild_edges",
+  "memory_v2_reembed",
+] as const;
+
+/**
+ * Job handler. See file header for the full lifecycle. Returns a discriminated
+ * union so tests can assert on the path taken (flag-off / locked / empty /
+ * invoked) without having to spy on the filesystem.
+ */
+export type ConsolidationOutcome =
+  | { kind: "flag_off" }
+  | { kind: "locked"; holder: string }
+  | { kind: "empty_buffer" }
+  | { kind: "wake_failed"; reason?: string }
+  | {
+      kind: "invoked";
+      conversationId: string;
+      cutoff: string;
+      followUpJobIds: string[];
+    };
+
+export async function memoryV2ConsolidateJob(
+  _job: MemoryJob,
+  config: AssistantConfig,
+): Promise<ConsolidationOutcome> {
+  if (!isAssistantFeatureFlagEnabled("memory-v2-enabled", config)) {
+    log.debug("memory-v2-enabled flag off; consolidation skipped");
+    return { kind: "flag_off" };
+  }
+
+  const memoryDir = join(getWorkspaceDir(), "memory");
+  const lockPath = join(memoryDir, ".v2-state", "consolidation.lock");
+  const bufferPath = join(memoryDir, "buffer.md");
+
+  // Step 1: acquire lock. Bails immediately if another consolidation is
+  // already in flight — the next scheduled run can pick up where we leave off.
+  const holder = tryAcquireLock(lockPath);
+  if (holder !== null) {
+    log.warn({ lockPath, holder }, "consolidation skipped: lock already held");
+    return { kind: "locked", holder };
+  }
+
+  try {
+    // Step 2: capture cutoff. ISO-8601 is the convention; it's a total order
+    // string that compares correctly via lexicographic <, which is all the
+    // prompt asks the agent to do. Captured here (not at enqueue time) so
+    // late-claimed rows still get a fresh cutoff.
+    const cutoff = new Date().toISOString();
+
+    // Step 3: bail on empty buffer. Nothing for the agent to consolidate.
+    // The lock is released in finally below.
+    const bufferContent = readBufferContent(bufferPath);
+    if (bufferContent.trim().length === 0) {
+      log.debug("buffer.md empty; consolidation skipped");
+      return { kind: "empty_buffer" };
+    }
+
+    // Step 4: bootstrap a background conversation and wake the assistant
+    // with the cutoff-templated prompt. Mirrors the UPDATES.md pattern in
+    // `runUpdateBulletinJobIfNeeded` — the wake runs `mainAgent` against
+    // the assistant's full system prompt, so consolidation thinks and
+    // writes in the assistant's voice.
+    const conversation = bootstrapConversation({
+      conversationType: "background",
+      source: WAKE_SOURCE,
+      origin: "memory_consolidation",
+      systemHint: "Running memory consolidation",
+      groupId: "system:background",
+    });
+
+    let wakeInvoked = false;
+    let failureReason: string | undefined;
+    try {
+      const result = await wakeAgentForOpportunity({
+        conversationId: conversation.id,
+        hint: renderConsolidationPrompt(cutoff),
+        source: WAKE_SOURCE,
+      });
+      wakeInvoked = result.invoked;
+      failureReason = result.reason;
+    } catch (err) {
+      failureReason = err instanceof Error ? err.message : String(err);
+      log.error(
+        { err, conversationId: conversation.id },
+        "consolidation wake threw; cleaning up and re-enqueuing follow-ups skipped",
+      );
+    }
+
+    // If the wake never ran (resolver missing, conversation archived,
+    // timeout, exception), clean up the orphan background conversation —
+    // matches the cleanup logic in `runUpdateBulletinJobIfNeeded`. We
+    // do NOT enqueue follow-ups in this branch because no pages changed.
+    if (!wakeInvoked) {
+      try {
+        deleteConversation(conversation.id);
+      } catch (err) {
+        log.warn(
+          { err, conversationId: conversation.id },
+          "consolidation: failed to delete orphan background conversation; continuing",
+        );
+      }
+      return failureReason !== undefined
+        ? { kind: "wake_failed", reason: failureReason }
+        : { kind: "wake_failed" };
+    }
+
+    // Step 5: enqueue follow-up jobs. Enqueueing now keeps the dispatch
+    // wiring exercised end-to-end so PR 21 only has to swap in the handler
+    // bodies.
+    const followUpJobIds: string[] = [];
+    for (const jobType of FOLLOW_UP_JOB_TYPES) {
+      try {
+        followUpJobIds.push(enqueueMemoryJob(jobType, {}));
+      } catch (err) {
+        // Best-effort: a failed enqueue here doesn't undo the agent's writes,
+        // and the next scheduled consolidation will attempt the same fan-out.
+        log.warn(
+          { err, jobType },
+          "consolidation: failed to enqueue follow-up job; continuing",
+        );
+      }
+    }
+
+    log.info(
+      {
+        conversationId: conversation.id,
+        cutoff,
+        followUpJobIds,
+      },
+      "consolidation invoked",
+    );
+    return {
+      kind: "invoked",
+      conversationId: conversation.id,
+      cutoff,
+      followUpJobIds,
+    };
+  } finally {
+    releaseLock(lockPath);
+  }
+}
+
+/**
+ * Read `memory/buffer.md`. Missing file → empty string so the skip-on-empty
+ * branch doesn't have to distinguish "no file" from "blank file".
+ */
+function readBufferContent(bufferPath: string): string {
+  try {
+    return readFileSync(bufferPath, "utf-8");
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return "";
+    throw err;
+  }
+}
+
+/**
+ * Atomically create the lock file with `wx` (O_CREAT | O_EXCL) flags. Returns
+ * `null` on success, or the current holder string (file contents, typically
+ * `pid timestamp`) when the file already exists — the holder is surfaced for
+ * log diagnostics so operators can identify a stuck lock without re-reading.
+ *
+ * Crash recovery: if the prior daemon died with the lock held, the file will
+ * still be on disk on the next start. PR 20 keeps the lock simple per the
+ * plan instructions; a future iteration can probe liveness via `kill(pid, 0)`
+ * the way `snapshot-lock.ts` does. Until then, an operator can clear a
+ * stale lock by removing the file.
+ */
+function tryAcquireLock(lockPath: string): string | null {
+  // The workspace migration seeds `memory/.v2-state/`, but tests and
+  // ad-hoc workspaces may not have it yet. `mkdirSync({ recursive: true })`
+  // is idempotent, so the call is cheap when the dir already exists.
+  mkdirSync(dirname(lockPath), { recursive: true });
+
+  let fd: number;
+  try {
+    // `wx` = create-if-not-exists, fail with EEXIST if it does.
+    fd = openSync(lockPath, "wx");
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
+    try {
+      return readFileSync(lockPath, "utf-8").trim() || "unknown";
+    } catch {
+      return "unknown";
+    }
+  }
+
+  // Best-effort PID + timestamp payload so a stale lock can be diagnosed.
+  // The worker only cares that the file exists; the contents are advisory.
+  try {
+    writeSync(fd, `${process.pid} ${Date.now()}\n`);
+  } catch {
+    // best-effort
+  } finally {
+    try {
+      closeSync(fd);
+    } catch {
+      // best-effort
+    }
+  }
+  return null;
+}
+
+/**
+ * Idempotent unlink of the lock file. Called from the `finally` block so a
+ * crash in the wake path doesn't leave the lock stranded. ENOENT is swallowed
+ * because the lock may have been released by an operator or never created
+ * (acquire failed before reaching the lock-write step).
+ */
+function releaseLock(lockPath: string): void {
+  try {
+    unlinkSync(lockPath);
+  } catch (err) {
+    const code = (err as NodeJS.ErrnoException).code;
+    if (code === "ENOENT") return;
+    log.warn(
+      { err, lockPath },
+      "consolidation: failed to release lock (best-effort)",
+    );
+  }
+}

package/src/memory/v2/edges.ts

@@ -0,0 +1,217 @@
+/**
+ * Memory v2 — `memory/edges.json` read/write and neighbor resolution.
+ *
+ * This module is the sole owner of the `memory/edges.json` file. The edges
+ * index is the source of truth for v2 graph topology — concept-page
+ * frontmatter only mirrors a derived view of it.
+ *
+ * Edges are unweighted, undirected, and stored as canonicalized 2-tuples
+ * (alphabetically-first slug first). Self-loops are rejected at write time.
+ *
+ * Writes are atomic (write to `<file>.tmp-<uuid>`, then `rename(2)` onto the
+ * destination) so a crashed writer never leaves a torn JSON file behind —
+ * readers always see either the previous or the new index.
+ */
+import { randomUUID } from "node:crypto";
+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+
+import type { EdgesIndex } from "./types.js";
+import { EdgesIndexSchema } from "./types.js";
+
+const EDGES_FILENAME = "edges.json";
+
+/** Path to `memory/edges.json` inside `workspaceDir`. */
+function edgesPath(workspaceDir: string): string {
+  return join(workspaceDir, "memory", EDGES_FILENAME);
+}
+
+/**
+ * Canonicalize a single tuple so `[a, b]` and `[b, a]` collapse to the same
+ * representation. Keeps writes deterministic regardless of caller order.
+ */
+function canonicalTuple(a: string, b: string): [string, string] {
+  return a <= b ? [a, b] : [b, a];
+}
+
+/** Stable string key for a canonical tuple, used for de-duplication. */
+function tupleKey(t: readonly [string, string]): string {
+  return `${t[0]}\u0000${t[1]}`;
+}
+
+/**
+ * Read `memory/edges.json` and validate it against `EdgesIndexSchema`.
+ * Returns the canonical empty index when the file is missing.
+ *
+ * Any JSON parse error or schema-validation failure throws — the file should
+ * never be silently quarantined, since a broken edges.json signals a bug in
+ * a writer or external tampering.
+ */
+export async function readEdges(workspaceDir: string): Promise<EdgesIndex> {
+  let raw: string;
+  try {
+    raw = await readFile(edgesPath(workspaceDir), "utf-8");
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") {
+      return { version: 1, edges: [] };
+    }
+    throw err;
+  }
+  const parsed = JSON.parse(raw);
+  return EdgesIndexSchema.parse(parsed);
+}
+
+/**
+ * Write `memory/edges.json` atomically. The on-disk tuples are canonicalized
+ * and de-duplicated before serialization, so the output is independent of
+ * insertion order and free of redundant pairs.
+ *
+ * Atomicity: bytes are written to a sibling temp file then renamed onto the
+ * destination. `rename(2)` on the same filesystem is atomic, so a reader
+ * concurrent with the writer sees either the prior file or the new one in
+ * full — never a partial write.
+ */
+export async function writeEdges(
+  workspaceDir: string,
+  idx: EdgesIndex,
+): Promise<void> {
+  const canonical = canonicalizeIndex(idx);
+  const finalPath = edgesPath(workspaceDir);
+  await mkdir(dirname(finalPath), { recursive: true });
+  const tmpPath = `${finalPath}.tmp-${randomUUID()}`;
+  const body = `${JSON.stringify(canonical, null, 2)}\n`;
+  await writeFile(tmpPath, body, "utf-8");
+  await rename(tmpPath, finalPath);
+}
+
+/**
+ * Pure helper: return a new index with `(a, b)` added (canonicalized). If the
+ * edge already exists the index is returned unchanged. Self-loops are
+ * rejected — concept-page graphs are simple graphs.
+ */
+export function addEdge(idx: EdgesIndex, a: string, b: string): EdgesIndex {
+  if (a === b) {
+    throw new Error(`addEdge: refusing to add self-loop on slug "${a}"`);
+  }
+  const tuple = canonicalTuple(a, b);
+  const key = tupleKey(tuple);
+  for (const existing of idx.edges) {
+    if (tupleKey(canonicalTuple(existing[0], existing[1])) === key) {
+      return idx;
+    }
+  }
+  return { version: idx.version, edges: [...idx.edges, tuple] };
+}
+
+/**
+ * Pure helper: return a new index with `(a, b)` removed (matched on its
+ * canonical form). No-op if the edge is not present.
+ */
+export function removeEdge(idx: EdgesIndex, a: string, b: string): EdgesIndex {
+  const target = tupleKey(canonicalTuple(a, b));
+  const filtered = idx.edges.filter(
+    ([x, y]) => tupleKey(canonicalTuple(x, y)) !== target,
+  );
+  if (filtered.length === idx.edges.length) return idx;
+  return { version: idx.version, edges: filtered };
+}
+
+/**
+ * Iterative BFS over the undirected edge graph starting at `slug`. Returns
+ * every slug reachable within `hops` edges, *excluding* the start slug. An
+ * orphan node (or unknown slug) yields the empty set.
+ *
+ * `hops` is clamped at 0 — a non-positive value collapses to an immediate
+ * empty result so callers never need to special-case it.
+ */
+export function getNeighbors(
+  idx: EdgesIndex,
+  slug: string,
+  hops: number,
+): Set<string> {
+  const result = new Set<string>();
+  if (hops <= 0) return result;
+
+  const adjacency = buildAdjacency(idx);
+  const visited = new Set<string>([slug]);
+  let frontier: string[] = [slug];
+
+  for (let depth = 0; depth < hops && frontier.length > 0; depth++) {
+    const next: string[] = [];
+    for (const node of frontier) {
+      const neighbors = adjacency.get(node);
+      if (!neighbors) continue;
+      for (const neighbor of neighbors) {
+        if (visited.has(neighbor)) continue;
+        visited.add(neighbor);
+        result.add(neighbor);
+        next.push(neighbor);
+      }
+    }
+    frontier = next;
+  }
+
+  return result;
+}
+
+/**
+ * Validate an edges index against a known set of concept-page slugs. Returns
+ * the unique list of endpoints that are referenced by an edge but missing
+ * from `knownSlugs`. `ok` mirrors `missing.length === 0` for caller
+ * convenience.
+ */
+export function validateEdges(
+  idx: EdgesIndex,
+  knownSlugs: Set<string>,
+): { ok: boolean; missing: string[] } {
+  const missing = new Set<string>();
+  for (const [a, b] of idx.edges) {
+    if (!knownSlugs.has(a)) missing.add(a);
+    if (!knownSlugs.has(b)) missing.add(b);
+  }
+  const list = [...missing].sort();
+  return { ok: list.length === 0, missing: list };
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Canonicalize tuples + dedup, returning a deterministically-sorted index.
+ * Self-loops are dropped silently here (writer-side) so a malformed in-memory
+ * value can still be persisted as a clean file.
+ */
+function canonicalizeIndex(idx: EdgesIndex): EdgesIndex {
+  const seen = new Map<string, [string, string]>();
+  for (const [a, b] of idx.edges) {
+    if (a === b) continue;
+    const tuple = canonicalTuple(a, b);
+    seen.set(tupleKey(tuple), tuple);
+  }
+  // Sort by canonical key (NUL-separated) — gives a lexicographic order on
+  // the (left, right) pair without a multi-clause comparator.
+  const tuples = [...seen.entries()]
+    .sort(([keyX], [keyY]) => (keyX < keyY ? -1 : keyX > keyY ? 1 : 0))
+    .map(([, tuple]) => tuple);
+  return { version: idx.version, edges: tuples };
+}
+
+/** Build slug → neighbors map from canonicalized tuples (undirected). */
+function buildAdjacency(idx: EdgesIndex): Map<string, Set<string>> {
+  const adjacency = new Map<string, Set<string>>();
+  const ensure = (slug: string): Set<string> => {
+    let set = adjacency.get(slug);
+    if (!set) {
+      set = new Set<string>();
+      adjacency.set(slug, set);
+    }
+    return set;
+  };
+  for (const [a, b] of idx.edges) {
+    if (a === b) continue;
+    ensure(a).add(b);
+    ensure(b).add(a);
+  }
+  return adjacency;
+}