@vellumai/assistant 0.6.6 → 0.7.1

package/src/memory/v2/backfill-jobs.ts

@@ -0,0 +1,357 @@
+// ---------------------------------------------------------------------------
+// Memory v2 — Backfill job handlers
+// ---------------------------------------------------------------------------
+//
+// Three operator-triggered backfills, all wired through the same job queue so
+// they can be enqueued from the IPC route, the CLI, or recovery paths:
+//
+//   - `memory_v2_migrate`              — one-shot v1→v2 synthesis (PR 16).
+//   - `memory_v2_reembed`              — fan out an `embed_concept_page` job
+//     per slug, plus four reserved-slug jobs for the meta files
+//     (`__essentials__`, `__threads__`, `__recent__`, `__buffer__`).
+//   - `memory_v2_activation_recompute` — recompute persisted activation
+//     state for every conversation, no rendering. Used after consolidation
+//     replaces or deletes pages that other conversations still reference.
+//
+// Each handler is intentionally small — heavy lifting lives in the modules
+// they delegate to (`migration.ts`, `page-store.ts`, `embed-concept-page.ts`,
+// `activation.ts`, `activation-store.ts`). Keeping the wrappers thin means
+// the same code paths exercised by tests of those modules run unchanged when
+// a backfill kicks them off.
+
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import type { AssistantConfig } from "../../config/types.js";
+import { getLogger } from "../../util/logger.js";
+import { getWorkspaceDir } from "../../util/platform.js";
+import { getMessages } from "../conversation-crud.js";
+import { listConversations } from "../conversation-queries.js";
+import { getDb } from "../db-connection.js";
+import { enqueueEmbedConceptPageJob } from "../jobs/embed-concept-page.js";
+import type { MemoryJob } from "../jobs-store.js";
+import {
+  computeOwnActivation,
+  selectCandidates,
+  spreadActivation,
+} from "./activation.js";
+import { hydrate, save } from "./activation-store.js";
+import { getEdgeIndex } from "./edge-index.js";
+import {
+  MigrationAlreadyAppliedError,
+  runMemoryV2Migration,
+} from "./migration.js";
+import { listPages } from "./page-store.js";
+
+const log = getLogger("memory-v2-backfill");
+
+/**
+ * Reserved slugs the reembed job enqueues alongside the concept-page slugs.
+ * These name the four prose meta files (essentials/threads/recent/buffer)
+ * loaded into the system prompt by PR 11. Embedding them is forward-looking
+ * — the existing `embed-concept-page` handler treats unknown slugs as
+ * deletions (a no-op when no embedding exists), so enqueueing here is safe
+ * regardless of whether the meta files are ever embedded for retrieval.
+ */
+export const META_FILE_SLUGS = [
+  "__essentials__",
+  "__threads__",
+  "__recent__",
+  "__buffer__",
+] as const;
+
+// ---------------------------------------------------------------------------
+// memory_v2_migrate — wraps runMemoryV2Migration
+// ---------------------------------------------------------------------------
+
+/**
+ * Job handler: run the one-shot v1→v2 migration. Pass `{ force: true }` in the
+ * payload to overwrite an existing v2 state when the sentinel is already
+ * present (mirrors the CLI's `--force` flag in PR 25). Sentinel-gated re-runs
+ * surface as `MigrationAlreadyAppliedError` — the worker logs and treats them
+ * as a successful completion (no rethrow), so the job row clears without
+ * spinning the retry/deferral counters.
+ */
+export async function memoryV2MigrateJob(
+  job: MemoryJob,
+  config: AssistantConfig,
+): Promise<void> {
+  const force =
+    typeof job.payload.force === "boolean" ? job.payload.force : false;
+
+  try {
+    const result = await runMemoryV2Migration({
+      workspaceDir: getWorkspaceDir(),
+      database: getDb(),
+      force,
+      config,
+    });
+    log.info(
+      {
+        pagesCreated: result.pagesCreated,
+        edgesWritten: result.edgesWritten,
+        embedsEnqueued: result.embedsEnqueued,
+      },
+      "Memory v2 migration complete",
+    );
+  } catch (err) {
+    if (err instanceof MigrationAlreadyAppliedError) {
+      log.info(
+        "Memory v2 migration sentinel already present; skipping. Pass force: true to re-run.",
+      );
+      return;
+    }
+    throw err;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// memory_v2_reembed — fan out embed jobs for every page + meta file
+// ---------------------------------------------------------------------------
+
+/**
+ * Job handler: enqueue an `embed_concept_page` job per concept-page slug, plus
+ * one job per reserved meta-file slug ({@link META_FILE_SLUGS}).
+ *
+ * Returns the total number of jobs enqueued — `concept-page count + 4`.
+ * Callers (and tests) use the return value to assert progress without
+ * inspecting the job table directly.
+ */
+export async function memoryV2ReembedJob(
+  _job: MemoryJob,
+  _config: AssistantConfig,
+): Promise<number> {
+  const workspaceDir = getWorkspaceDir();
+  const slugs = await listPages(workspaceDir);
+
+  for (const slug of slugs) {
+    enqueueEmbedConceptPageJob({ slug });
+  }
+  for (const slug of META_FILE_SLUGS) {
+    enqueueEmbedConceptPageJob({ slug });
+  }
+
+  const total = slugs.length + META_FILE_SLUGS.length;
+  log.info(
+    { conceptPages: slugs.length, metaFiles: META_FILE_SLUGS.length, total },
+    "Memory v2 reembed enqueued",
+  );
+  return total;
+}
+
+// ---------------------------------------------------------------------------
+// memory_v2_activation_recompute — refresh persisted activation state
+// ---------------------------------------------------------------------------
+
+/**
+ * Hard cap on the number of conversations we touch per backfill run. Mirrors
+ * the v1 graph maintenance scheduler's bias toward bounded work: a workspace
+ * with thousands of inactive conversations should not block the worker.
+ * Scheduling can re-enqueue the job to walk further once existing pages are
+ * embedded.
+ */
+const ACTIVATION_RECOMPUTE_CONVERSATION_LIMIT = 500;
+
+/**
+ * Job handler: for every conversation with a persisted activation row, fetch
+ * its last user/assistant exchange, recompute the activation map via the
+ * standard pipeline (`selectCandidates` → `computeOwnActivation` →
+ * `spreadActivation`), and persist the new map. No rendering, no injection
+ * delta — this is the state-update side of the per-turn pipeline.
+ *
+ * Used after consolidation replaces or deletes pages that other conversations
+ * still reference: without recompute, a stale slug above `epsilon` keeps
+ * decaying in `state` and contributing to the candidate set even though its
+ * page is gone. Recompute drops it to zero and lets it fall out of the
+ * sparse map on the next save.
+ */
+export async function memoryV2ActivationRecomputeJob(
+  _job: MemoryJob,
+  config: AssistantConfig,
+): Promise<number> {
+  const workspaceDir = getWorkspaceDir();
+  const database = getDb();
+
+  const conversations = listConversations(
+    ACTIVATION_RECOMPUTE_CONVERSATION_LIMIT,
+  );
+  const edgeIndex = await getEdgeIndex(workspaceDir);
+  const nowText = await loadNowText(workspaceDir);
+
+  let updated = 0;
+  for (const conv of conversations) {
+    const priorState = await hydrate(database, conv.id);
+    if (!priorState) continue; // Nothing to recompute when no row exists.
+
+    let nextState;
+    try {
+      nextState = await recomputeForConversation({
+        conversationId: conv.id,
+        priorState,
+        edgeIndex,
+        nowText,
+        config,
+      });
+    } catch (err) {
+      log.warn(
+        { err, conversationId: conv.id },
+        "Activation recompute failed for conversation; leaving prior state in place",
+      );
+      continue;
+    }
+
+    if (!nextState) continue;
+    await save(database, conv.id, nextState);
+    updated += 1;
+  }
+
+  log.info(
+    { conversationsScanned: conversations.length, updated },
+    "Memory v2 activation recompute complete",
+  );
+  return updated;
+}
+
+interface RecomputeForConversationParams {
+  conversationId: string;
+  priorState: NonNullable<Awaited<ReturnType<typeof hydrate>>>;
+  edgeIndex: Awaited<ReturnType<typeof getEdgeIndex>>;
+  nowText: string;
+  config: AssistantConfig;
+}
+
+/**
+ * Run the per-turn activation pipeline against the conversation's most
+ * recent user/assistant texts and return the new state, or `null` if the
+ * conversation has no usable messages (empty conv, fork-only, etc.).
+ *
+ * Filters out the prior `state` keys whose recomputed value falls at or below
+ * `epsilon` so the persisted sparse map shrinks rather than growing.
+ */
+async function recomputeForConversation(
+  params: RecomputeForConversationParams,
+): Promise<Awaited<ReturnType<typeof hydrate>> | null> {
+  const { conversationId, priorState, edgeIndex, nowText, config } = params;
+
+  const { userText, assistantText } = lastExchangeTexts(conversationId);
+  if (!userText && !assistantText) return null;
+
+  const { candidates } = await selectCandidates({
+    priorState,
+    userText,
+    assistantText,
+    nowText,
+    config,
+  });
+  const { activation: ownActivation } = await computeOwnActivation({
+    candidates,
+    priorState,
+    userText,
+    assistantText,
+    nowText,
+    config,
+  });
+  const { final: spread } = spreadActivation(
+    ownActivation,
+    edgeIndex,
+    config.memory.v2.k,
+    config.memory.v2.hops,
+  );
+
+  const epsilon = config.memory.v2.epsilon;
+  const sparseState: Record<string, number> = {};
+  for (const [slug, value] of spread) {
+    if (value > epsilon) sparseState[slug] = value;
+  }
+
+  return {
+    messageId: priorState.messageId,
+    state: sparseState,
+    everInjected: priorState.everInjected,
+    currentTurn: priorState.currentTurn,
+    updatedAt: Date.now(),
+  };
+}
+
+/**
+ * Pull the most recent user message and the most recent assistant message
+ * from a conversation. We deliberately don't require them to be on the same
+ * turn — a conversation that ended on the user's question still has both
+ * texts to feed the recompute, just from different turns.
+ *
+ * Returns empty strings on missing rows so the recompute call can short-
+ * circuit cleanly. Tool-call content is dropped (only `text` blocks survive)
+ * — same shape `loadRecentMessagesText` produces in `sweep-job.ts`.
+ */
+function lastExchangeTexts(conversationId: string): {
+  userText: string;
+  assistantText: string;
+} {
+  const all = getMessages(conversationId);
+  if (all.length === 0) return { userText: "", assistantText: "" };
+
+  let userText = "";
+  let assistantText = "";
+  for (let i = all.length - 1; i >= 0; i--) {
+    const row = all[i];
+    if (!userText && row.role === "user") {
+      userText = stringifyMessageContent(row.content);
+    } else if (!assistantText && row.role === "assistant") {
+      assistantText = stringifyMessageContent(row.content);
+    }
+    if (userText && assistantText) break;
+  }
+  return { userText, assistantText };
+}
+
+/**
+ * Coerce stored message content (JSON-serialized `ContentBlock[]` *or* plain
+ * string in legacy rows) into a single text string. Image / tool blocks are
+ * dropped — recompute only needs the spoken text.
+ */
+function stringifyMessageContent(stored: string): string {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(stored);
+  } catch {
+    return stored.trim();
+  }
+  if (typeof parsed === "string") return parsed.trim();
+  if (!Array.isArray(parsed)) return "";
+  const parts: string[] = [];
+  for (const block of parsed) {
+    if (
+      block &&
+      typeof block === "object" &&
+      (block as { type?: string }).type === "text" &&
+      typeof (block as { text?: unknown }).text === "string"
+    ) {
+      parts.push((block as { text: string }).text);
+    }
+  }
+  return parts.join("\n").trim();
+}
+
+/**
+ * Read the prose meta files that compose the "NOW" context the activation
+ * pipeline correlates against. Mirrors the autoload order in
+ * `system-prompt.ts` so the same prose drives both injection and recompute.
+ * Missing or unreadable files are treated as empty.
+ */
+async function loadNowText(workspaceDir: string): Promise<string> {
+  const filenames = ["essentials.md", "threads.md", "recent.md"];
+  const reads = await Promise.all(
+    filenames.map(async (filename) => {
+      try {
+        const text = await readFile(
+          join(workspaceDir, "memory", filename),
+          "utf-8",
+        );
+        return text.trim();
+      } catch {
+        return "";
+      }
+    }),
+  );
+  return reads.filter((part) => part.length > 0).join("\n\n");
+}

package/src/memory/v2/consolidation-job.ts

@@ -0,0 +1,306 @@
+/**
+ * Memory v2 — `memory_v2_consolidate` job handler.
+ *
+ * The consolidation job is the centerpiece of v2: an hourly background pass
+ * that routes accumulated `memory/buffer.md` entries into concept pages,
+ * rewrites `memory/recent.md`, promotes new essentials/threads, and trims the
+ * buffer down to entries that arrived after the run started.
+ *
+ * Unlike `sweep`, consolidation runs as the assistant: `wakeAgentForOpportunity()`
+ * loads the standard system prompt (SOUL.md + IDENTITY.md + persona + memory/*
+ * autoloads) and the standard tool surface (read_file, write_file, edit_file,
+ * list_files, bash). The hint string carries the prompt body from §10 of the
+ * design doc with the cutoff timestamp templated in. Care, judgment, and the
+ * assistant's voice are the point — there is no "consolidator persona" to
+ * substitute in.
+ *
+ * Lifecycle:
+ *   1. Bail if the `memory-v2-enabled` feature flag is off (the worker may
+ *      have claimed a stale row at flag-flip time).
+ *   2. Acquire a single-process lock at `memory/.v2-state/consolidation.lock`
+ *      so two overlapping schedule windows can't fight over the same files.
+ *      The lock contains the holder's PID + timestamp so a crashed run leaves
+ *      a diagnosable trace.
+ *   3. Capture the cutoff timestamp at dispatch. Any buffer entry timestamped
+ *      at or after the cutoff arrived AFTER the run started — leave it for
+ *      the next pass.
+ *   4. Read `memory/buffer.md`. Bail if empty (no work to do, but the lock
+ *      and skip path still log so operators can confirm the schedule fired).
+ *   5. Bootstrap a background conversation (mirrors `runUpdateBulletinJobIfNeeded`)
+ *      and call `wakeAgentForOpportunity()` with the templated hint. The wake
+ *      reuses the assistant's full system prompt + tools.
+ *   6. On wake success, enqueue `memory_v2_reembed` to re-index any pages the
+ *      agent touched. Tracking touched pages via mtime would be more precise
+ *      but is fragile across filesystems; the embedder's content-hash cache
+ *      makes a conservative full-reembed effectively free. On wake failure
+ *      no follow-ups are enqueued — the agent didn't run, so there's nothing
+ *      to re-embed.
+ *   7. Release the lock.
+ *
+ * The handler never propagates a wake exception: it logs, cleans up the
+ * orphan conversation, releases the lock, and returns `wake_failed` so the
+ * next scheduled run can re-attempt. A thrown bootstrap error bubbles up and
+ * the jobs-worker treats it as a retryable failure.
+ */
+
+import {
+  closeSync,
+  mkdirSync,
+  openSync,
+  readFileSync,
+  unlinkSync,
+  writeSync,
+} from "node:fs";
+import { dirname, join } from "node:path";
+
+import { isAssistantFeatureFlagEnabled } from "../../config/assistant-feature-flags.js";
+import type { AssistantConfig } from "../../config/types.js";
+import { INTERNAL_GUARDIAN_TRUST_CONTEXT } from "../../daemon/trust-context.js";
+import { wakeAgentForOpportunity } from "../../runtime/agent-wake.js";
+import { getLogger } from "../../util/logger.js";
+import { getWorkspaceDir } from "../../util/platform.js";
+import { bootstrapConversation } from "../conversation-bootstrap.js";
+import { deleteConversation } from "../conversation-crud.js";
+import {
+  enqueueMemoryJob,
+  type MemoryJob,
+  type MemoryJobType,
+} from "../jobs-store.js";
+import { resolveConsolidationPrompt } from "./prompts/consolidation.js";
+
+const log = getLogger("memory-v2-consolidate");
+
+/** Source string identifying this wake in `agent-wake` logs and surfaces. */
+const WAKE_SOURCE = "memory_v2_consolidation";
+
+/**
+ * Follow-up jobs to fan out after a successful consolidation.
+ *
+ * Conservatively re-embeds every page rather than tracking which pages the
+ * agent touched: mtime-diffing is fragile across filesystems, and the
+ * embedder's content-hash cache makes unchanged pages effectively free.
+ */
+const FOLLOW_UP_JOB_TYPES: readonly MemoryJobType[] = [
+  "memory_v2_reembed",
+] as const;
+
+/**
+ * Job handler. See file header for the full lifecycle. Returns a discriminated
+ * union so tests can assert on the path taken (flag-off / locked / empty /
+ * invoked) without having to spy on the filesystem.
+ */
+export type ConsolidationOutcome =
+  | { kind: "flag_off" }
+  | { kind: "locked"; holder: string }
+  | { kind: "empty_buffer" }
+  | { kind: "wake_failed"; reason?: string }
+  | {
+      kind: "invoked";
+      conversationId: string;
+      cutoff: string;
+      followUpJobIds: string[];
+    };
+
+export async function memoryV2ConsolidateJob(
+  _job: MemoryJob,
+  config: AssistantConfig,
+): Promise<ConsolidationOutcome> {
+  if (!isAssistantFeatureFlagEnabled("memory-v2-enabled", config)) {
+    log.debug("memory-v2-enabled flag off; consolidation skipped");
+    return { kind: "flag_off" };
+  }
+
+  const memoryDir = join(getWorkspaceDir(), "memory");
+  const lockPath = join(memoryDir, ".v2-state", "consolidation.lock");
+  const bufferPath = join(memoryDir, "buffer.md");
+
+  // Step 1: acquire lock. Bails immediately if another consolidation is
+  // already in flight — the next scheduled run can pick up where we leave off.
+  const holder = tryAcquireLock(lockPath);
+  if (holder !== null) {
+    log.warn({ lockPath, holder }, "consolidation skipped: lock already held");
+    return { kind: "locked", holder };
+  }
+
+  try {
+    // Step 2: capture cutoff. ISO-8601 is the convention; it's a total order
+    // string that compares correctly via lexicographic <, which is all the
+    // prompt asks the agent to do. Captured here (not at enqueue time) so
+    // late-claimed rows still get a fresh cutoff.
+    const cutoff = new Date().toISOString();
+
+    // Step 3: bail on empty buffer. Nothing for the agent to consolidate.
+    // The lock is released in finally below.
+    const bufferContent = readBufferContent(bufferPath);
+    if (bufferContent.trim().length === 0) {
+      log.debug("buffer.md empty; consolidation skipped");
+      return { kind: "empty_buffer" };
+    }
+
+    // Step 4: bootstrap a background conversation and wake the assistant
+    // with the cutoff-templated prompt. Mirrors the UPDATES.md pattern in
+    // `runUpdateBulletinJobIfNeeded` — the wake runs `mainAgent` against
+    // the assistant's full system prompt, so consolidation thinks and
+    // writes in the assistant's voice.
+    const conversation = bootstrapConversation({
+      conversationType: "background",
+      source: WAKE_SOURCE,
+      origin: "memory_consolidation",
+      systemHint: "Running memory consolidation",
+      groupId: "system:background",
+    });
+
+    let wakeInvoked = false;
+    let failureReason: string | undefined;
+    try {
+      const result = await wakeAgentForOpportunity({
+        conversationId: conversation.id,
+        hint: resolveConsolidationPrompt(
+          config.memory.v2.consolidation_prompt_path,
+          cutoff,
+        ),
+        source: WAKE_SOURCE,
+        trustContext: INTERNAL_GUARDIAN_TRUST_CONTEXT,
+      });
+      wakeInvoked = result.invoked;
+      failureReason = result.reason;
+    } catch (err) {
+      failureReason = err instanceof Error ? err.message : String(err);
+      log.error(
+        { err, conversationId: conversation.id },
+        "consolidation wake threw; cleaning up and re-enqueuing follow-ups skipped",
+      );
+    }
+
+    // If the wake never ran (resolver missing, conversation archived,
+    // timeout, exception), clean up the orphan background conversation —
+    // matches the cleanup logic in `runUpdateBulletinJobIfNeeded`. We
+    // do NOT enqueue follow-ups in this branch because no pages changed.
+    if (!wakeInvoked) {
+      try {
+        deleteConversation(conversation.id);
+      } catch (err) {
+        log.warn(
+          { err, conversationId: conversation.id },
+          "consolidation: failed to delete orphan background conversation; continuing",
+        );
+      }
+      return failureReason !== undefined
+        ? { kind: "wake_failed", reason: failureReason }
+        : { kind: "wake_failed" };
+    }
+
+    // Step 5: enqueue follow-up jobs. Enqueueing now keeps the dispatch
+    // wiring exercised end-to-end so PR 21 only has to swap in the handler
+    // bodies.
+    const followUpJobIds: string[] = [];
+    for (const jobType of FOLLOW_UP_JOB_TYPES) {
+      try {
+        followUpJobIds.push(enqueueMemoryJob(jobType, {}));
+      } catch (err) {
+        // Best-effort: a failed enqueue here doesn't undo the agent's writes,
+        // and the next scheduled consolidation will attempt the same fan-out.
+        log.warn(
+          { err, jobType },
+          "consolidation: failed to enqueue follow-up job; continuing",
+        );
+      }
+    }
+
+    log.info(
+      {
+        conversationId: conversation.id,
+        cutoff,
+        followUpJobIds,
+      },
+      "consolidation invoked",
+    );
+    return {
+      kind: "invoked",
+      conversationId: conversation.id,
+      cutoff,
+      followUpJobIds,
+    };
+  } finally {
+    releaseLock(lockPath);
+  }
+}
+
+/**
+ * Read `memory/buffer.md`. Missing file → empty string so the skip-on-empty
+ * branch doesn't have to distinguish "no file" from "blank file".
+ */
+function readBufferContent(bufferPath: string): string {
+  try {
+    return readFileSync(bufferPath, "utf-8");
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return "";
+    throw err;
+  }
+}
+
+/**
+ * Atomically create the lock file with `wx` (O_CREAT | O_EXCL) flags. Returns
+ * `null` on success, or the current holder string (file contents, typically
+ * `pid timestamp`) when the file already exists — the holder is surfaced for
+ * log diagnostics so operators can identify a stuck lock without re-reading.
+ *
+ * Crash recovery: if the prior daemon died with the lock held, the file will
+ * still be on disk on the next start. PR 20 keeps the lock simple per the
+ * plan instructions; a future iteration can probe liveness via `kill(pid, 0)`
+ * the way `snapshot-lock.ts` does. Until then, an operator can clear a
+ * stale lock by removing the file.
+ */
+function tryAcquireLock(lockPath: string): string | null {
+  // The workspace migration seeds `memory/.v2-state/`, but tests and
+  // ad-hoc workspaces may not have it yet. `mkdirSync({ recursive: true })`
+  // is idempotent, so the call is cheap when the dir already exists.
+  mkdirSync(dirname(lockPath), { recursive: true });
+
+  let fd: number;
+  try {
+    // `wx` = create-if-not-exists, fail with EEXIST if it does.
+    fd = openSync(lockPath, "wx");
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
+    try {
+      return readFileSync(lockPath, "utf-8").trim() || "unknown";
+    } catch {
+      return "unknown";
+    }
+  }
+
+  // Best-effort PID + timestamp payload so a stale lock can be diagnosed.
+  // The worker only cares that the file exists; the contents are advisory.
+  try {
+    writeSync(fd, `${process.pid} ${Date.now()}\n`);
+  } catch {
+    // best-effort
+  } finally {
+    try {
+      closeSync(fd);
+    } catch {
+      // best-effort
+    }
+  }
+  return null;
+}
+
+/**
+ * Idempotent unlink of the lock file. Called from the `finally` block so a
+ * crash in the wake path doesn't leave the lock stranded. ENOENT is swallowed
+ * because the lock may have been released by an operator or never created
+ * (acquire failed before reaching the lock-write step).
+ */
+function releaseLock(lockPath: string): void {
+  try {
+    unlinkSync(lockPath);
+  } catch (err) {
+    const code = (err as NodeJS.ErrnoException).code;
+    if (code === "ENOENT") return;
+    log.warn(
+      { err, lockPath },
+      "consolidation: failed to release lock (best-effort)",
+    );
+  }
+}