@vellumai/assistant 0.8.3 → 0.8.5

package/src/memory/v3/consolidation-job.ts

@@ -0,0 +1,323 @@
+/**
+ * Memory v3 — `memory_v3_consolidate` job handler.
+ *
+ * The v3 consolidation job drains the SHARED `memory/buffer.md` (the same
+ * buffer v2 uses — there is no v3 buffer) into the SHARED concept pages AND the
+ * v3 **tree** overlay, while maintaining the SHARED standing-context files
+ * (`essentials.md` / `threads.md` / `recent.md`) byte-for-byte the way v2 does.
+ * It is the v3 counterpart to `assistant/src/memory/v2/consolidation-job.ts`
+ * and mirrors its orchestration exactly — the only divergences are the gating
+ * flag (`memory.v3.write.enabled`), the lock path (`memory/.v3-state/`), and the
+ * prompt body (which additionally asks the agent to author/refresh the tree).
+ *
+ * Because the buffer and the standing-context files are shared, exactly one
+ * consolidator may own the drain at a time. The scheduler enforces this: when
+ * `memory.v3.write.enabled` is on it enqueues `memory_v3_consolidate` INSTEAD of
+ * `memory_v2_consolidate` (see `maybeEnqueueGraphMaintenanceJobs` in
+ * `jobs-worker.ts`). Concept pages stay the shared canonical store, so the v2
+ * router keeps working off pages v3 writes — it just ignores the tree overlay.
+ *
+ * Lifecycle (identical to v2 except the flag + lock path + tree-authoring
+ * prompt):
+ *   1. Bail if `config.memory.v3.write.enabled` is false (the worker may have
+ *      claimed a stale row from before the flag was flipped off).
+ *   2. Acquire a single-process lock at `memory/.v3-state/consolidation.lock`.
+ *   3. Capture the cutoff timestamp at dispatch.
+ *   4. Read the shared `memory/buffer.md`. Bail if empty.
+ *   5. Hand off to `runBackgroundJob()` with the v3 consolidation prompt
+ *      (`suppressFailureNotifications: true`).
+ *   6. On success, enqueue follow-ups: `memory_v3_index_maintenance` (mechanical
+ *      tree/DAG upkeep) and `embed_concept_page` reembed (pages are shared, so
+ *      reembed is still needed — reuse the existing `memory_v2_reembed` fan-out
+ *      job type, which enqueues one `embed_concept_page` per slug).
+ *   7. Release the lock.
+ */
+
+import {
+  closeSync,
+  mkdirSync,
+  openSync,
+  readFileSync,
+  unlinkSync,
+  writeSync,
+} from "node:fs";
+import { dirname, join } from "node:path";
+
+import type { AssistantConfig } from "../../config/types.js";
+import { runBackgroundJob } from "../../runtime/background-job-runner.js";
+import { getLogger } from "../../util/logger.js";
+import { getWorkspaceDir } from "../../util/platform.js";
+import { isProcessAlive } from "../../util/process-liveness.js";
+import { formatBufferTimestamp } from "../graph/tool-handlers.js";
+import {
+  enqueueMemoryJob,
+  type MemoryJob,
+  type MemoryJobType,
+} from "../jobs-store.js";
+// The consolidation conversation `source` is a UI/routing concern shared with
+// v2 (the route layer recognizes "this conversation IS background memory
+// consolidation" by this string). v2 and v3 are mutually exclusive drainers, so
+// reusing the same source keeps that recognition working for both without
+// forking a v3 constant.
+import { MEMORY_V2_CONSOLIDATION_SOURCE } from "../v2/constants.js";
+import { resolveConsolidationPrompt } from "./prompts/consolidation.js";
+
+const log = getLogger("memory-v3-consolidate");
+
+/** Stable identifier surfaced in `runBackgroundJob` logs and notifications. */
+const JOB_NAME = "memory.consolidate";
+
+/**
+ * Hard timeout for the consolidation run. Matches v2: consolidation reads the
+ * buffer, rewrites several files, re-encodes essentials/threads, and authors
+ * the tree — generous upper bound so a slow run isn't killed mid-edit, but
+ * bounded so a stuck provider can't pin the worker indefinitely.
+ */
+const CONSOLIDATION_TIMEOUT_MS = 15 * 60 * 1000;
+
+/**
+ * Follow-up jobs to fan out after a successful consolidation:
+ *   - `memory_v3_index_maintenance` — mechanical (no-LLM) tree/DAG upkeep:
+ *     validate the tree, report stale composed indices, cycle-check the DAG.
+ *   - `memory_v2_reembed` — re-embed every shared concept page (the fan-out job
+ *     enqueues one `embed_concept_page` per slug). Pages are shared, so a v3
+ *     consolidation that touches them still needs the reembed. Conservatively
+ *     re-embeds every page; the embedder's content-hash cache makes unchanged
+ *     pages effectively free.
+ */
+const FOLLOW_UP_JOB_TYPES: readonly MemoryJobType[] = [
+  "memory_v3_index_maintenance",
+  "memory_v2_reembed",
+] as const;
+
+/**
+ * Job handler. See file header for the full lifecycle. Returns a discriminated
+ * union so tests can assert on the path taken (disabled / locked / empty /
+ * invoked / failed) without having to spy on the filesystem. Mirrors v2's
+ * `ConsolidationOutcome`.
+ */
+export type ConsolidationOutcome =
+  | { kind: "disabled" }
+  | { kind: "locked"; holder: string }
+  | { kind: "empty_buffer" }
+  | { kind: "run_failed"; reason?: string }
+  | {
+      kind: "invoked";
+      conversationId: string;
+      cutoff: string;
+      followUpJobIds: string[];
+    };
+
+export async function memoryV3ConsolidateJob(
+  _job: MemoryJob,
+  config: AssistantConfig,
+): Promise<ConsolidationOutcome> {
+  if (!config.memory.v3.write.enabled) {
+    log.debug("memory.v3.write.enabled is false; consolidation skipped");
+    return { kind: "disabled" };
+  }
+
+  const memoryDir = join(getWorkspaceDir(), "memory");
+  const lockPath = join(memoryDir, ".v3-state", "consolidation.lock");
+  const bufferPath = join(memoryDir, "buffer.md");
+
+  // Step 1: acquire lock. Bails immediately if another consolidation is
+  // already in flight — the next scheduled run can pick up where we leave off.
+  const holder = tryAcquireLock(lockPath);
+  if (holder !== null) {
+    log.warn({ lockPath, holder }, "consolidation skipped: lock already held");
+    return { kind: "locked", holder };
+  }
+
+  try {
+    // Step 2: capture cutoff. Formatted to match `buffer.md` entry timestamps
+    // (`Mon D, h:mm AM/PM`) so the agent's "timestamp ≥ cutoff" check compares
+    // like-with-like at minute precision. Captured here (not at enqueue time)
+    // so late-claimed rows get a fresh cutoff.
+    const cutoff = formatBufferTimestamp(new Date());
+
+    // Step 3: bail on empty buffer. The shared buffer has no work to drain.
+    const bufferContent = readBufferContent(bufferPath);
+    if (bufferContent.trim().length === 0) {
+      log.debug("buffer.md empty; consolidation skipped");
+      return { kind: "empty_buffer" };
+    }
+
+    // Step 4: hand off to the centralized background-job runner. As with v2,
+    // `suppressFailureNotifications: true` opts out of `activity.failed`
+    // notifications so a network blip on the tight consolidation interval does
+    // not spam the home feed; Sentry-side reporting is unchanged.
+    //
+    // The prompt override config key (`memory.v2.consolidation_prompt_path`) is
+    // shared — there is no separate v3 key, so an operator points one file at
+    // whichever consolidator owns the drain.
+    const runResult = await runBackgroundJob({
+      jobName: JOB_NAME,
+      source: MEMORY_V2_CONSOLIDATION_SOURCE,
+      prompt: resolveConsolidationPrompt(
+        config.memory.v2.consolidation_prompt_path,
+        cutoff,
+      ),
+      trustContext: { sourceChannel: "vellum", trustClass: "guardian" },
+      callSite: "mainAgent",
+      timeoutMs: CONSOLIDATION_TIMEOUT_MS,
+      origin: "memory_consolidation",
+      suppressFailureNotifications: true,
+    });
+
+    if (!runResult.ok) {
+      log.error(
+        {
+          conversationId: runResult.conversationId,
+          errorKind: runResult.errorKind,
+          err: runResult.error?.message,
+        },
+        "consolidation run failed; follow-ups skipped",
+      );
+      return runResult.error?.message !== undefined
+        ? { kind: "run_failed", reason: runResult.error.message }
+        : { kind: "run_failed" };
+    }
+
+    // Step 5: enqueue follow-up jobs (tree maintenance + page reembed).
+    const followUpJobIds: string[] = [];
+    for (const jobType of FOLLOW_UP_JOB_TYPES) {
+      try {
+        followUpJobIds.push(enqueueMemoryJob(jobType, {}));
+      } catch (err) {
+        // Best-effort: a failed enqueue here doesn't undo the agent's writes,
+        // and the next scheduled consolidation will attempt the same fan-out.
+        log.warn(
+          { err, jobType },
+          "consolidation: failed to enqueue follow-up job; continuing",
+        );
+      }
+    }
+
+    log.info(
+      {
+        conversationId: runResult.conversationId,
+        cutoff,
+        followUpJobIds,
+      },
+      "consolidation invoked",
+    );
+    return {
+      kind: "invoked",
+      conversationId: runResult.conversationId,
+      cutoff,
+      followUpJobIds,
+    };
+  } finally {
+    releaseLock(lockPath);
+  }
+}
+
+/**
+ * Read `memory/buffer.md`. Missing file → empty string so the skip-on-empty
+ * branch doesn't have to distinguish "no file" from "blank file".
+ */
+function readBufferContent(bufferPath: string): string {
+  try {
+    return readFileSync(bufferPath, "utf-8");
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return "";
+    throw err;
+  }
+}
+
+/**
+ * Atomically create the lock file with `wx` (O_CREAT | O_EXCL) flags. Returns
+ * `null` on success, or the current holder string when the file already exists
+ * and the holder is still alive. Mirrors v2's lock machinery exactly — single
+ * writer per workspace, so a holder whose process died is unambiguously stale
+ * and is taken over automatically.
+ */
+function tryAcquireLock(lockPath: string): string | null {
+  mkdirSync(dirname(lockPath), { recursive: true });
+
+  const firstHolder = tryCreate(lockPath);
+  if (firstHolder === null) return null;
+  if (!isHolderStale(firstHolder)) return firstHolder;
+
+  log.info(
+    { lockPath, holder: firstHolder },
+    "consolidation: taking over stale lock (holder not running)",
+  );
+  try {
+    unlinkSync(lockPath);
+  } catch (err) {
+    const code = (err as NodeJS.ErrnoException).code;
+    if (code !== "ENOENT") {
+      log.warn(
+        { err, lockPath },
+        "consolidation: failed to unlink stale lock; reporting as locked",
+      );
+      return firstHolder;
+    }
+  }
+  return tryCreate(lockPath);
+}
+
+/**
+ * Atomically create the lock file. Returns `null` on success, or the holder
+ * string read from the file when it already exists (`"unknown"` if the read
+ * itself fails). Rethrows any non-EEXIST errno from `openSync`.
+ */
+function tryCreate(lockPath: string): string | null {
+  let fd: number;
+  try {
+    fd = openSync(lockPath, "wx");
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
+    try {
+      return readFileSync(lockPath, "utf-8").trim() || "unknown";
+    } catch {
+      return "unknown";
+    }
+  }
+  try {
+    writeSync(fd, `${process.pid} ${Date.now()}\n`);
+  } catch {
+    // best-effort — payload is advisory, the file's existence is the lock
+  } finally {
+    try {
+      closeSync(fd);
+    } catch {
+      // best-effort
+    }
+  }
+  return null;
+}
+
+/**
+ * A holder string is stale when its PID parses to a non-running process. An
+ * unparseable / empty / `"unknown"` payload is also treated as stale: the only
+ * writer is `tryCreate`, so corruption indicates a partial write from a crashed
+ * prior holder rather than a live writer mid-flush.
+ */
+function isHolderStale(holder: string): boolean {
+  const match = /^\d+/.exec(holder);
+  if (!match) return true;
+  const pid = Number.parseInt(match[0], 10);
+  if (!Number.isFinite(pid) || pid <= 0) return true;
+  return !isProcessAlive(pid);
+}
+
+/**
+ * Idempotent unlink of the lock file. Called from the `finally` block so a
+ * crash in the run path doesn't leave the lock stranded. ENOENT is swallowed
+ * because the lock may have been released by an operator or never created.
+ */
+function releaseLock(lockPath: string): void {
+  try {
+    unlinkSync(lockPath);
+  } catch (err) {
+    const code = (err as NodeJS.ErrnoException).code;
+    if (code === "ENOENT") return;
+    log.warn(
+      { err, lockPath },
+      "consolidation: failed to release lock (best-effort)",
+    );
+  }
+}

package/src/memory/v3/edge-learning-job.ts

@@ -0,0 +1,160 @@
+/**
+ * Memory v3 — `memory_v3_edge_learning` job (fast lane, no LLM).
+ *
+ * Reconciles the raw co-activation log (`memory_v3_coactivation`, migration
+ * 262) into the weighted learned-edge graph (`memory_v3_auto_edges`, migration
+ * 263). One pass does three things:
+ *
+ *   1. **Decay** — multiplicatively age all existing auto-edge weights toward
+ *      zero on a half-life schedule (the rich-get-richer counterweight: an edge
+ *      that stops being reinforced fades, so a once-hot pair can't dominate the
+ *      adjacency forever).
+ *   2. **Reinforce** — for each recent co-activation whose `used` flag is set,
+ *      bump the `source → target` weight. *Used-only*: we learn associations
+ *      that proved load-bearing for a turn, not pairs that merely surfaced
+ *      together. The watermark checkpoint advances so each co-activation is
+ *      counted once.
+ *   3. **Propose** — surface the top-weight auto-edges as advisory promotion
+ *      *candidates* for the assistant to ratify into curated `edges:` during
+ *      consolidation. This job PROPOSES; it never auto-writes page frontmatter.
+ *      Diversity counterweight: candidates are capped and a single source's
+ *      out-edges are bounded so one hub can't monopolize the slate.
+ *
+ * Decay runs before reinforce so a fresh reinforcement isn't immediately aged
+ * by the same pass. The job is idempotent in effect: re-running with no new
+ * co-activations only decays (which is itself elapsed-time-bounded).
+ */
+
+import { getLogger } from "../../util/logger.js";
+import { getMemoryCheckpoint, setMemoryCheckpoint } from "../checkpoints.js";
+import type { DrizzleDb } from "../db-connection.js";
+import { getDb } from "../db-connection.js";
+import type { MemoryJob } from "../jobs-store.js";
+import {
+  type AutoEdgeRow,
+  decay,
+  reinforce,
+  topByWeight,
+} from "./auto-edges.js";
+import { readCoactivations } from "./coactivation-store.js";
+
+const log = getLogger("memory-v3-edge-learning");
+
+/**
+ * Half-life of auto-edge weight decay. Matches the v2 injection-score cadence
+ * (3 days) — a pair reinforced 3 days ago and never since contributes half its
+ * weight, 6 days ago a quarter.
+ */
+export const EDGE_DECAY_HALF_LIFE_MS = 3 * 24 * 60 * 60 * 1000;
+
+/** Max promotion candidates surfaced per run (the diversity cap). */
+export const MAX_PROMOTION_CANDIDATES = 20;
+
+/** Max candidates contributed by any single source slug (anti-hub diversity). */
+export const MAX_CANDIDATES_PER_SOURCE = 3;
+
+/**
+ * Minimum weight for an auto-edge to be eligible for promotion. A pair must
+ * accrue more than a single reinforcement (which decays away) before it's worth
+ * proposing as a curated edge.
+ */
+export const PROMOTION_WEIGHT_FLOOR = 1.5;
+
+/** Checkpoint key for the high-water mark of reconciled co-activations. */
+const WATERMARK_KEY = "memory_v3_edge_learning:coactivation_watermark";
+
+/** Summary of one edge-learning pass, returned for the dispatcher log + tests. */
+export interface EdgeLearningResult {
+  /** Used co-activations reinforced this pass. */
+  reinforced: number;
+  /** Co-activations skipped because `used` was falsy. */
+  skippedUnused: number;
+  /** Auto-edges pruned by decay (fell below the floor). */
+  pruned: number;
+  /** Advisory promotion candidates, heaviest first, after the diversity cap. */
+  candidates: AutoEdgeRow[];
+}
+
+/**
+ * Run one edge-learning pass against `database`. Pure of LLM and workspace I/O —
+ * the whole pass is bounded DB work, hence the fast lane.
+ */
+export function runEdgeLearning(
+  database: DrizzleDb,
+  now = Date.now(),
+): EdgeLearningResult {
+  // 1. Decay first so this pass's reinforcements aren't immediately aged.
+  const pruned = decay(database, now, EDGE_DECAY_HALF_LIFE_MS);
+
+  // 2. Reinforce from co-activations newer than the watermark. The watermark is
+  //    a created_at boundary; `since` is inclusive so we nudge it forward by 1ms
+  //    to avoid re-counting the boundary row.
+  const watermark = parseInt(getMemoryCheckpoint(WATERMARK_KEY) ?? "0", 10);
+  const since = watermark > 0 ? watermark + 1 : undefined;
+  const coactivations = readCoactivations(database, since);
+
+  let reinforced = 0;
+  let skippedUnused = 0;
+  let maxCreatedAt = watermark;
+  for (const row of coactivations) {
+    if (row.createdAt > maxCreatedAt) maxCreatedAt = row.createdAt;
+    // Reinforce usefulness, not mere retrieval: skip co-activations the loop
+    // (or a later usefulness reconciliation) did not mark as used.
+    if (!row.used) {
+      skippedUnused += 1;
+      continue;
+    }
+    reinforce(database, row.sourceSlug, row.targetSlug, now);
+    reinforced += 1;
+  }
+  if (maxCreatedAt > watermark) {
+    setMemoryCheckpoint(WATERMARK_KEY, String(maxCreatedAt));
+  }
+
+  // 3. Propose promotion candidates: heaviest auto-edges above the floor, capped
+  //    overall and per-source so a single hub can't monopolize the slate.
+  const candidates = selectPromotionCandidates(
+    topByWeight(database, MAX_PROMOTION_CANDIDATES * MAX_CANDIDATES_PER_SOURCE),
+  );
+
+  log.info(
+    {
+      reinforced,
+      skippedUnused,
+      pruned,
+      candidateCount: candidates.length,
+    },
+    "v3 edge learning complete",
+  );
+
+  return { reinforced, skippedUnused, pruned, candidates };
+}
+
+/**
+ * Apply the weight floor and the overall / per-source diversity caps to a
+ * weight-sorted list of auto-edges. Input must already be sorted heaviest-first
+ * (as {@link topByWeight} returns).
+ */
+function selectPromotionCandidates(sorted: AutoEdgeRow[]): AutoEdgeRow[] {
+  const out: AutoEdgeRow[] = [];
+  const perSource = new Map<string, number>();
+  for (const edge of sorted) {
+    if (out.length >= MAX_PROMOTION_CANDIDATES) break;
+    if (edge.weight < PROMOTION_WEIGHT_FLOOR) continue;
+    const count = perSource.get(edge.sourceSlug) ?? 0;
+    if (count >= MAX_CANDIDATES_PER_SOURCE) continue;
+    perSource.set(edge.sourceSlug, count + 1);
+    out.push(edge);
+  }
+  return out;
+}
+
+/**
+ * Job handler for `memory_v3_edge_learning`. Thin wrapper over
+ * {@link runEdgeLearning} so the heavy lifting (and its tests) live in one
+ * place. The job carries no payload — it always reconciles the whole recent
+ * co-activation log.
+ */
+export function memoryV3EdgeLearningJob(_job: MemoryJob): EdgeLearningResult {
+  return runEdgeLearning(getDb());
+}