@vellumai/assistant 0.8.5 → 0.8.7

package/src/memory/v3/consolidation-job.ts

@@ -1,323 +0,0 @@
-/**
- * Memory v3 — `memory_v3_consolidate` job handler.
- *
- * The v3 consolidation job drains the SHARED `memory/buffer.md` (the same
- * buffer v2 uses — there is no v3 buffer) into the SHARED concept pages AND the
- * v3 **tree** overlay, while maintaining the SHARED standing-context files
- * (`essentials.md` / `threads.md` / `recent.md`) byte-for-byte the way v2 does.
- * It is the v3 counterpart to `assistant/src/memory/v2/consolidation-job.ts`
- * and mirrors its orchestration exactly — the only divergences are the gating
- * flag (`memory.v3.write.enabled`), the lock path (`memory/.v3-state/`), and the
- * prompt body (which additionally asks the agent to author/refresh the tree).
- *
- * Because the buffer and the standing-context files are shared, exactly one
- * consolidator may own the drain at a time. The scheduler enforces this: when
- * `memory.v3.write.enabled` is on it enqueues `memory_v3_consolidate` INSTEAD of
- * `memory_v2_consolidate` (see `maybeEnqueueGraphMaintenanceJobs` in
- * `jobs-worker.ts`). Concept pages stay the shared canonical store, so the v2
- * router keeps working off pages v3 writes — it just ignores the tree overlay.
- *
- * Lifecycle (identical to v2 except the flag + lock path + tree-authoring
- * prompt):
- *   1. Bail if `config.memory.v3.write.enabled` is false (the worker may have
- *      claimed a stale row from before the flag was flipped off).
- *   2. Acquire a single-process lock at `memory/.v3-state/consolidation.lock`.
- *   3. Capture the cutoff timestamp at dispatch.
- *   4. Read the shared `memory/buffer.md`. Bail if empty.
- *   5. Hand off to `runBackgroundJob()` with the v3 consolidation prompt
- *      (`suppressFailureNotifications: true`).
- *   6. On success, enqueue follow-ups: `memory_v3_index_maintenance` (mechanical
- *      tree/DAG upkeep) and `embed_concept_page` reembed (pages are shared, so
- *      reembed is still needed — reuse the existing `memory_v2_reembed` fan-out
- *      job type, which enqueues one `embed_concept_page` per slug).
- *   7. Release the lock.
- */
-
-import {
-  closeSync,
-  mkdirSync,
-  openSync,
-  readFileSync,
-  unlinkSync,
-  writeSync,
-} from "node:fs";
-import { dirname, join } from "node:path";
-
-import type { AssistantConfig } from "../../config/types.js";
-import { runBackgroundJob } from "../../runtime/background-job-runner.js";
-import { getLogger } from "../../util/logger.js";
-import { getWorkspaceDir } from "../../util/platform.js";
-import { isProcessAlive } from "../../util/process-liveness.js";
-import { formatBufferTimestamp } from "../graph/tool-handlers.js";
-import {
-  enqueueMemoryJob,
-  type MemoryJob,
-  type MemoryJobType,
-} from "../jobs-store.js";
-// The consolidation conversation `source` is a UI/routing concern shared with
-// v2 (the route layer recognizes "this conversation IS background memory
-// consolidation" by this string). v2 and v3 are mutually exclusive drainers, so
-// reusing the same source keeps that recognition working for both without
-// forking a v3 constant.
-import { MEMORY_V2_CONSOLIDATION_SOURCE } from "../v2/constants.js";
-import { resolveConsolidationPrompt } from "./prompts/consolidation.js";
-
-const log = getLogger("memory-v3-consolidate");
-
-/** Stable identifier surfaced in `runBackgroundJob` logs and notifications. */
-const JOB_NAME = "memory.consolidate";
-
-/**
- * Hard timeout for the consolidation run. Matches v2: consolidation reads the
- * buffer, rewrites several files, re-encodes essentials/threads, and authors
- * the tree — generous upper bound so a slow run isn't killed mid-edit, but
- * bounded so a stuck provider can't pin the worker indefinitely.
- */
-const CONSOLIDATION_TIMEOUT_MS = 15 * 60 * 1000;
-
-/**
- * Follow-up jobs to fan out after a successful consolidation:
- *   - `memory_v3_index_maintenance` — mechanical (no-LLM) tree/DAG upkeep:
- *     validate the tree, report stale composed indices, cycle-check the DAG.
- *   - `memory_v2_reembed` — re-embed every shared concept page (the fan-out job
- *     enqueues one `embed_concept_page` per slug). Pages are shared, so a v3
- *     consolidation that touches them still needs the reembed. Conservatively
- *     re-embeds every page; the embedder's content-hash cache makes unchanged
- *     pages effectively free.
- */
-const FOLLOW_UP_JOB_TYPES: readonly MemoryJobType[] = [
-  "memory_v3_index_maintenance",
-  "memory_v2_reembed",
-] as const;
-
-/**
- * Job handler. See file header for the full lifecycle. Returns a discriminated
- * union so tests can assert on the path taken (disabled / locked / empty /
- * invoked / failed) without having to spy on the filesystem. Mirrors v2's
- * `ConsolidationOutcome`.
- */
-export type ConsolidationOutcome =
-  | { kind: "disabled" }
-  | { kind: "locked"; holder: string }
-  | { kind: "empty_buffer" }
-  | { kind: "run_failed"; reason?: string }
-  | {
-      kind: "invoked";
-      conversationId: string;
-      cutoff: string;
-      followUpJobIds: string[];
-    };
-
-export async function memoryV3ConsolidateJob(
-  _job: MemoryJob,
-  config: AssistantConfig,
-): Promise<ConsolidationOutcome> {
-  if (!config.memory.v3.write.enabled) {
-    log.debug("memory.v3.write.enabled is false; consolidation skipped");
-    return { kind: "disabled" };
-  }
-
-  const memoryDir = join(getWorkspaceDir(), "memory");
-  const lockPath = join(memoryDir, ".v3-state", "consolidation.lock");
-  const bufferPath = join(memoryDir, "buffer.md");
-
-  // Step 1: acquire lock. Bails immediately if another consolidation is
-  // already in flight — the next scheduled run can pick up where we leave off.
-  const holder = tryAcquireLock(lockPath);
-  if (holder !== null) {
-    log.warn({ lockPath, holder }, "consolidation skipped: lock already held");
-    return { kind: "locked", holder };
-  }
-
-  try {
-    // Step 2: capture cutoff. Formatted to match `buffer.md` entry timestamps
-    // (`Mon D, h:mm AM/PM`) so the agent's "timestamp ≥ cutoff" check compares
-    // like-with-like at minute precision. Captured here (not at enqueue time)
-    // so late-claimed rows get a fresh cutoff.
-    const cutoff = formatBufferTimestamp(new Date());
-
-    // Step 3: bail on empty buffer. The shared buffer has no work to drain.
-    const bufferContent = readBufferContent(bufferPath);
-    if (bufferContent.trim().length === 0) {
-      log.debug("buffer.md empty; consolidation skipped");
-      return { kind: "empty_buffer" };
-    }
-
-    // Step 4: hand off to the centralized background-job runner. As with v2,
-    // `suppressFailureNotifications: true` opts out of `activity.failed`
-    // notifications so a network blip on the tight consolidation interval does
-    // not spam the home feed; Sentry-side reporting is unchanged.
-    //
-    // The prompt override config key (`memory.v2.consolidation_prompt_path`) is
-    // shared — there is no separate v3 key, so an operator points one file at
-    // whichever consolidator owns the drain.
-    const runResult = await runBackgroundJob({
-      jobName: JOB_NAME,
-      source: MEMORY_V2_CONSOLIDATION_SOURCE,
-      prompt: resolveConsolidationPrompt(
-        config.memory.v2.consolidation_prompt_path,
-        cutoff,
-      ),
-      trustContext: { sourceChannel: "vellum", trustClass: "guardian" },
-      callSite: "mainAgent",
-      timeoutMs: CONSOLIDATION_TIMEOUT_MS,
-      origin: "memory_consolidation",
-      suppressFailureNotifications: true,
-    });
-
-    if (!runResult.ok) {
-      log.error(
-        {
-          conversationId: runResult.conversationId,
-          errorKind: runResult.errorKind,
-          err: runResult.error?.message,
-        },
-        "consolidation run failed; follow-ups skipped",
-      );
-      return runResult.error?.message !== undefined
-        ? { kind: "run_failed", reason: runResult.error.message }
-        : { kind: "run_failed" };
-    }
-
-    // Step 5: enqueue follow-up jobs (tree maintenance + page reembed).
-    const followUpJobIds: string[] = [];
-    for (const jobType of FOLLOW_UP_JOB_TYPES) {
-      try {
-        followUpJobIds.push(enqueueMemoryJob(jobType, {}));
-      } catch (err) {
-        // Best-effort: a failed enqueue here doesn't undo the agent's writes,
-        // and the next scheduled consolidation will attempt the same fan-out.
-        log.warn(
-          { err, jobType },
-          "consolidation: failed to enqueue follow-up job; continuing",
-        );
-      }
-    }
-
-    log.info(
-      {
-        conversationId: runResult.conversationId,
-        cutoff,
-        followUpJobIds,
-      },
-      "consolidation invoked",
-    );
-    return {
-      kind: "invoked",
-      conversationId: runResult.conversationId,
-      cutoff,
-      followUpJobIds,
-    };
-  } finally {
-    releaseLock(lockPath);
-  }
-}
-
-/**
- * Read `memory/buffer.md`. Missing file → empty string so the skip-on-empty
- * branch doesn't have to distinguish "no file" from "blank file".
- */
-function readBufferContent(bufferPath: string): string {
-  try {
-    return readFileSync(bufferPath, "utf-8");
-  } catch (err) {
-    if ((err as NodeJS.ErrnoException).code === "ENOENT") return "";
-    throw err;
-  }
-}
-
-/**
- * Atomically create the lock file with `wx` (O_CREAT | O_EXCL) flags. Returns
- * `null` on success, or the current holder string when the file already exists
- * and the holder is still alive. Mirrors v2's lock machinery exactly — single
- * writer per workspace, so a holder whose process died is unambiguously stale
- * and is taken over automatically.
- */
-function tryAcquireLock(lockPath: string): string | null {
-  mkdirSync(dirname(lockPath), { recursive: true });
-
-  const firstHolder = tryCreate(lockPath);
-  if (firstHolder === null) return null;
-  if (!isHolderStale(firstHolder)) return firstHolder;
-
-  log.info(
-    { lockPath, holder: firstHolder },
-    "consolidation: taking over stale lock (holder not running)",
-  );
-  try {
-    unlinkSync(lockPath);
-  } catch (err) {
-    const code = (err as NodeJS.ErrnoException).code;
-    if (code !== "ENOENT") {
-      log.warn(
-        { err, lockPath },
-        "consolidation: failed to unlink stale lock; reporting as locked",
-      );
-      return firstHolder;
-    }
-  }
-  return tryCreate(lockPath);
-}
-
-/**
- * Atomically create the lock file. Returns `null` on success, or the holder
- * string read from the file when it already exists (`"unknown"` if the read
- * itself fails). Rethrows any non-EEXIST errno from `openSync`.
- */
-function tryCreate(lockPath: string): string | null {
-  let fd: number;
-  try {
-    fd = openSync(lockPath, "wx");
-  } catch (err) {
-    if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
-    try {
-      return readFileSync(lockPath, "utf-8").trim() || "unknown";
-    } catch {
-      return "unknown";
-    }
-  }
-  try {
-    writeSync(fd, `${process.pid} ${Date.now()}\n`);
-  } catch {
-    // best-effort — payload is advisory, the file's existence is the lock
-  } finally {
-    try {
-      closeSync(fd);
-    } catch {
-      // best-effort
-    }
-  }
-  return null;
-}
-
-/**
- * A holder string is stale when its PID parses to a non-running process. An
- * unparseable / empty / `"unknown"` payload is also treated as stale: the only
- * writer is `tryCreate`, so corruption indicates a partial write from a crashed
- * prior holder rather than a live writer mid-flush.
- */
-function isHolderStale(holder: string): boolean {
-  const match = /^\d+/.exec(holder);
-  if (!match) return true;
-  const pid = Number.parseInt(match[0], 10);
-  if (!Number.isFinite(pid) || pid <= 0) return true;
-  return !isProcessAlive(pid);
-}
-
-/**
- * Idempotent unlink of the lock file. Called from the `finally` block so a
- * crash in the run path doesn't leave the lock stranded. ENOENT is swallowed
- * because the lock may have been released by an operator or never created.
- */
-function releaseLock(lockPath: string): void {
-  try {
-    unlinkSync(lockPath);
-  } catch (err) {
-    const code = (err as NodeJS.ErrnoException).code;
-    if (code === "ENOENT") return;
-    log.warn(
-      { err, lockPath },
-      "consolidation: failed to release lock (best-effort)",
-    );
-  }
-}

package/src/memory/v3/edge-learning-job.ts

@@ -1,160 +0,0 @@
-/**
- * Memory v3 — `memory_v3_edge_learning` job (fast lane, no LLM).
- *
- * Reconciles the raw co-activation log (`memory_v3_coactivation`, migration
- * 262) into the weighted learned-edge graph (`memory_v3_auto_edges`, migration
- * 263). One pass does three things:
- *
- *   1. **Decay** — multiplicatively age all existing auto-edge weights toward
- *      zero on a half-life schedule (the rich-get-richer counterweight: an edge
- *      that stops being reinforced fades, so a once-hot pair can't dominate the
- *      adjacency forever).
- *   2. **Reinforce** — for each recent co-activation whose `used` flag is set,
- *      bump the `source → target` weight. *Used-only*: we learn associations
- *      that proved load-bearing for a turn, not pairs that merely surfaced
- *      together. The watermark checkpoint advances so each co-activation is
- *      counted once.
- *   3. **Propose** — surface the top-weight auto-edges as advisory promotion
- *      *candidates* for the assistant to ratify into curated `edges:` during
- *      consolidation. This job PROPOSES; it never auto-writes page frontmatter.
- *      Diversity counterweight: candidates are capped and a single source's
- *      out-edges are bounded so one hub can't monopolize the slate.
- *
- * Decay runs before reinforce so a fresh reinforcement isn't immediately aged
- * by the same pass. The job is idempotent in effect: re-running with no new
- * co-activations only decays (which is itself elapsed-time-bounded).
- */
-
-import { getLogger } from "../../util/logger.js";
-import { getMemoryCheckpoint, setMemoryCheckpoint } from "../checkpoints.js";
-import type { DrizzleDb } from "../db-connection.js";
-import { getDb } from "../db-connection.js";
-import type { MemoryJob } from "../jobs-store.js";
-import {
-  type AutoEdgeRow,
-  decay,
-  reinforce,
-  topByWeight,
-} from "./auto-edges.js";
-import { readCoactivations } from "./coactivation-store.js";
-
-const log = getLogger("memory-v3-edge-learning");
-
-/**
- * Half-life of auto-edge weight decay. Matches the v2 injection-score cadence
- * (3 days) — a pair reinforced 3 days ago and never since contributes half its
- * weight, 6 days ago a quarter.
- */
-export const EDGE_DECAY_HALF_LIFE_MS = 3 * 24 * 60 * 60 * 1000;
-
-/** Max promotion candidates surfaced per run (the diversity cap). */
-export const MAX_PROMOTION_CANDIDATES = 20;
-
-/** Max candidates contributed by any single source slug (anti-hub diversity). */
-export const MAX_CANDIDATES_PER_SOURCE = 3;
-
-/**
- * Minimum weight for an auto-edge to be eligible for promotion. A pair must
- * accrue more than a single reinforcement (which decays away) before it's worth
- * proposing as a curated edge.
- */
-export const PROMOTION_WEIGHT_FLOOR = 1.5;
-
-/** Checkpoint key for the high-water mark of reconciled co-activations. */
-const WATERMARK_KEY = "memory_v3_edge_learning:coactivation_watermark";
-
-/** Summary of one edge-learning pass, returned for the dispatcher log + tests. */
-export interface EdgeLearningResult {
-  /** Used co-activations reinforced this pass. */
-  reinforced: number;
-  /** Co-activations skipped because `used` was falsy. */
-  skippedUnused: number;
-  /** Auto-edges pruned by decay (fell below the floor). */
-  pruned: number;
-  /** Advisory promotion candidates, heaviest first, after the diversity cap. */
-  candidates: AutoEdgeRow[];
-}
-
-/**
- * Run one edge-learning pass against `database`. Pure of LLM and workspace I/O —
- * the whole pass is bounded DB work, hence the fast lane.
- */
-export function runEdgeLearning(
-  database: DrizzleDb,
-  now = Date.now(),
-): EdgeLearningResult {
-  // 1. Decay first so this pass's reinforcements aren't immediately aged.
-  const pruned = decay(database, now, EDGE_DECAY_HALF_LIFE_MS);
-
-  // 2. Reinforce from co-activations newer than the watermark. The watermark is
-  //    a created_at boundary; `since` is inclusive so we nudge it forward by 1ms
-  //    to avoid re-counting the boundary row.
-  const watermark = parseInt(getMemoryCheckpoint(WATERMARK_KEY) ?? "0", 10);
-  const since = watermark > 0 ? watermark + 1 : undefined;
-  const coactivations = readCoactivations(database, since);
-
-  let reinforced = 0;
-  let skippedUnused = 0;
-  let maxCreatedAt = watermark;
-  for (const row of coactivations) {
-    if (row.createdAt > maxCreatedAt) maxCreatedAt = row.createdAt;
-    // Reinforce usefulness, not mere retrieval: skip co-activations the loop
-    // (or a later usefulness reconciliation) did not mark as used.
-    if (!row.used) {
-      skippedUnused += 1;
-      continue;
-    }
-    reinforce(database, row.sourceSlug, row.targetSlug, now);
-    reinforced += 1;
-  }
-  if (maxCreatedAt > watermark) {
-    setMemoryCheckpoint(WATERMARK_KEY, String(maxCreatedAt));
-  }
-
-  // 3. Propose promotion candidates: heaviest auto-edges above the floor, capped
-  //    overall and per-source so a single hub can't monopolize the slate.
-  const candidates = selectPromotionCandidates(
-    topByWeight(database, MAX_PROMOTION_CANDIDATES * MAX_CANDIDATES_PER_SOURCE),
-  );
-
-  log.info(
-    {
-      reinforced,
-      skippedUnused,
-      pruned,
-      candidateCount: candidates.length,
-    },
-    "v3 edge learning complete",
-  );
-
-  return { reinforced, skippedUnused, pruned, candidates };
-}
-
-/**
- * Apply the weight floor and the overall / per-source diversity caps to a
- * weight-sorted list of auto-edges. Input must already be sorted heaviest-first
- * (as {@link topByWeight} returns).
- */
-function selectPromotionCandidates(sorted: AutoEdgeRow[]): AutoEdgeRow[] {
-  const out: AutoEdgeRow[] = [];
-  const perSource = new Map<string, number>();
-  for (const edge of sorted) {
-    if (out.length >= MAX_PROMOTION_CANDIDATES) break;
-    if (edge.weight < PROMOTION_WEIGHT_FLOOR) continue;
-    const count = perSource.get(edge.sourceSlug) ?? 0;
-    if (count >= MAX_CANDIDATES_PER_SOURCE) continue;
-    perSource.set(edge.sourceSlug, count + 1);
-    out.push(edge);
-  }
-  return out;
-}
-
-/**
- * Job handler for `memory_v3_edge_learning`. Thin wrapper over
- * {@link runEdgeLearning} so the heavy lifting (and its tests) live in one
- * place. The job carries no payload — it always reconciles the whole recent
- * co-activation log.
- */
-export function memoryV3EdgeLearningJob(_job: MemoryJob): EdgeLearningResult {
-  return runEdgeLearning(getDb());
-}