@vellumai/assistant 0.8.6 → 0.8.7

package/src/memory/v3/consolidation-job.ts

@@ -1,323 +0,0 @@
-/**
- * Memory v3 — `memory_v3_consolidate` job handler.
- *
- * The v3 consolidation job drains the SHARED `memory/buffer.md` (the same
- * buffer v2 uses — there is no v3 buffer) into the SHARED concept pages AND the
- * v3 **tree** overlay, while maintaining the SHARED standing-context files
- * (`essentials.md` / `threads.md` / `recent.md`) byte-for-byte the way v2 does.
- * It is the v3 counterpart to `assistant/src/memory/v2/consolidation-job.ts`
- * and mirrors its orchestration exactly — the only divergences are the gating
- * flag (`memory.v3.write.enabled`), the lock path (`memory/.v3-state/`), and the
- * prompt body (which additionally asks the agent to author/refresh the tree).
- *
- * Because the buffer and the standing-context files are shared, exactly one
- * consolidator may own the drain at a time. The scheduler enforces this: when
- * `memory.v3.write.enabled` is on it enqueues `memory_v3_consolidate` INSTEAD of
- * `memory_v2_consolidate` (see `maybeEnqueueGraphMaintenanceJobs` in
- * `jobs-worker.ts`). Concept pages stay the shared canonical store, so the v2
- * router keeps working off pages v3 writes — it just ignores the tree overlay.
- *
- * Lifecycle (identical to v2 except the flag + lock path + tree-authoring
- * prompt):
- *   1. Bail if `config.memory.v3.write.enabled` is false (the worker may have
- *      claimed a stale row from before the flag was flipped off).
- *   2. Acquire a single-process lock at `memory/.v3-state/consolidation.lock`.
- *   3. Capture the cutoff timestamp at dispatch.
- *   4. Read the shared `memory/buffer.md`. Bail if empty.
- *   5. Hand off to `runBackgroundJob()` with the v3 consolidation prompt
- *      (`suppressFailureNotifications: true`).
- *   6. On success, enqueue follow-ups: `memory_v3_index_maintenance` (mechanical
- *      tree/DAG upkeep) and `embed_concept_page` reembed (pages are shared, so
- *      reembed is still needed — reuse the existing `memory_v2_reembed` fan-out
- *      job type, which enqueues one `embed_concept_page` per slug).
- *   7. Release the lock.
- */
-
-import {
-  closeSync,
-  mkdirSync,
-  openSync,
-  readFileSync,
-  unlinkSync,
-  writeSync,
-} from "node:fs";
-import { dirname, join } from "node:path";
-
-import type { AssistantConfig } from "../../config/types.js";
-import { runBackgroundJob } from "../../runtime/background-job-runner.js";
-import { getLogger } from "../../util/logger.js";
-import { getWorkspaceDir } from "../../util/platform.js";
-import { isProcessAlive } from "../../util/process-liveness.js";
-import { formatBufferTimestamp } from "../graph/tool-handlers.js";
-import {
-  enqueueMemoryJob,
-  type MemoryJob,
-  type MemoryJobType,
-} from "../jobs-store.js";
-// The consolidation conversation `source` is a UI/routing concern shared with
-// v2 (the route layer recognizes "this conversation IS background memory
-// consolidation" by this string). v2 and v3 are mutually exclusive drainers, so
-// reusing the same source keeps that recognition working for both without
-// forking a v3 constant.
-import { MEMORY_V2_CONSOLIDATION_SOURCE } from "../v2/constants.js";
-import { resolveConsolidationPrompt } from "./prompts/consolidation.js";
-
-const log = getLogger("memory-v3-consolidate");
-
-/** Stable identifier surfaced in `runBackgroundJob` logs and notifications. */
-const JOB_NAME = "memory.consolidate";
-
-/**
- * Hard timeout for the consolidation run. Matches v2: consolidation reads the
- * buffer, rewrites several files, re-encodes essentials/threads, and authors
- * the tree — generous upper bound so a slow run isn't killed mid-edit, but
- * bounded so a stuck provider can't pin the worker indefinitely.
- */
-const CONSOLIDATION_TIMEOUT_MS = 15 * 60 * 1000;
-
-/**
- * Follow-up jobs to fan out after a successful consolidation:
- *   - `memory_v3_index_maintenance` — mechanical (no-LLM) tree/DAG upkeep:
- *     validate the tree, report stale composed indices, cycle-check the DAG.
- *   - `memory_v2_reembed` — re-embed every shared concept page (the fan-out job
- *     enqueues one `embed_concept_page` per slug). Pages are shared, so a v3
- *     consolidation that touches them still needs the reembed. Conservatively
- *     re-embeds every page; the embedder's content-hash cache makes unchanged
- *     pages effectively free.
- */
-const FOLLOW_UP_JOB_TYPES: readonly MemoryJobType[] = [
-  "memory_v3_index_maintenance",
-  "memory_v2_reembed",
-] as const;
-
-/**
- * Job handler. See file header for the full lifecycle. Returns a discriminated
- * union so tests can assert on the path taken (disabled / locked / empty /
- * invoked / failed) without having to spy on the filesystem. Mirrors v2's
- * `ConsolidationOutcome`.
- */
-export type ConsolidationOutcome =
-  | { kind: "disabled" }
-  | { kind: "locked"; holder: string }
-  | { kind: "empty_buffer" }
-  | { kind: "run_failed"; reason?: string }
-  | {
-      kind: "invoked";
-      conversationId: string;
-      cutoff: string;
-      followUpJobIds: string[];
-    };
-
-export async function memoryV3ConsolidateJob(
-  _job: MemoryJob,
-  config: AssistantConfig,
-): Promise<ConsolidationOutcome> {
-  if (!config.memory.v3.write.enabled) {
-    log.debug("memory.v3.write.enabled is false; consolidation skipped");
-    return { kind: "disabled" };
-  }
-
-  const memoryDir = join(getWorkspaceDir(), "memory");
-  const lockPath = join(memoryDir, ".v3-state", "consolidation.lock");
-  const bufferPath = join(memoryDir, "buffer.md");
-
-  // Step 1: acquire lock. Bails immediately if another consolidation is
-  // already in flight — the next scheduled run can pick up where we leave off.
-  const holder = tryAcquireLock(lockPath);
-  if (holder !== null) {
-    log.warn({ lockPath, holder }, "consolidation skipped: lock already held");
-    return { kind: "locked", holder };
-  }
-
-  try {
-    // Step 2: capture cutoff. Formatted to match `buffer.md` entry timestamps
-    // (`Mon D, h:mm AM/PM`) so the agent's "timestamp ≥ cutoff" check compares
-    // like-with-like at minute precision. Captured here (not at enqueue time)
-    // so late-claimed rows get a fresh cutoff.
-    const cutoff = formatBufferTimestamp(new Date());
-
-    // Step 3: bail on empty buffer. The shared buffer has no work to drain.
-    const bufferContent = readBufferContent(bufferPath);
-    if (bufferContent.trim().length === 0) {
-      log.debug("buffer.md empty; consolidation skipped");
-      return { kind: "empty_buffer" };
-    }
-
-    // Step 4: hand off to the centralized background-job runner. As with v2,
-    // `suppressFailureNotifications: true` opts out of `activity.failed`
-    // notifications so a network blip on the tight consolidation interval does
-    // not spam the home feed; Sentry-side reporting is unchanged.
-    //
-    // The prompt override config key (`memory.v2.consolidation_prompt_path`) is
-    // shared — there is no separate v3 key, so an operator points one file at
-    // whichever consolidator owns the drain.
-    const runResult = await runBackgroundJob({
-      jobName: JOB_NAME,
-      source: MEMORY_V2_CONSOLIDATION_SOURCE,
-      prompt: resolveConsolidationPrompt(
-        config.memory.v2.consolidation_prompt_path,
-        cutoff,
-      ),
-      trustContext: { sourceChannel: "vellum", trustClass: "guardian" },
-      callSite: "mainAgent",
-      timeoutMs: CONSOLIDATION_TIMEOUT_MS,
-      origin: "memory_consolidation",
-      suppressFailureNotifications: true,
-    });
-
-    if (!runResult.ok) {
-      log.error(
-        {
-          conversationId: runResult.conversationId,
-          errorKind: runResult.errorKind,
-          err: runResult.error?.message,
-        },
-        "consolidation run failed; follow-ups skipped",
-      );
-      return runResult.error?.message !== undefined
-        ? { kind: "run_failed", reason: runResult.error.message }
-        : { kind: "run_failed" };
-    }
-
-    // Step 5: enqueue follow-up jobs (tree maintenance + page reembed).
-    const followUpJobIds: string[] = [];
-    for (const jobType of FOLLOW_UP_JOB_TYPES) {
-      try {
-        followUpJobIds.push(enqueueMemoryJob(jobType, {}));
-      } catch (err) {
-        // Best-effort: a failed enqueue here doesn't undo the agent's writes,
-        // and the next scheduled consolidation will attempt the same fan-out.
-        log.warn(
-          { err, jobType },
-          "consolidation: failed to enqueue follow-up job; continuing",
-        );
-      }
-    }
-
-    log.info(
-      {
-        conversationId: runResult.conversationId,
-        cutoff,
-        followUpJobIds,
-      },
-      "consolidation invoked",
-    );
-    return {
-      kind: "invoked",
-      conversationId: runResult.conversationId,
-      cutoff,
-      followUpJobIds,
-    };
-  } finally {
-    releaseLock(lockPath);
-  }
-}
-
-/**
- * Read `memory/buffer.md`. Missing file → empty string so the skip-on-empty
- * branch doesn't have to distinguish "no file" from "blank file".
- */
-function readBufferContent(bufferPath: string): string {
-  try {
-    return readFileSync(bufferPath, "utf-8");
-  } catch (err) {
-    if ((err as NodeJS.ErrnoException).code === "ENOENT") return "";
-    throw err;
-  }
-}
-
-/**
- * Atomically create the lock file with `wx` (O_CREAT | O_EXCL) flags. Returns
- * `null` on success, or the current holder string when the file already exists
- * and the holder is still alive. Mirrors v2's lock machinery exactly — single
- * writer per workspace, so a holder whose process died is unambiguously stale
- * and is taken over automatically.
- */
-function tryAcquireLock(lockPath: string): string | null {
-  mkdirSync(dirname(lockPath), { recursive: true });
-
-  const firstHolder = tryCreate(lockPath);
-  if (firstHolder === null) return null;
-  if (!isHolderStale(firstHolder)) return firstHolder;
-
-  log.info(
-    { lockPath, holder: firstHolder },
-    "consolidation: taking over stale lock (holder not running)",
-  );
-  try {
-    unlinkSync(lockPath);
-  } catch (err) {
-    const code = (err as NodeJS.ErrnoException).code;
-    if (code !== "ENOENT") {
-      log.warn(
-        { err, lockPath },
-        "consolidation: failed to unlink stale lock; reporting as locked",
-      );
-      return firstHolder;
-    }
-  }
-  return tryCreate(lockPath);
-}
-
-/**
- * Atomically create the lock file. Returns `null` on success, or the holder
- * string read from the file when it already exists (`"unknown"` if the read
- * itself fails). Rethrows any non-EEXIST errno from `openSync`.
- */
-function tryCreate(lockPath: string): string | null {
-  let fd: number;
-  try {
-    fd = openSync(lockPath, "wx");
-  } catch (err) {
-    if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
-    try {
-      return readFileSync(lockPath, "utf-8").trim() || "unknown";
-    } catch {
-      return "unknown";
-    }
-  }
-  try {
-    writeSync(fd, `${process.pid} ${Date.now()}\n`);
-  } catch {
-    // best-effort — payload is advisory, the file's existence is the lock
-  } finally {
-    try {
-      closeSync(fd);
-    } catch {
-      // best-effort
-    }
-  }
-  return null;
-}
-
-/**
- * A holder string is stale when its PID parses to a non-running process. An
- * unparseable / empty / `"unknown"` payload is also treated as stale: the only
- * writer is `tryCreate`, so corruption indicates a partial write from a crashed
- * prior holder rather than a live writer mid-flush.
- */
-function isHolderStale(holder: string): boolean {
-  const match = /^\d+/.exec(holder);
-  if (!match) return true;
-  const pid = Number.parseInt(match[0], 10);
-  if (!Number.isFinite(pid) || pid <= 0) return true;
-  return !isProcessAlive(pid);
-}
-
-/**
- * Idempotent unlink of the lock file. Called from the `finally` block so a
- * crash in the run path doesn't leave the lock stranded. ENOENT is swallowed
- * because the lock may have been released by an operator or never created.
- */
-function releaseLock(lockPath: string): void {
-  try {
-    unlinkSync(lockPath);
-  } catch (err) {
-    const code = (err as NodeJS.ErrnoException).code;
-    if (code === "ENOENT") return;
-    log.warn(
-      { err, lockPath },
-      "consolidation: failed to release lock (best-effort)",
-    );
-  }
-}

package/src/memory/v3/coretrieval-seed.ts

@@ -1,240 +0,0 @@
-/**
- * Memory v3 — co-retrieval edge seeding.
- *
- * Warm-starts the learned association graph (`memory_v3_auto_edges`, migration
- * 263) from the v2 router's selection history. The live edge-learning job
- * (`edge-learning-job.ts`) only accrues edges from v3's own retrievals, which is
- * cold until v3 has run for a long time; the v2 router has thousands of turns of
- * co-selection data already. Seeding projects that history into the same table
- * the `aboveThreshold` read path already consumes, so the edge-expansion lane
- * can merge it (via `expandEdges`'s `extraAdjacency` seam) the moment it's wired.
- *
- * Signal: two pages that the router *selected together* on a turn are associated.
- * Scoring is **NPMI** (normalized pointwise mutual information), not raw
- * co-occurrence — NPMI discounts high-frequency pages, so an always-injected page
- * doesn't edge to everything it merely co-occurs with. A min co-occurrence floor
- * drops noise, and an "always-on" frequency ceiling drops pages selected on a
- * large fraction of all turns (heartbeat/now-md-style infrastructure) that carry
- * no associative signal.
- *
- * `buildCoretrievalGraph` is pure (no I/O) so it is unit-testable; the
- * `seedCoretrievalEdges` driver reads the rows and writes the table.
- */
-
-import { getLogger } from "../../util/logger.js";
-import type { DrizzleDb } from "../db-connection.js";
-import { getSqliteFrom } from "../db-connection.js";
-
-const log = getLogger("memory-v3-coretrieval-seed");
-
-/** A pair must co-occur on at least this many turns to earn an edge. */
-export const DEFAULT_MIN_COUNT = 5;
-/**
- * A candidate *neighbor* selected on more than this fraction of all turns is
- * "always-on" infrastructure (co-occurs with everything) and is excluded — it
- * carries no associative signal, only base-rate noise.
- */
-export const DEFAULT_MAX_NEIGHBOR_FREQ_RATIO = 0.4;
-/** Neighbors kept per source node, ranked by NPMI descending. */
-export const DEFAULT_TOP_K = 20;
-/**
- * Weight of a perfect-association (NPMI = 1) edge. Each edge is persisted at
- * `seedWeight × its NPMI score`, so the read threshold
- * (`memory.v3.edges.learnedAdjacencyThreshold`) acts as a minimum-association
- * cutoff: with the default 2.0, threshold 1.0 ≈ NPMI ≥ 0.5 and 1.2 ≈ NPMI ≥ 0.6.
- */
-export const DEFAULT_SEED_WEIGHT = 2.0;
-
-/** Tuning knobs for {@link buildCoretrievalGraph}. */
-export interface CoretrievalGraphOptions {
-  minCount: number;
-  maxNeighborFreqRatio: number;
-  topK: number;
-}
-
-/** Driver knobs: graph tuning plus the persisted seed weight. */
-export interface SeedCoretrievalOptions extends CoretrievalGraphOptions {
-  seedWeight: number;
-}
-
-export const DEFAULT_SEED_OPTIONS: SeedCoretrievalOptions = {
-  minCount: DEFAULT_MIN_COUNT,
-  maxNeighborFreqRatio: DEFAULT_MAX_NEIGHBOR_FREQ_RATIO,
-  topK: DEFAULT_TOP_K,
-  seedWeight: DEFAULT_SEED_WEIGHT,
-};
-
-/** One scored outgoing edge. */
-export interface ScoredEdge {
-  target: string;
-  score: number;
-}
-
-/** Summary of one seeding run, for the CLI/route report. */
-export interface SeedCoretrievalResult {
-  turnsScanned: number;
-  nodes: number;
-  edgesWritten: number;
-  avgDegree: number;
-}
-
-/**
- * Build the co-retrieval adjacency from per-turn selected-slug sets.
- *
- * Pure: takes the already-extracted selection sets (one array per turn) and
- * returns `source -> ScoredEdge[]` (top-K NPMI neighbors, heaviest first). Turns
- * with fewer than two distinct slugs contribute no pairs and are ignored.
- */
-export function buildCoretrievalGraph(
-  turns: ReadonlyArray<ReadonlyArray<string>>,
-  options: CoretrievalGraphOptions = DEFAULT_SEED_OPTIONS,
-): Map<string, ScoredEdge[]> {
-  const sets = turns.map((t) => [...new Set(t)]).filter((s) => s.length >= 2);
-  const n = sets.length;
-  const graph = new Map<string, ScoredEdge[]>();
-  if (n === 0) return graph;
-
-  const freq = new Map<string, number>();
-  const cooccur = new Map<string, Map<string, number>>();
-  const bump = (a: string, b: string) => {
-    let row = cooccur.get(a);
-    if (!row) cooccur.set(a, (row = new Map()));
-    row.set(b, (row.get(b) ?? 0) + 1);
-  };
-  for (const slugs of sets) {
-    for (const slug of slugs) freq.set(slug, (freq.get(slug) ?? 0) + 1);
-    for (let i = 0; i < slugs.length; i++) {
-      for (let j = i + 1; j < slugs.length; j++) {
-        bump(slugs[i], slugs[j]);
-        bump(slugs[j], slugs[i]);
-      }
-    }
-  }
-
-  const freqCap = options.maxNeighborFreqRatio * n;
-  for (const [source, neighbors] of cooccur) {
-    const fSource = freq.get(source)!;
-    const scored: ScoredEdge[] = [];
-    for (const [target, pairCount] of neighbors) {
-      if (pairCount < options.minCount) continue;
-      const fTarget = freq.get(target)!;
-      if (fTarget > freqCap) continue;
-      // NPMI = pmi / -ln(p(a,b)), in [-1, 1]. Higher = stronger association.
-      // p(a,b)=1 (the pair co-occurs on every turn) is perfect association — its
-      // NPMI limit is 1, but the formula is 0/0, so pin it to avoid NaN.
-      const pab = pairCount / n;
-      if (pab >= 1) {
-        scored.push({ target, score: 1 });
-        continue;
-      }
-      const pmi = Math.log(pab / ((fSource / n) * (fTarget / n)));
-      scored.push({ target, score: pmi / -Math.log(pab) });
-    }
-    scored.sort((a, b) => b.score - a.score);
-    if (scored.length > 0) graph.set(source, scored.slice(0, options.topK));
-  }
-  return graph;
-}
-
-/**
- * Extract one selection set per v2 router turn from `memory_v2_activation_logs`.
- * The set is the router's *fresh* per-turn pick (`status === 'injected'`), which
- * is the co-selection signal we want — not the accumulated in-context carry-over.
- *
- * Best-effort: a missing table (db predates the activation log) or an unparseable
- * row yields no turns rather than throwing — the caller degrades to "no seed".
- */
-function readRouterSelections(database: DrizzleDb): string[][] {
-  const turns: string[][] = [];
-  try {
-    const raw = getSqliteFrom(database);
-    const rows = raw
-      .query(
-        `SELECT concepts_json FROM memory_v2_activation_logs WHERE mode = 'router'`,
-      )
-      .all() as Array<{ concepts_json: string }>;
-    for (const row of rows) {
-      let concepts: unknown;
-      try {
-        concepts = JSON.parse(row.concepts_json);
-      } catch {
-        continue;
-      }
-      if (!Array.isArray(concepts)) continue;
-      const selected = new Set<string>();
-      for (const entry of concepts) {
-        if (
-          entry &&
-          typeof entry === "object" &&
-          (entry as { status?: unknown }).status === "injected" &&
-          typeof (entry as { slug?: unknown }).slug === "string"
-        ) {
-          selected.add((entry as { slug: string }).slug);
-        }
-      }
-      if (selected.size >= 2) turns.push([...selected]);
-    }
-  } catch (err) {
-    log.warn(
-      { err },
-      "failed to read router selections for seeding; continuing",
-    );
-  }
-  return turns;
-}
-
-/**
- * Build the co-retrieval graph from the v2 router history and persist it into
- * `memory_v3_auto_edges`, weighting each edge by its association strength:
- * `weight = seedWeight × NPMI`. Edges whose weight is non-positive (NPMI ≤ 0 —
- * not an association) are skipped. This makes the read threshold a real
- * minimum-association cutoff rather than all-or-nothing.
- *
- * Idempotent: each edge is upserted by OVERWRITING its weight to the current
- * NPMI-derived value (the seed baseline), so re-running is stable. Tradeoff: the
- * overwrite resets any reinforcement the edge-learning job accrued between seeds.
- * That is acceptable while the seed is the dominant signal and organic
- * co-activation reinforcement is minimal; revisit if/when live reinforcement
- * becomes load-bearing.
- */
-export function seedCoretrievalEdges(
-  database: DrizzleDb,
-  options: SeedCoretrievalOptions = DEFAULT_SEED_OPTIONS,
-): SeedCoretrievalResult {
-  const turns = readRouterSelections(database);
-  const graph = buildCoretrievalGraph(turns, options);
-
-  const now = Date.now();
-  let edgesWritten = 0;
-  try {
-    const raw = getSqliteFrom(database);
-    const upsert = raw.prepare(
-      `INSERT INTO memory_v3_auto_edges
-         (source_slug, target_slug, weight, last_reinforced_at)
-         VALUES (?, ?, ?, ?)
-       ON CONFLICT(source_slug, target_slug) DO UPDATE SET
-         weight = excluded.weight,
-         last_reinforced_at = excluded.last_reinforced_at`,
-    );
-    const apply = raw.transaction(() => {
-      for (const [source, edges] of graph) {
-        for (const edge of edges) {
-          const weight = options.seedWeight * edge.score;
-          if (weight <= 0) continue;
-          upsert.run(source, edge.target, weight, now);
-          edgesWritten += 1;
-        }
-      }
-    });
-    apply();
-  } catch (err) {
-    log.warn({ err }, "failed to persist seeded co-retrieval edges");
-  }
-
-  return {
-    turnsScanned: turns.length,
-    nodes: graph.size,
-    edgesWritten,
-    avgDegree: graph.size > 0 ? edgesWritten / graph.size : 0,
-  };
-}