@vellumai/assistant 0.10.12 → 0.11.0-staging.2

package/src/plugins/defaults/memory/substrate/spread.ts

@@ -0,0 +1,165 @@
+// ---------------------------------------------------------------------------
+// Memory substrate — edge-graph activation spreading over the concept-page
+// substrate.
+// ---------------------------------------------------------------------------
+//
+// Implements the spread step of the activation formula (§4 of the design
+// doc):
+//
+//   A(n, t+1) = [ A_o(n)
+//               + k  · Σ_{m∈in1(n)} A_o(m)
+//               + k² · Σ_{m∈in2(n)} A_o(m) ]
+//             / (1 + k · #in1(n) + k² · #in2(n))
+//
+// Edges are directed: edge A→B means A's activation contributes to B's. The
+// per-target BFS walks *incoming* adjacency, so `in1(n)` is the set of nodes
+// with an edge A→n and `in2(n)` adds another hop in the same direction.
+//
+// Bounded in [0, 1]. Pure sources (no incoming edges within `hops`) reduce to
+// A == A_o because both numerator and denominator collapse to `A_o(n)` and
+// `1`, respectively.
+//
+// Consumed by both the v2 injection engine (`v2/activation.ts`) and the
+// `memory` recall source (`context-search/sources/memory-v2.ts`), which runs
+// whenever `usesConceptPageMemory()` holds — i.e. under v3 too.
+
+import { clampUnitInterval } from "../validation.js";
+import type { EdgeIndex } from "./edge-index.js";
+
+interface SpreadActivationResult {
+  /** Final activation value per slug after spreading. */
+  final: Map<string, number>;
+  /**
+   * Per-slug spread delta: `final[slug] - own[slug]`. Captures how much
+   * the spread step nudged each node above (or below) its own activation —
+   * useful for inspector views that want to show graph contributions
+   * separate from raw sim contributions. Always 0 when `hops == 0` or
+   * `k == 0` because both short-circuit to `final == own`.
+   */
+  contribution: Map<string, number>;
+}
+
+/**
+ * Apply 2-hop spreading activation with neighborhood normalization. Edges are
+ * directed: an edge A→B means A's activation contributes to B's final value.
+ *
+ *   A(n) = [ A_o(n) + Σ_{r: |active_inR(n)| > 0} k^r · L2(active_inR(n)) ]
+ *        / [ 1     + Σ_{r: |active_inR(n)| > 0} k^r ]
+ *
+ * `active_inR(n)` is the subset of structural predecessors at hop `r` that
+ * also appear in `ownActivation` (i.e. made the candidate set). `L2(.)` is
+ * the quadratic mean √(mean(A_o²)) — a mild bias toward strong outliers
+ * compared to the arithmetic mean, without letting a single high-cosine
+ * predecessor dominate the way `max` would.
+ *
+ * Hops with **no** active predecessors are dropped from BOTH numerator and
+ * denominator so a high-in-degree hub with mostly-inactive neighbors stays
+ * near `A_o` instead of being crushed by the structural count. A pure
+ * source (no incoming edges, or every edge points at a non-candidate)
+ * collapses to `A == A_o`.
+ *
+ * Bounded in [0, 1]: every `L2` term ≤ max active A_o ≤ 1, so the numerator
+ * is at most `1 + Σ k^r` — exactly the denominator — so the ratio is at most
+ * 1. `clampUnitInterval` guards against numerical drift and out-of-range
+ * inputs.
+ *
+ * Pure function — no I/O. Reads the precomputed `incoming` map from
+ * `edgeIndex` and runs a per-source BFS bounded by `hops`.
+ */
+export function spreadActivation(
+  ownActivation: ReadonlyMap<string, number>,
+  edgeIndex: EdgeIndex,
+  k: number,
+  hops: number,
+): SpreadActivationResult {
+  const final = new Map<string, number>();
+  const contribution = new Map<string, number>();
+  if (ownActivation.size === 0) {
+    return { final, contribution };
+  }
+
+  // Short-circuit: with no spread the formula collapses to A == A_o.
+  if (hops <= 0 || k <= 0) {
+    for (const [slug, ownValue] of ownActivation) {
+      final.set(slug, clampUnitInterval(ownValue));
+      contribution.set(slug, 0);
+    }
+    return { final, contribution };
+  }
+
+  for (const [slug, ownValue] of ownActivation) {
+    // Single bounded BFS from `slug` over incoming edges. `distance` maps
+    // predecessor → hop count (1..hops). Source is excluded so it contributes
+    // hop-0 only via `numerator = ownValue`.
+    const distance = bfsPredecessorDistances(edgeIndex.incoming, slug, hops);
+
+    // Bucket only predecessors that are in `ownActivation` (the candidate
+    // set). Structural predecessors that didn't make the cut contribute
+    // nothing — neither to the numerator nor the denominator — so hub
+    // in-degree alone never penalizes a node.
+    const ringActiveCounts: number[] = new Array(hops + 1).fill(0);
+    const ringSquareSums: number[] = new Array(hops + 1).fill(0);
+    for (const [predecessor, hop] of distance) {
+      const predValue = ownActivation.get(predecessor);
+      if (predValue === undefined) {
+        continue;
+      }
+      ringActiveCounts[hop] += 1;
+      ringSquareSums[hop] += predValue * predValue;
+    }
+
+    let numerator = ownValue;
+    let denominator = 1;
+    let kPow = 1;
+    for (let r = 1; r <= hops; r++) {
+      kPow *= k;
+      if (ringActiveCounts[r] === 0) {
+        continue;
+      }
+      const rms = Math.sqrt(ringSquareSums[r] / ringActiveCounts[r]);
+      numerator += kPow * rms;
+      denominator += kPow;
+    }
+
+    const finalValue = clampUnitInterval(numerator / denominator);
+    final.set(slug, finalValue);
+    contribution.set(slug, finalValue - ownValue);
+  }
+
+  return { final, contribution };
+}
+
+/**
+ * Bounded BFS over the *incoming* adjacency map. Returns each reachable
+ * predecessor's hop-distance in [1, maxHops] from `target` — i.e. nodes from
+ * which a directed path of that length leads into `target`. The target itself
+ * is excluded.
+ */
+function bfsPredecessorDistances(
+  incoming: ReadonlyMap<string, ReadonlySet<string>>,
+  target: string,
+  maxHops: number,
+): Map<string, number> {
+  const distance = new Map<string, number>();
+  let frontier: string[] = [target];
+  const visited = new Set<string>([target]);
+  for (let hop = 1; hop <= maxHops && frontier.length > 0; hop++) {
+    const next: string[] = [];
+    for (const node of frontier) {
+      const predecessors = incoming.get(node);
+      if (!predecessors) {
+        continue;
+      }
+      for (const predecessor of predecessors) {
+        if (visited.has(predecessor)) {
+          continue;
+        }
+        visited.add(predecessor);
+        distance.set(predecessor, hop);
+        next.push(predecessor);
+      }
+    }
+    frontier = next;
+  }
+  return distance;
+}

package/src/plugins/defaults/memory/{v3/substrate → substrate}/static-context.ts

@@ -1,7 +1,10 @@
 // ---------------------------------------------------------------------------
-// Memory v2 — Static context loader for user-message auto-injection
+// Memory substrate — Static context loader for user-message auto-injection
 // ---------------------------------------------------------------------------
 //
+// Shared by the v2 injection engine and memory-v3; active whenever
+// `usesConceptPageMemory()` holds.
+//
 // Reads the four top-level memory files (essentials/threads/recent/buffer)
 // and returns a concatenated, header-wrapped block ready to splice into the
 // current user message via the injector chain.
@@ -17,11 +20,11 @@
 // content through when `mode === "full"` (first turn / post-compaction),
 // matching the existing PKB auto-inject pattern.
 
-import type { ChannelId } from "../../../../../channels/types.js";
-import { usesConceptPageMemory } from "../../../../../config/memory-v3-gate.js";
-import { readPromptFile } from "../../../../../prompts/system-prompt.js";
-import { getMemoryConfig } from "../../config.js";
-import { getWorkspacePromptPath } from "../../paths.js";
+import type { ChannelId } from "../../../../channels/types.js";
+import { usesConceptPageMemory } from "../../../../config/memory-v3-gate.js";
+import { readPromptFile } from "../../../../prompts/system-prompt.js";
+import { getMemoryConfig } from "../config.js";
+import { getWorkspacePromptPath } from "../paths.js";
 
 interface MemoryV2StaticBlock {
   heading: string;
@@ -61,7 +64,9 @@ export function readMemoryV2StaticContent(
       continue;
     }
     const content = readPromptFile(getWorkspacePromptPath(file));
-    if (!content) continue;
+    if (!content) {
+      continue;
+    }
     sections.push(`${heading}\n\n${content}`);
   }
   return sections.length > 0 ? sections.join("\n\n") : null;

package/src/plugins/defaults/memory/{v3/substrate → substrate}/sweep-job.ts

@@ -1,5 +1,8 @@
 /**
- * Memory v2 — `memory_v2_sweep` job handler.
+ * Memory substrate — `memory_v2_sweep` job handler.
+ *
+ * Shared by the v2 injection engine and memory-v3; active whenever
+ * `usesConceptPageMemory()` holds.
  *
  * The sweep is the auto-extraction analog of v1's `graph_extract`: idle-
  * debounced, prompted with the current `memory/buffer.md` so it dedupes
@@ -13,8 +16,8 @@
  * extraction-trigger path. Until then this handler is invoked only by
  * `memory_v2_sweep` rows enqueued explicitly (tests, future CLI).
  *
- * Skipped entirely when concept-page memory is not active, or when
- * `config.memory.v2.sweep_enabled` is false — keeps the sweep dormant in
+ * Skipped entirely when concept-page memory is not active, or when the
+ * `sweep_enabled` substrate tunable is false — keeps the sweep dormant in
  * v1-only workspaces and in concept-page workspaces that haven't opted in,
  * even if a stale row sits in the queue when the gate is off.
  */
@@ -31,31 +34,33 @@ import {
 import { and, desc, eq, gt, notInArray } from "drizzle-orm";
 import { z } from "zod";
 
-import { usesConceptPageMemory } from "../../../../../config/memory-v3-gate.js";
-import type { AssistantConfig } from "../../../../../config/types.js";
-import { emitNotificationSignal } from "../../../../../notifications/emit-signal.js";
-import { getDb } from "../../../../../persistence/db-connection.js";
-import type { MemoryJob } from "../../../../../persistence/jobs-store.js";
+import { usesConceptPageMemory } from "../../../../config/memory-v3-gate.js";
+import type { AssistantConfig } from "../../../../config/types.js";
+import { emitNotificationSignal } from "../../../../notifications/emit-signal.js";
+import { getDb } from "../../../../persistence/db-connection.js";
+import type { MemoryJob } from "../../../../persistence/jobs-store.js";
 import {
   conversations,
   messages,
-} from "../../../../../persistence/schema/index.js";
+} from "../../../../persistence/schema/index.js";
 import {
   appendBufferAndArchive,
   formatRememberEntry,
-} from "../../graph/tool-handlers.js";
+} from "../graph/tool-handlers.js";
 import {
   extractToolUse,
   type ToolDefinition,
   userMessage,
-} from "../../llm-helpers.js";
-import { getLogger } from "../../logging.js";
-import { getWorkspaceDir } from "../../paths.js";
+} from "../llm-helpers.js";
+import { getLogger } from "../logging.js";
+import { getWorkspaceDir } from "../paths.js";
 import { renderSweepPrompt } from "./prompts/sweep.js";
+import { resolveSubstrateTuning } from "./tuning.js";
 
 const log = getLogger("memory-v2-sweep");
 
 /** Stable job identifier surfaced in `activity.failed` notifications. */
+// FROZEN: wire-visible notification job identifier — never rename the value.
 const JOB_NAME = "memory.v2.sweep";
 
 /** Window of conversation history the sweep inspects on each run. */
@@ -120,8 +125,8 @@ export async function memoryV2SweepJob(
     return 0;
   }
 
-  if (!config.memory.v2.sweep_enabled) {
-    log.debug("memory.v2.sweep_enabled is false; sweep skipped");
+  if (!resolveSubstrateTuning(config.memory).sweep_enabled) {
+    log.debug("the sweep_enabled substrate tunable is false; sweep skipped");
     return 0;
   }
 

package/src/plugins/defaults/memory/substrate/tuning.ts

@@ -0,0 +1,93 @@
+// ---------------------------------------------------------------------------
+// Memory substrate — Config tuning resolver
+// ---------------------------------------------------------------------------
+
+import type { MemoryConfig } from "../../../../config/types.js";
+
+/**
+ * Effective substrate tunables after namespace resolution. Key names match
+ * the `memory.substrate` schema; `spread_k` / `spread_hops` are the
+ * substrate names for `memory.v2.k` / `memory.v2.hops`.
+ */
+export type SubstrateTuning = {
+  sweep_enabled: boolean;
+  dense_weight: number;
+  sparse_weight: number;
+  min_sparse_spread: number | undefined;
+  full_sparse_spread: number | undefined;
+  bm25_k1: number;
+  bm25_b: number;
+  consolidation_interval_hours: number;
+  consolidation_max_buffer_lines: number | null;
+  consolidation_max_entries_per_run: number | null;
+  max_page_chars: number;
+  consolidation_prompt_path: string | null;
+  spread_k: number;
+  spread_hops: number;
+  ann_candidate_limit: number | null;
+};
+
+/**
+ * `memory.substrate` value when the key is present — an explicit `null` on a
+ * nullable key counts as present and wins — and the `memory.v2` twin
+ * otherwise.
+ */
+function orV2<T>(substrateValue: T | undefined, v2Value: T): T {
+  return substrateValue !== undefined ? substrateValue : v2Value;
+}
+
+/**
+ * The memory-config subset the resolver reads. `substrate` is accepted as
+ * absent so hand-built partial config slices (test fixtures) resolve to the
+ * `memory.v2` values; schema-parsed configs always carry it.
+ */
+type SubstrateTuningSource = Pick<MemoryConfig, "v2"> &
+  Partial<Pick<MemoryConfig, "substrate">>;
+
+/**
+ * Single choke point for substrate tunables. Values resolve from
+ * `memory.substrate`, falling back to the historical `memory.v2` keys, which
+ * the v2 injection engine also reads — the v2 schema supplies the effective
+ * defaults for keys neither namespace sets explicitly.
+ */
+export function resolveSubstrateTuning(
+  memory: SubstrateTuningSource,
+): SubstrateTuning {
+  const substrate = memory.substrate ?? {};
+  const v2 = memory.v2;
+  return {
+    sweep_enabled: orV2(substrate.sweep_enabled, v2.sweep_enabled),
+    dense_weight: orV2(substrate.dense_weight, v2.dense_weight),
+    sparse_weight: orV2(substrate.sparse_weight, v2.sparse_weight),
+    min_sparse_spread: orV2(substrate.min_sparse_spread, v2.min_sparse_spread),
+    full_sparse_spread: orV2(
+      substrate.full_sparse_spread,
+      v2.full_sparse_spread,
+    ),
+    bm25_k1: orV2(substrate.bm25_k1, v2.bm25_k1),
+    bm25_b: orV2(substrate.bm25_b, v2.bm25_b),
+    consolidation_interval_hours: orV2(
+      substrate.consolidation_interval_hours,
+      v2.consolidation_interval_hours,
+    ),
+    consolidation_max_buffer_lines: orV2(
+      substrate.consolidation_max_buffer_lines,
+      v2.consolidation_max_buffer_lines,
+    ),
+    consolidation_max_entries_per_run: orV2(
+      substrate.consolidation_max_entries_per_run,
+      v2.consolidation_max_entries_per_run,
+    ),
+    max_page_chars: orV2(substrate.max_page_chars, v2.max_page_chars),
+    consolidation_prompt_path: orV2(
+      substrate.consolidation_prompt_path,
+      v2.consolidation_prompt_path,
+    ),
+    spread_k: orV2(substrate.spread_k, v2.k),
+    spread_hops: orV2(substrate.spread_hops, v2.hops),
+    ann_candidate_limit: orV2(
+      substrate.ann_candidate_limit,
+      v2.ann_candidate_limit,
+    ),
+  };
+}

package/src/plugins/defaults/memory/{v3/substrate → substrate}/types.ts

@@ -1,15 +1,18 @@
 // ---------------------------------------------------------------------------
-// Memory v2 — Shared types
+// Memory substrate — Shared types
 // ---------------------------------------------------------------------------
 //
-// Types shared across the v2 memory subsystem. Most values here cross a
+// Shared by the v2 injection engine and memory-v3; active whenever
+// `usesConceptPageMemory()` holds.
+//
+// Types shared across the concept-page substrate. Most values here cross a
 // serialization boundary — YAML frontmatter, on-disk JSON, or a SQLite JSON
 // column — so they ship as Zod schemas with inferred TypeScript types so
 // runtime validation runs wherever they are read. The skill-autoinjection
 // entry stays a plain `interface` because it is purely in-process.
 //
-// This file must not import from any other `memory/v2/*` module — it is the
-// leaf of the v2 dependency graph.
+// This file must not import from any other substrate module — it is the
+// leaf of the substrate dependency graph.
 
 import { z } from "zod";
 

package/src/plugins/defaults/memory/tools.test.ts

@@ -32,7 +32,7 @@ const recallCalls: Array<{
 }> = [];
 let recallContent = "agentic recall answer";
 
-mock.module("./jobs/embed-pkb-file.js", () => ({
+mock.module("./v1/jobs/embed-pkb-file.js", () => ({
   enqueuePkbIndexJob: (input: { pkbRoot: string; absPath: string }) => {
     enqueueCalls.push(input);
     if (enqueueShouldThrow) {

package/src/plugins/defaults/memory/tools.ts

@@ -25,7 +25,7 @@ import {
   graphRememberDefinition,
 } from "./graph/tools.js";
 import { getWorkspaceDir } from "./paths.js";
-import { deletePage } from "./v3/substrate/page-store.js";
+import { deletePage } from "./substrate/page-store.js";
 
 // ── remember ────────────────────────────────────────────────────────
 

package/src/plugins/defaults/memory/v1/README.md

@@ -0,0 +1,5 @@
+# Memory v1
+
+The legacy (tier-v1) memory engine: PKB knowledge base, segment/summary embedding jobs, and PKB filing. Active only when `isMemoryV1Active()` (see `assistant/src/config/memory-v3-gate.ts`). The v1 graph read engine lives in `v1/graph/`; the all-tier graph store and dispatcher remain in `../graph/`. Do not add features here.
+
+Slated for deletion — see the runbook in `../AGENTS.md` ("Deletion runbook: v1") for the blocking prerequisites, the external consumers to repoint or retire, and the ordered steps. In particular `migrateToolCreatedItems` in `v1/graph/bootstrap.ts` must be MOVED, not deleted: it is a registered, append-only migration step.

package/src/plugins/defaults/memory/{__tests__ → v1/__tests__}/filing-jobs.test.ts

@@ -19,15 +19,15 @@ import {
   test,
 } from "bun:test";
 
-import { createMockLoggerModule } from "../../../../__tests__/helpers/mock-logger.js";
+import { createMockLoggerModule } from "../../../../../__tests__/helpers/mock-logger.js";
 
-mock.module("../../../../util/logger.js", () => createMockLoggerModule());
+mock.module("../../../../../util/logger.js", () => createMockLoggerModule());
 
 const runBackgroundJobMock = mock(
   async (_opts: Record<string, unknown>) =>
     ({ ok: true, conversationId: "conv-filing-test" }) as const,
 );
-mock.module("../../../../runtime/background-job-runner.js", () => ({
+mock.module("../../../../../runtime/background-job-runner.js", () => ({
   runBackgroundJob: (opts: Record<string, unknown>) =>
     runBackgroundJobMock(opts),
 }));
@@ -54,7 +54,7 @@ afterAll(() => {
 
 const { pkbCompactionJob, pkbFilingJob } = await import("../filing-jobs.js");
 
-import type { MemoryJob } from "../../../../persistence/jobs-store.js";
+import type { MemoryJob } from "../../../../../persistence/jobs-store.js";
 
 function makeJob(payload: Record<string, unknown> = {}): MemoryJob {
   return {

package/src/plugins/defaults/memory/{filing-jobs.ts → v1/filing-jobs.ts}

@@ -17,9 +17,9 @@
 
 import type { LLMCallSite } from "@vellumai/plugin-api";
 
-import type { MemoryJob } from "../../../persistence/jobs-store.js";
-import { runBackgroundJob } from "../../../runtime/background-job-runner.js";
-import { getLogger } from "./logging.js";
+import type { MemoryJob } from "../../../../persistence/jobs-store.js";
+import { runBackgroundJob } from "../../../../runtime/background-job-runner.js";
+import { getLogger } from "../logging.js";
 import { hasPkbBufferContent } from "./pkb-schedule.js";
 
 const log = getLogger("filing-jobs");

package/src/plugins/defaults/memory/{graph → v1/graph}/bootstrap.test.ts

@@ -1,15 +1,32 @@
-import { beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { mkdirSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import {
+  afterAll,
+  beforeAll,
+  beforeEach,
+  describe,
+  expect,
+  test,
+} from "bun:test";
+
+import { eq } from "drizzle-orm";
 
-import { setMemoryCheckpoint } from "../../../../persistence/checkpoints.js";
-import { getMemoryDb } from "../../../../persistence/db-connection.js";
-import { initializeDb } from "../../../../persistence/db-init.js";
+import { assertNotLiveDb } from "../../../../../__tests__/assert-not-live-db.js";
+import { setConfig } from "../../../../../__tests__/helpers/set-config.js";
+import { setMemoryCheckpoint } from "../../../../../persistence/checkpoints.js";
+import { getMemoryDb } from "../../../../../persistence/db-connection.js";
+import { initializeDb } from "../../../../../persistence/db-init.js";
 import {
   rawAll,
   rawGet,
   rawRun,
   resetTestTables,
-} from "../../../../persistence/raw-query.js";
-import { migrateToolCreatedItems } from "./bootstrap.js";
+} from "../../../../../persistence/raw-query.js";
+import { memoryJobs } from "../../../../../persistence/schema/index.js";
+import {
+  maybeEnqueueGraphBootstrap,
+  migrateToolCreatedItems,
+} from "./bootstrap.js";
 
 // ---------------------------------------------------------------------------
 // The checkpoint key used by migrateToolCreatedItems (not exported, so we
@@ -56,6 +73,51 @@ beforeEach(() => {
   getMemoryDb()!.run("DELETE FROM memory_jobs");
 });
 
+// ---------------------------------------------------------------------------
+// maybeEnqueueGraphBootstrap
+// ---------------------------------------------------------------------------
+
+describe("maybeEnqueueGraphBootstrap", () => {
+  // A journal directory counts as historical data to bootstrap from, so on a
+  // v1 config an empty graph plus this directory triggers an enqueue.
+  const journalDir = join(process.env.VELLUM_WORKSPACE_DIR!, "journal");
+
+  beforeEach(() => {
+    mkdirSync(journalDir, { recursive: true });
+  });
+
+  afterAll(() => {
+    assertNotLiveDb(journalDir);
+    rmSync(journalDir, { recursive: true, force: true });
+    setConfig("memory", {});
+  });
+
+  function countBootstrapJobs(): number {
+    return getMemoryDb()!
+      .select()
+      .from(memoryJobs)
+      .where(eq(memoryJobs.type, "graph_bootstrap"))
+      .all().length;
+  }
+
+  test("does not enqueue while concept-page memory is active", () => {
+    // `memory.v2.enabled` defaults true, so the substrate is active.
+    setConfig("memory", {});
+
+    maybeEnqueueGraphBootstrap();
+
+    expect(countBootstrapJobs()).toBe(0);
+  });
+
+  test("enqueues on a v1 config when historical data exists", () => {
+    setConfig("memory", { v2: { enabled: false } });
+
+    maybeEnqueueGraphBootstrap();
+
+    expect(countBootstrapJobs()).toBe(1);
+  });
+});
+
 // ---------------------------------------------------------------------------
 // migrateToolCreatedItems
 // ---------------------------------------------------------------------------