@vellumai/assistant 0.7.1 → 0.7.2

package/src/memory/v2/activation.ts

@@ -32,6 +32,7 @@
 //     for the next turn and drop below `epsilon` if no longer relevant.
 
 import type { AssistantConfig } from "../../config/types.js";
+import { applyCorrectionIfCalibrated } from "../anisotropy.js";
 import {
   embedWithBackend,
   generateSparseEmbedding,
@@ -40,22 +41,29 @@ import { clampUnitInterval } from "../validation.js";
 import type { EdgeIndex } from "./edge-index.js";
 import { hybridQueryConceptPages } from "./qdrant.js";
 import { simBatch, simSkillBatch } from "./sim.js";
-import { hybridQuerySkills } from "./skill-qdrant.js";
 import type { ActivationState, EverInjectedEntry } from "./types.js";
 
 /**
- * Top-K size for the un-restricted ANN candidate query against the v2
- * concept-page collection. The design doc fixes this at 50 — small enough to
- * keep the per-turn round-trip cheap, large enough to surface relevant pages
- * outside the prior active set.
+ * Sentinel passed to Qdrant when `config.memory.v2.ann_candidate_limit` is
+ * `null` (unlimited). Qdrant's query API requires an explicit numeric
+ * `limit`, so unlimited is represented as a number large enough that any
+ * realistic concept-page collection is returned in full.
+ *
+ * Why not `Number.MAX_SAFE_INTEGER`: Qdrant's sparse-vector `SearchContext`
+ * pre-allocates `limit * 16` bytes per query, so passing `MAX_SAFE_INTEGER`
+ * triggers a ~144 PB allocation and SIGABRTs the Qdrant process. 1_000_000
+ * is ~16 MB of pre-allocation in Qdrant — generous headroom over realistic
+ * concept-page counts (low thousands today) while staying well clear of
+ * the OOM cliff. Bump explicitly via `ann_candidate_limit` if you ever
+ * outgrow it.
  */
-const ANN_CANDIDATE_LIMIT = 50;
+const UNLIMITED_ANN_CANDIDATE_LIMIT = 1_000_000;
 
 // ---------------------------------------------------------------------------
 // Candidate selection
 // ---------------------------------------------------------------------------
 
-export interface SelectCandidatesParams {
+interface SelectCandidatesParams {
   /**
    * Prior-turn activation snapshot. Slugs with activation strictly greater
    * than `config.memory.v2.epsilon` are carried forward as candidates so the
@@ -71,7 +79,7 @@ export interface SelectCandidatesParams {
   config: AssistantConfig;
 }
 
-export interface SelectCandidatesResult {
+interface SelectCandidatesResult {
   /** Union of `fromPrior` and `fromAnn` — the per-turn candidate set. */
   candidates: Set<string>;
   /** Slugs carried forward from `priorState` because their activation > epsilon. */
@@ -118,13 +126,15 @@ export async function selectCandidates(
 
   if (annQueryText.length > 0) {
     const denseResult = await embedWithBackend(config, [annQueryText]);
-    const dense = denseResult.vectors[0];
-    const sparse = generateSparseEmbedding(annQueryText);
-    const hits = await hybridQueryConceptPages(
-      dense,
-      sparse,
-      ANN_CANDIDATE_LIMIT,
+    const dense = await applyCorrectionIfCalibrated(
+      denseResult.vectors[0],
+      denseResult.provider,
+      denseResult.model,
     );
+    const sparse = generateSparseEmbedding(annQueryText);
+    const limit =
+      config.memory.v2.ann_candidate_limit ?? UNLIMITED_ANN_CANDIDATE_LIMIT;
+    const hits = await hybridQueryConceptPages(dense, sparse, limit);
     for (const hit of hits) fromAnn.add(hit.slug);
   }
 
@@ -137,7 +147,7 @@ export async function selectCandidates(
 // Own activation
 // ---------------------------------------------------------------------------
 
-export interface ComputeOwnActivationParams {
+interface ComputeOwnActivationParams {
   candidates: ReadonlySet<string>;
   priorState: ActivationState | null;
   userText: string;
@@ -151,7 +161,7 @@ export interface ComputeOwnActivationParams {
  * coefficient weighting is applied. Surfaced for telemetry / inspector views
  * so the UI can show how each term contributed to the final value.
  */
-export interface OwnActivationBreakdown {
+interface OwnActivationBreakdown {
   /** `d * prev(slug)` — the decayed prior-turn activation contribution. */
   priorContribution: number;
   /** Raw `sim(user, slug)` similarity, before `c_user` weighting. */
@@ -162,7 +172,7 @@ export interface OwnActivationBreakdown {
   simNow: number;
 }
 
-export interface ComputeOwnActivationResult {
+interface ComputeOwnActivationResult {
   /** Final clamped own-activation value per slug. */
   activation: Map<string, number>;
   /** Per-slug breakdown of the inputs that fed into `activation`. */
@@ -222,7 +232,7 @@ export async function computeOwnActivation(
 // Spreading activation
 // ---------------------------------------------------------------------------
 
-export interface SpreadActivationResult {
+interface SpreadActivationResult {
   /** Final activation value per slug after spreading. */
   final: Map<string, number>;
   /**
@@ -239,20 +249,25 @@ export interface SpreadActivationResult {
  * Apply 2-hop spreading activation with neighborhood normalization. Edges are
  * directed: an edge A→B means A's activation contributes to B's final value.
  *
- *   A(n) = [ A_o(n) + k · Σ_{m∈in1(n)} A_o(m) + k² · Σ_{m∈in2(n)} A_o(m) ]
- *        / (1 + k · #in1(n) + k² · #in2(n))
+ *   A(n) = [ A_o(n) + Σ_{r: |active_inR(n)| > 0} k^r · L2(active_inR(n)) ]
+ *        / [ 1     + Σ_{r: |active_inR(n)| > 0} k^r ]
+ *
+ * `active_inR(n)` is the subset of structural predecessors at hop `r` that
+ * also appear in `ownActivation` (i.e. made the candidate set). `L2(.)` is
+ * the quadratic mean √(mean(A_o²)) — a mild bias toward strong outliers
+ * compared to the arithmetic mean, without letting a single high-cosine
+ * predecessor dominate the way `max` would.
  *
- * For each candidate slug `n`, BFS walks `incoming` adjacency to gather
- * predecessors at distance 1, 2 (i.e. nodes from which a directed path of
- * that length leads into `n`). The denominator counts those structural
- * predecessors at each hop (whether or not they appear in `ownActivation`)
- * so a pure source — no incoming edges — collapses to `A == A_o`. Missing
- * predecessors contribute 0 to the numerator.
+ * Hops with **no** active predecessors are dropped from BOTH numerator and
+ * denominator so a high-in-degree hub with mostly-inactive neighbors stays
+ * near `A_o` instead of being crushed by the structural count. A pure
+ * source (no incoming edges, or every edge points at a non-candidate)
+ * collapses to `A == A_o`.
  *
- * Bounded in [0, 1]: with `A_o ∈ [0, 1]` and `k ∈ [0, 1]`, the numerator is
- * at most `1 + k · #in1 + k² · #in2` — exactly the denominator — so the
- * ratio is at most 1. `clampUnitInterval` guards against numerical drift
- * and out-of-range inputs.
+ * Bounded in [0, 1]: every `L2` term ≤ max active A_o ≤ 1, so the numerator
+ * is at most `1 + Σ k^r` — exactly the denominator — so the ratio is at most
+ * 1. `clampUnitInterval` guards against numerical drift and out-of-range
+ * inputs.
  *
  * Pure function — no I/O. Reads the precomputed `incoming` map from
  * `edgeIndex` and runs a per-source BFS bounded by `hops`.
@@ -282,22 +297,28 @@ export function spreadActivation(
     // hop-0 only via `numerator = ownValue`.
     const distance = bfsPredecessorDistances(edgeIndex.incoming, slug, hops);
 
+    // Bucket only predecessors that are in `ownActivation` (the candidate
+    // set). Structural predecessors that didn't make the cut contribute
+    // nothing — neither to the numerator nor the denominator — so hub
+    // in-degree alone never penalizes a node.
+    const ringActiveCounts: number[] = new Array(hops + 1).fill(0);
+    const ringSquareSums: number[] = new Array(hops + 1).fill(0);
+    for (const [predecessor, hop] of distance) {
+      const predValue = ownActivation.get(predecessor);
+      if (predValue === undefined) continue;
+      ringActiveCounts[hop] += 1;
+      ringSquareSums[hop] += predValue * predValue;
+    }
+
     let numerator = ownValue;
     let denominator = 1;
     let kPow = 1;
-    // Accumulate per-hop contributions in a single pass. We need per-hop
-    // counts to weight by k^r, so bucket as we go.
-    const ringCounts: number[] = new Array(hops + 1).fill(0);
-    const ringSums: number[] = new Array(hops + 1).fill(0);
-    for (const [predecessor, hop] of distance) {
-      ringCounts[hop] += 1;
-      ringSums[hop] += ownActivation.get(predecessor) ?? 0;
-    }
     for (let r = 1; r <= hops; r++) {
       kPow *= k;
-      if (ringCounts[r] === 0) continue;
-      numerator += kPow * ringSums[r];
-      denominator += kPow * ringCounts[r];
+      if (ringActiveCounts[r] === 0) continue;
+      const rms = Math.sqrt(ringSquareSums[r] / ringActiveCounts[r]);
+      numerator += kPow * rms;
+      denominator += kPow;
     }
 
     const finalValue = clampUnitInterval(numerator / denominator);
@@ -347,7 +368,7 @@ function bfsPredecessorDistances(
 // Injection selection
 // ---------------------------------------------------------------------------
 
-export interface SelectInjectionsParams {
+interface SelectInjectionsParams {
   /** Final activation map after spread. */
   A: ReadonlyMap<string, number>;
   /** Slugs already attached to a prior user message (with their turn). */
@@ -356,7 +377,7 @@ export interface SelectInjectionsParams {
   topK: number;
 }
 
-export interface SelectInjectionsResult {
+interface SelectInjectionsResult {
   /** Top-K slugs by activation (descending), used for the cached top-now view. */
   topNow: string[];
   /**
@@ -408,47 +429,7 @@ export function selectInjections(
 // c_assistant, c_now}` — the design doc (§9) deliberately shares them with
 // concept-page activation rather than introducing parallel knobs.
 
-export interface SelectSkillCandidatesParams {
-  userText: string;
-  assistantText: string;
-  nowText: string;
-  config: AssistantConfig;
-  /** Top-K size for the ANN query against `memory_v2_skills`. */
-  topK: number;
-}
-
-/**
- * ANN top-K against the skills collection using the concatenated turn text.
- * Runs a single embedding pass over `concat(user, assistant, now)` and a
- * single hybrid Qdrant query — there is no prior-state carry-forward (skills
- * are stateless).
- *
- * Returns a `Set<string>` of skill ids that hit either channel. Empty when
- * the joined text is empty or `topK <= 0`.
- */
-export async function selectSkillCandidates(
-  params: SelectSkillCandidatesParams,
-): Promise<Set<string>> {
-  const { userText, assistantText, nowText, config, topK } = params;
-
-  const candidates = new Set<string>();
-  if (topK <= 0) return candidates;
-
-  const annQueryText = [userText, assistantText, nowText]
-    .filter((s) => s.length > 0)
-    .join("\n");
-  if (annQueryText.length === 0) return candidates;
-
-  const denseResult = await embedWithBackend(config, [annQueryText]);
-  const dense = denseResult.vectors[0];
-  const sparse = generateSparseEmbedding(annQueryText);
-  const hits = await hybridQuerySkills(dense, sparse, topK);
-  for (const hit of hits) candidates.add(hit.id);
-
-  return candidates;
-}
-
-export interface ComputeSkillActivationParams {
+interface ComputeSkillActivationParams {
   candidates: ReadonlySet<string>;
   userText: string;
   assistantText: string;
@@ -461,7 +442,7 @@ export interface ComputeSkillActivationParams {
  * coefficient weighting. Skills have no decay term, so the breakdown is just
  * the three raw sims. Surfaced for telemetry / inspector views.
  */
-export interface SkillActivationBreakdown {
+interface SkillActivationBreakdown {
   /** Raw `sim(user, skill)` similarity, before `c_user` weighting. */
   simUser: number;
   /** Raw `sim(assistant, skill)` similarity, before `c_assistant` weighting. */
@@ -470,7 +451,7 @@ export interface SkillActivationBreakdown {
   simNow: number;
 }
 
-export interface ComputeSkillActivationResult {
+interface ComputeSkillActivationResult {
   /** Final clamped skill-activation value per id. */
   activation: Map<string, number>;
   /** Per-skill breakdown of the raw sim inputs that fed into `activation`. */
@@ -520,14 +501,14 @@ export async function computeSkillActivation(
   return { activation, breakdown };
 }
 
-export interface SelectSkillInjectionsParams {
+interface SelectSkillInjectionsParams {
   /** Final skill activation map. */
   A: ReadonlyMap<string, number>;
   /** Cap on the per-turn skill slate, e.g. `config.memory.v2.skills_top_k`. */
   topK: number;
 }
 
-export interface SelectSkillInjectionsResult {
+interface SelectSkillInjectionsResult {
   /**
    * Top-K skill ids by activation (descending), tie-broken lexicographically.
    * Skills are re-presented every turn — no `toInject` delta — so the caller

package/src/memory/v2/consolidation-job.ts

@@ -66,13 +66,11 @@ import {
   type MemoryJob,
   type MemoryJobType,
 } from "../jobs-store.js";
+import { MEMORY_V2_CONSOLIDATION_SOURCE } from "./constants.js";
 import { resolveConsolidationPrompt } from "./prompts/consolidation.js";
 
 const log = getLogger("memory-v2-consolidate");
 
-/** Source string identifying this wake in `agent-wake` logs and surfaces. */
-const WAKE_SOURCE = "memory_v2_consolidation";
-
 /**
  * Follow-up jobs to fan out after a successful consolidation.
  *
@@ -144,7 +142,7 @@ export async function memoryV2ConsolidateJob(
     // writes in the assistant's voice.
     const conversation = bootstrapConversation({
       conversationType: "background",
-      source: WAKE_SOURCE,
+      source: MEMORY_V2_CONSOLIDATION_SOURCE,
       origin: "memory_consolidation",
       systemHint: "Running memory consolidation",
       groupId: "system:background",
@@ -159,7 +157,7 @@ export async function memoryV2ConsolidateJob(
           config.memory.v2.consolidation_prompt_path,
           cutoff,
         ),
-        source: WAKE_SOURCE,
+        source: MEMORY_V2_CONSOLIDATION_SOURCE,
         trustContext: INTERNAL_GUARDIAN_TRUST_CONTEXT,
       });
       wakeInvoked = result.invoked;

package/src/memory/v2/constants.ts

@@ -0,0 +1,7 @@
+/**
+ * Canonical `conversations.source` string for background memory v2
+ * consolidation runs. Lives in a tiny constants module so the route layer can
+ * recognize consolidation conversations without importing the consolidation
+ * job (which pulls in agent-wake + bootstrap dependencies).
+ */
+export const MEMORY_V2_CONSOLIDATION_SOURCE = "memory_v2_consolidation";

package/src/memory/v2/injection.ts

@@ -37,14 +37,13 @@ import {
   computeSkillActivation,
   selectCandidates,
   selectInjections,
-  selectSkillCandidates,
   selectSkillInjections,
   spreadActivation,
 } from "./activation.js";
 import { hydrate, save } from "./activation-store.js";
 import { getEdgeIndex } from "./edge-index.js";
 import { readPage, renderPageContent } from "./page-store.js";
-import { getSkillCapability } from "./skill-store.js";
+import { getAllSkillIds, getSkillCapability } from "./skill-store.js";
 import type { ActivationState, EverInjectedEntry } from "./types.js";
 
 const log = getLogger("memory-v2-injection");
@@ -89,10 +88,11 @@ export interface InjectMemoryV2BlockParams {
 
 export interface InjectMemoryV2BlockResult {
   /**
-   * Rendered `<memory>` block, ready to prepend to the current
-   * user message — or `null` when nothing new is eligible for injection.
-   * `null` is the cache-stable default: the caller adds nothing to the new
-   * user message and prior attachments stay byte-identical.
+   * Inner content for the `<memory>` block, ready for the caller to wrap
+   * exactly once at injection time — or `null` when nothing new is eligible
+   * for injection. `null` is the cache-stable default: the caller adds
+   * nothing to the new user message and prior attachments stay
+   * byte-identical.
    */
   block: string | null;
   /**
@@ -183,16 +183,11 @@ export async function injectMemoryV2Block(
   const slugsToRender = mode === "context-load" ? topNow : toInject;
 
   // (6b) Skill pipeline — a sibling pipeline to the concept-page one above.
-  // Skills are stateless: no decay carry-over, no spread, no `everInjected`
-  // dedup. The top-K relevant skills are re-presented every turn so the
-  // agent can drop and pick them up freely.
-  const skillCandidates = await selectSkillCandidates({
-    userText: userMessage,
-    assistantText: assistantMessage,
-    nowText,
-    config,
-    topK: config.memory.v2.top_k_skills,
-  });
+  // Skills are stateless (no decay, no spread, no `everInjected` dedup) and
+  // the catalog is small, so every known skill is scored every turn. The
+  // top-K injection slate is re-presented every turn so the agent can drop
+  // and pick skills up freely; the inspector renders the full ranked list.
+  const skillCandidates = new Set(getAllSkillIds());
   const { activation: skillActivation, breakdown: skillBreakdown } =
     await computeSkillActivation({
       candidates: skillCandidates,
@@ -241,9 +236,30 @@ export async function injectMemoryV2Block(
 
   await save(database, conversationId, nextActivationState);
 
-  // Record per-turn activation telemetry. This runs *before* the cache-stable
-  // empty-block return so we capture diagnostics even on no-op turns. Failures
-  // are warn-logged and never block memory injection.
+  // Render before recording telemetry so the activation log can mark slugs
+  // whose backing file is gone — those are no-op renders that would otherwise
+  // be indistinguishable from successful "injected" rows in the log.
+  // `renderInjectionBlock` itself short-circuits on empty inputs.
+  const { block, missingSlugs } = await renderInjectionBlock(
+    workspaceDir,
+    slugsToRender,
+    topSkillIds,
+  );
+  const missingSlugSet = new Set(missingSlugs);
+  if (missingSlugs.length > 0) {
+    log.warn(
+      {
+        conversationId,
+        turn: currentTurn,
+        missingSlugs,
+        renderedCount: slugsToRender.length - missingSlugs.length,
+      },
+      "Memory v2 injection skipped slugs whose page was missing on disk — Qdrant index may be stale; consider reembed",
+    );
+  }
+
+  // Record per-turn activation telemetry. Failures are warn-logged and never
+  // block memory injection.
   const toInjectSet = new Set(toInject);
   const renderedSet = new Set(slugsToRender);
   const topSkillIdSet = new Set(topSkillIds);
@@ -260,6 +276,10 @@ export async function injectMemoryV2Block(
       //   - per-turn: cached attachments from prior turns are still on the
       //     user message, so prior-everInjected slugs are `in_context` and
       //     the delta (`toInject`) is `injected`.
+      // `page_missing` overrides any "would-have-been-injected" status when
+      // `readPage` returned null for the slug — telemetry surfaces stale
+      // ANN/edge entries instead of silently masquerading as a successful
+      // injection.
       let status: MemoryV2ConceptRowRecord["status"];
       if (mode === "context-load") {
         status = renderedSet.has(slug) ? "injected" : "not_injected";
@@ -270,6 +290,9 @@ export async function injectMemoryV2Block(
       } else {
         status = "not_injected";
       }
+      if (status === "injected" && missingSlugSet.has(slug)) {
+        status = "page_missing";
+      }
       return {
         slug,
         finalActivation: finalActivation.get(slug) ?? 0,
@@ -327,23 +350,6 @@ export async function injectMemoryV2Block(
     );
   }
 
-  // (7) Cache-stable empty path: nothing to render AND no ranked skills.
-  if (slugsToRender.length === 0 && topSkillIds.length === 0) {
-    return { block: null, toInject: [] };
-  }
-
-  // (8) Render. Both `topNow` and `toInject` are activation-descending
-  // (selectInjections sorts before slicing), so `slugsToRender` doubles as
-  // the render order. Per-turn: only the new slugs render (prior turns'
-  // attachments stay cached on prior user messages). Context-load: full
-  // top-K renders so the fresh user message gets a complete activation dump.
-  // Skills are appended after concept-page sections.
-  const block = await renderInjectionBlock(
-    workspaceDir,
-    slugsToRender,
-    topSkillIds,
-  );
-
   return { block, toInject: newlyInjected };
 }
 
@@ -351,14 +357,38 @@ export async function injectMemoryV2Block(
 // Internal helpers
 // ---------------------------------------------------------------------------
 
+interface RenderInjectionBlockResult {
+  /**
+   * Inner content for the `<memory>` block (concept-page sections + optional
+   * skills suffix), or `null` when both the concept-page list and the skill
+   * list collapse to empty after cache misses (no on-disk pages, no
+   * resolvable skill ids). Returned unwrapped so the caller can wrap it
+   * exactly once at injection time, matching v1's contract: callers that
+   * cache the value (`lastInjectedBlock`) or persist it (`memoryInjectedBlock`
+   * in message metadata) re-wrap on use, and storing the wrapped form here
+   * caused a double wrap on reinject after compaction and on rehydrate from
+   * DB.
+   */
+  block: string | null;
+  /**
+   * Slugs that `readPage` returned null for. Surfaced so the caller can
+   * mark them in the activation log (`status: "page_missing"`) and emit
+   * a warning — silent drops here previously masked stale Qdrant /
+   * edge-index entries that pointed at pages no longer on disk.
+   */
+  missingSlugs: string[];
+}
+
 /**
- * Render the `<memory>` block for a list of slugs and a list of
- * ranked skill ids.
+ * Render the inner content of the `<memory>` block for a list of slugs and
+ * a list of ranked skill ids. The caller wraps the result in
+ * `<memory>...</memory>` exactly once at injection time.
  *
  * Concept pages are read in parallel via `readPage`. Pages whose file has
  * gone missing between selection and render (e.g. consolidation deleted
- * them) are silently dropped — the activation state still records them in
- * `everInjected` so we don't keep re-attempting on every turn.
+ * them, folder reorg renamed the slug) are dropped from the rendered
+ * block but reported back via `missingSlugs` so callers can surface the
+ * divergence.
  *
  * Skill ids are looked up via `getSkillCapability`. Ids that the cache no
  * longer knows (e.g. uninstalled mid-run) are silently dropped, mirroring
@@ -370,7 +400,6 @@ export async function injectMemoryV2Block(
  * the agent sees the page's edges and any referenced media paths alongside
  * the prose:
  *
- *   <memory>
  *   ### <slug-1>
  *   ---
  *   edges:
@@ -390,28 +419,29 @@ export async function injectMemoryV2Block(
  *   ### Skills You Can Use
  *   - <skill-1 content>
  *   - <skill-2 content>
- *   </memory>
- *
- * Returns `null` when both lists collapse to empty after cache misses so
- * the caller can fall through to its empty-block path instead of attaching
- * an empty `<memory>` wrapper.
  */
 async function renderInjectionBlock(
   workspaceDir: string,
   slugs: string[],
   skillIds: string[],
-): Promise<string | null> {
+): Promise<RenderInjectionBlockResult> {
   const pages = await Promise.all(
     slugs.map(async (slug) => {
       const page = await readPage(workspaceDir, slug);
-      return page ? { slug, content: renderPageContent(page).trim() } : null;
+      return { slug, page };
     }),
   );
 
   const sections: string[] = [];
-  for (const entry of pages) {
-    if (!entry || entry.content.length === 0) continue;
-    sections.push(`### ${entry.slug}\n${entry.content}`);
+  const missingSlugs: string[] = [];
+  for (const { slug, page } of pages) {
+    if (!page) {
+      missingSlugs.push(slug);
+      continue;
+    }
+    const content = renderPageContent(page).trim();
+    if (content.length === 0) continue;
+    sections.push(`### ${slug}\n${content}`);
   }
 
   // v2's skills collection is skills-only, so the activation suffix always applies.
@@ -425,7 +455,10 @@ async function renderInjectionBlock(
     sections.push(`### Skills You Can Use\n${skillLines.join("\n")}`);
   }
 
-  if (sections.length === 0) return null;
+  if (sections.length === 0) return { block: null, missingSlugs };
 
-  return `<memory>\n${sections.join("\n\n")}\n</memory>`;
+  return {
+    block: sections.join("\n\n"),
+    missingSlugs,
+  };
 }