@vellumai/assistant 0.7.1 → 0.7.2

package/src/memory/v2/qdrant.ts

@@ -236,6 +236,7 @@ export async function hybridQueryConceptPages(
   sparse: SparseEmbedding,
   limit: number,
   restrictToSlugs?: readonly string[],
+  options?: { skipSparse?: boolean },
 ): Promise<ConceptPageQueryResult[]> {
   if (restrictToSlugs && restrictToSlugs.length === 0) {
     // An empty restriction means "no candidates"; skip the round-trip.
@@ -249,6 +250,13 @@ export async function hybridQueryConceptPages(
     ? { must: [{ key: "slug", match: { any: [...restrictToSlugs] } }] }
     : undefined;
 
+  // When the caller weighted sparse to zero, skip the round-trip entirely.
+  // The downstream fuser (`fuseHit` in `sim.ts`) already treats a missing
+  // sparse score as a 0 contribution, so omitting the query is a pure
+  // optimization — and it's also the kill switch operators use to dodge a
+  // Qdrant 1.13.x sparse-index crash that we've reproduced in the wild.
+  const skipSparse = options?.skipSparse ?? false;
+
   const denseQuery = () =>
     client.query(MEMORY_V2_COLLECTION, {
       query: dense,
@@ -267,7 +275,14 @@ export async function hybridQueryConceptPages(
     });
 
   // Run both queries concurrently — they hit independent named vectors.
-  const runQueries = async () => Promise.all([denseQuery(), sparseQuery()]);
+  // When sparse is gated off we still resolve a Promise so the destructuring
+  // below stays uniform; the empty `points: []` matches the shape of a
+  // no-hit Qdrant response.
+  const emptyResult = {
+    points: [] as Array<{ payload?: unknown; score?: number }>,
+  };
+  const runQueries = async () =>
+    Promise.all([denseQuery(), skipSparse ? emptyResult : sparseQuery()]);
 
   let denseResults;
   let sparseResults;
@@ -305,6 +320,89 @@ export async function hybridQueryConceptPages(
   return Array.from(merged.values());
 }
 
+/**
+ * Page through the v2 concept-page collection and return up to `maxSamples`
+ * stored dense vectors. Used by the anisotropy-fit pipeline to compute a
+ * corpus mean + top-k principal components without re-embedding every page.
+ *
+ * Sparse vectors are skipped — anisotropy is a dense-embedding phenomenon, and
+ * pulling the sparse side would just inflate the response. Payload is also
+ * skipped because the fit doesn't need slug identity.
+ *
+ * Returns an empty array when the collection is empty or missing. Caller
+ * decides what to do (typically: surface a "no vectors to fit" error).
+ */
+export async function sampleConceptPageDenseVectors(
+  maxSamples: number,
+): Promise<number[][]> {
+  if (maxSamples <= 0) return [];
+  await ensureConceptPageCollection();
+
+  const client = getClient();
+  const out: number[][] = [];
+  let offset: string | number | undefined = undefined;
+  // Same pagination guard pattern as the rest of the file — bounds the loop
+  // even if Qdrant somehow keeps handing back a non-null offset.
+  const maxIterations = 10_000;
+  const batchSize = Math.min(256, maxSamples);
+
+  for (let i = 0; i < maxIterations; i++) {
+    if (out.length >= maxSamples) break;
+    const remaining = maxSamples - out.length;
+    let result;
+    try {
+      result = await client.scroll(MEMORY_V2_COLLECTION, {
+        limit: Math.min(batchSize, remaining),
+        with_payload: false,
+        // Fetch only the dense named vector — sparse is irrelevant for
+        // anisotropy correction.
+        with_vector: ["dense"],
+        ...(offset !== undefined ? { offset } : {}),
+      });
+    } catch (err) {
+      if (isCollectionMissing(err)) {
+        _collectionReady = false;
+        return out;
+      }
+      throw err;
+    }
+
+    for (const point of result.points) {
+      const v = extractDenseVector(point.vector);
+      if (v) out.push(v);
+      if (out.length >= maxSamples) break;
+    }
+
+    const next = result.next_page_offset;
+    if (next == null) break;
+    offset = typeof next === "string" ? next : (next as number);
+  }
+
+  return out;
+}
+
+/**
+ * Pull the `dense` named-vector payload out of a Qdrant point. Defensively
+ * handles both the named-vector shape (`{ dense: [...] }`) and the legacy
+ * unnamed-vector shape (`number[]`) so older collection layouts don't trip
+ * the sampler. Returns `null` for shapes we don't recognise.
+ */
+function extractDenseVector(vector: unknown): number[] | null {
+  if (Array.isArray(vector)) {
+    if (vector.every((n) => typeof n === "number")) {
+      return vector as number[];
+    }
+    return null;
+  }
+  if (vector && typeof vector === "object") {
+    const dense = (vector as { dense?: unknown }).dense;
+    if (Array.isArray(dense) && dense.every((n) => typeof n === "number")) {
+      return dense as number[];
+    }
+  }
+  return null;
+}
+
 /**
  * Detect "collection not found" errors so callers can reset readiness and
  * retry after an external deletion (e.g. workspace reset).

package/src/memory/v2/sim.ts

@@ -26,13 +26,12 @@
 //   only as a per-turn ordering signal, not compared across turns.
 
 import type { AssistantConfig } from "../../config/types.js";
-import {
-  embedWithBackend,
-  generateSparseEmbedding,
-} from "../embedding-backend.js";
+import { applyCorrectionIfCalibrated } from "../anisotropy.js";
+import { embedWithBackend } from "../embedding-backend.js";
 import { clampUnitInterval } from "../validation.js";
 import { hybridQueryConceptPages } from "./qdrant.js";
 import { hybridQuerySkills } from "./skill-qdrant.js";
+import { generateBm25QueryEmbedding } from "./sparse-bm25.js";
 
 /**
  * Clamp a value into the closed unit interval [0, 1]. Re-exported under the
@@ -40,6 +39,79 @@ import { hybridQuerySkills } from "./skill-qdrant.js";
  */
 export const clamp01 = clampUnitInterval;
 
+/**
+ * Built-in defaults for adaptive sparse weighting. Live here (not in the
+ * config schema) so operators don't see two new knobs in their config until
+ * they actually want to tune them.
+ *
+ * Below `MIN_SPREAD`, the sparse channel is treated as no-signal (its scores
+ * are uniform across the candidate set, so it can't rank anything) and the
+ * sparse weight collapses to 0. At or above `FULL_SPREAD`, sparse weight
+ * stays at its configured value. Linear interpolation between.
+ */
+const ADAPTIVE_SPARSE_MIN_SPREAD = 0.2;
+const ADAPTIVE_SPARSE_FULL_SPREAD = 0.5;
+
+/**
+ * Per-query effective dense + sparse weights, derived from the configured
+ * base weights and the spread of normalized sparse scores across the hit
+ * set. When the sparse channel can't discriminate (low spread or fewer
+ * than two sparse-bearing candidates), its weight collapses and dense
+ * weight is boosted to compensate so `dense + sparse` still equals
+ * `baseDense + baseSparse` and `fused` stays interpretable as a [0, 1]
+ * similarity.
+ *
+ * Pure function — exported so the diagnostic surface in
+ * `memory-v2-routes.explain-similarity` can show the effective weights and
+ * the measured spread alongside per-channel score statistics.
+ */
+export function effectiveWeights(
+  hits: ReadonlyArray<{ sparseScore?: number }>,
+  maxSparse: number,
+  baseDense: number,
+  baseSparse: number,
+  config: AssistantConfig,
+): { dense: number; sparse: number; spread: number } {
+  // Short-circuit when the channel is already disabled or unscored. Returning
+  // base weights here keeps `fused` numerically identical to today's output
+  // for the no-sparse-signal cases the existing tests assume.
+  if (baseSparse === 0 || maxSparse === 0) {
+    return { dense: baseDense, sparse: baseSparse, spread: 0 };
+  }
+  let min = Infinity;
+  let max = -Infinity;
+  let count = 0;
+  for (const h of hits) {
+    if (h.sparseScore === undefined) continue;
+    const norm = h.sparseScore / maxSparse;
+    if (norm < min) min = norm;
+    if (norm > max) max = norm;
+    count++;
+  }
+  // With < 2 sparse-bearing hits the spread is undefined — fall back to base
+  // weights so single-hit retrievals still surface their sparse contribution
+  // (and the existing fusion-math tests stay green).
+  if (count < 2) {
+    return { dense: baseDense, sparse: baseSparse, spread: 0 };
+  }
+  const spread = max - min;
+
+  const minSpread =
+    config.memory.v2.min_sparse_spread ?? ADAPTIVE_SPARSE_MIN_SPREAD;
+  const fullSpread =
+    config.memory.v2.full_sparse_spread ?? ADAPTIVE_SPARSE_FULL_SPREAD;
+  // Degenerate config (full <= min): no interpolation range. Don't try to
+  // adapt; trust the operator's base weights and report the measured spread
+  // for diagnostics.
+  if (fullSpread <= minSpread) {
+    return { dense: baseDense, sparse: baseSparse, spread };
+  }
+  const factor = clamp01((spread - minSpread) / (fullSpread - minSpread));
+  const sparse = baseSparse * factor;
+  const dense = baseDense + (baseSparse - sparse);
+  return { dense, sparse, spread };
+}
+
 /**
  * Compute hybrid (dense + sparse) similarity scores between a query text and
  * a fixed set of candidate concept-page slugs.
@@ -63,8 +135,13 @@ export const clamp01 = clampUnitInterval;
  * Edge cases:
  *   - Empty `candidateSlugs` → returns an empty map without touching Qdrant
  *     or the embedding backend.
- *   - Empty query text or all-zero sparse vector → still queries (dense may
- *     still hit), and the sparse contribution to fusion is zero.
+ *   - Empty / whitespace-only `text` → returns an empty map without touching
+ *     Qdrant or the embedding backend. The Gemini embedding API rejects empty
+ *     content with HTTP 400, and short-circuiting here prevents the failure
+ *     from cascading through `Promise.all` in `computeOwnActivation` (e.g.
+ *     turn 1 has no prior assistant message, so its `simBatch` channel is
+ *     called with `""`). Treating the channel's contribution as 0 is the
+ *     same outcome a no-hit query would produce.
  */
 export async function simBatch(
   text: string,
@@ -74,12 +151,20 @@ export async function simBatch(
   if (candidateSlugs.length === 0) {
     return new Map();
   }
+  if (text.trim().length === 0) {
+    return new Map();
+  }
 
-  // Sparse uses the shared TF-IDF encoder so the query and stored vectors
-  // share a vocabulary with PKB indexing.
+  // Sparse uses BM25: the query side encodes binary occurrences per token,
+  // and the stored doc vectors carry the IDF · TF-saturated weights — Qdrant
+  // dot product then yields the BM25 score directly.
   const denseResult = await embedWithBackend(config, [text]);
-  const denseVector = denseResult.vectors[0];
-  const sparseVector = generateSparseEmbedding(text);
+  const denseVector = await applyCorrectionIfCalibrated(
+    denseResult.vectors[0],
+    denseResult.provider,
+    denseResult.model,
+  );
+  const sparseVector = generateBm25QueryEmbedding(text);
 
   const hits = await hybridQueryConceptPages(
     denseVector,
@@ -93,8 +178,15 @@ export async function simBatch(
   }
 
   const maxSparse = computeMaxSparse(hits);
-  const { dense_weight: denseWeight, sparse_weight: sparseWeight } =
+  const { dense_weight: baseDense, sparse_weight: baseSparse } =
     config.memory.v2;
+  const { dense: denseWeight, sparse: sparseWeight } = effectiveWeights(
+    hits,
+    maxSparse,
+    baseDense,
+    baseSparse,
+    config,
+  );
 
   const scores = new Map<string, number>();
   for (const hit of hits) {
@@ -122,8 +214,12 @@ export async function simBatch(
  * Edge cases:
  *   - Empty `ids` → returns an empty map without touching Qdrant or the
  *     embedding backend.
- *   - Empty query text → still queries (dense may still hit), and the sparse
- *     contribution is zero.
+ *   - Empty / whitespace-only `text` → returns an empty map without touching
+ *     Qdrant or the embedding backend. Same rationale as {@link simBatch}:
+ *     Gemini rejects empty content with HTTP 400, so the activation pipeline
+ *     would otherwise fail on turn 1 (where the assistant-text channel is
+ *     `""`). Treating the channel's contribution as 0 matches a no-hit
+ *     query.
  */
 export async function simSkillBatch(
   text: string,
@@ -133,10 +229,17 @@ export async function simSkillBatch(
   if (ids.length === 0) {
     return new Map();
   }
+  if (text.trim().length === 0) {
+    return new Map();
+  }
 
   const denseResult = await embedWithBackend(config, [text]);
-  const denseVector = denseResult.vectors[0];
-  const sparseVector = generateSparseEmbedding(text);
+  const denseVector = await applyCorrectionIfCalibrated(
+    denseResult.vectors[0],
+    denseResult.provider,
+    denseResult.model,
+  );
+  const sparseVector = generateBm25QueryEmbedding(text);
 
   const hits = await hybridQuerySkills(
     denseVector,
@@ -160,8 +263,15 @@ export async function simSkillBatch(
   }
 
   const maxSparse = computeMaxSparse(filtered);
-  const { dense_weight: denseWeight, sparse_weight: sparseWeight } =
+  const { dense_weight: baseDense, sparse_weight: baseSparse } =
     config.memory.v2;
+  const { dense: denseWeight, sparse: sparseWeight } = effectiveWeights(
+    filtered,
+    maxSparse,
+    baseDense,
+    baseSparse,
+    config,
+  );
 
   const scores = new Map<string, number>();
   for (const hit of filtered) {

package/src/memory/v2/skill-qdrant.ts

@@ -278,14 +278,14 @@ export async function pruneSkillsExcept(
  * candidate set is already known so the activation scorer gets scores for
  * exactly those ids rather than Qdrant's global top-`limit`. An empty list
  * short-circuits to no results — the caller is asking for "nothing", not
- * "everything". Undefined queries the full collection (used by
- * `selectSkillCandidates` to discover candidates from the global top-K).
+ * "everything". Undefined queries the full collection.
  */
 export async function hybridQuerySkills(
   dense: number[],
   sparse: SparseEmbedding,
   limit: number,
   restrictToIds?: readonly string[],
+  options?: { skipSparse?: boolean },
 ): Promise<SkillQueryResult[]> {
   if (restrictToIds && restrictToIds.length === 0) {
     // An empty restriction means "no candidates"; skip the round-trip.
@@ -299,6 +299,11 @@ export async function hybridQuerySkills(
     ? { must: [{ key: "id", match: { any: [...restrictToIds] } }] }
     : undefined;
 
+  // Same opt-in short-circuit as `hybridQueryConceptPages`: skip the sparse
+  // round-trip entirely so we sidestep the Qdrant 1.13.x sparse-index OOM
+  // crash when operators flip sparse off via `sparse_weight: 0`.
+  const skipSparse = options?.skipSparse ?? false;
+
   const denseQuery = () =>
     client.query(MEMORY_V2_SKILLS_COLLECTION, {
       query: dense,
@@ -317,7 +322,11 @@ export async function hybridQuerySkills(
     });
 
   // Run both queries concurrently — they hit independent named vectors.
-  const runQueries = async () => Promise.all([denseQuery(), sparseQuery()]);
+  const emptyResult = {
+    points: [] as Array<{ payload?: unknown; score?: number }>,
+  };
+  const runQueries = async () =>
+    Promise.all([denseQuery(), skipSparse ? emptyResult : sparseQuery()]);
 
   let denseResults;
   let sparseResults;

package/src/memory/v2/skill-store.ts

@@ -25,6 +25,7 @@ import {
   fromSkillSummary,
 } from "../../skills/skill-memory.js";
 import { getLogger } from "../../util/logger.js";
+import { applyCorrectionIfCalibrated } from "../anisotropy.js";
 import {
   embedWithBackend,
   generateSparseEmbedding,
@@ -122,10 +123,15 @@ export async function seedV2SkillEntries(): Promise<void> {
 
     // Embed all content strings in one batched call. Sparse vectors are
     // computed in-process (no network).
-    const { vectors: denseVectors } = await embedWithBackend(
+    const embedded = await embedWithBackend(
       config,
       seeds.map((s) => s.content),
     );
+    const denseVectors = await Promise.all(
+      embedded.vectors.map((v) =>
+        applyCorrectionIfCalibrated(v, embedded.provider, embedded.model),
+      ),
+    );
 
     const now = Date.now();
     const nextEntries = new Map<string, SkillEntry>();
@@ -170,6 +176,15 @@ export function getSkillCapability(id: string): SkillEntry | null {
   return entries?.get(id) ?? null;
 }
 
+/**
+ * Every skill id in the cache — both installed-and-enabled skills and
+ * uninstalled-catalog skills. Empty before the first `seedV2SkillEntries`
+ * run completes.
+ */
+export function getAllSkillIds(): string[] {
+  return entries ? [...entries.keys()] : [];
+}
+
 /** @internal Test-only: clear the module-level cache. */
 export function _resetSkillStoreForTests(): void {
   entries = null;

package/src/memory/v2/sparse-bm25.ts

@@ -0,0 +1,245 @@
+// ---------------------------------------------------------------------------
+// Memory v2 — BM25 sparse channel
+// ---------------------------------------------------------------------------
+//
+// Replaces the legacy TF-only sparse embedding (`generateSparseEmbedding` in
+// `../embedding-backend.ts`) with a real Okapi BM25 implementation. Common
+// words like "i", "am", "the" no longer dominate sparse matching the way they
+// did when every token was weighted equally.
+//
+// BM25 score for document `d` and query `q`:
+//
+//   score(d, q) = Σ_t∈q  IDF(t) · TF_sat(d, t)
+//   TF_sat(d, t) = tf(d, t) · (k1 + 1)
+//                / (tf(d, t) + k1 · (1 - b + b · |d| / avg_dl))
+//   IDF(t)       = log( (N - df(t) + 0.5) / (df(t) + 0.5) + 1 )
+//
+// `+1` inside the IDF log keeps the result non-negative even when df(t) > N/2,
+// matching the variant Lucene uses for `BM25Similarity`.
+//
+// **Asymmetric encoding**: documents carry the full BM25 weight per token
+// (IDF · TF_sat baked into the stored vector), and queries carry binary
+// occurrence per token. Qdrant's sparse dot product then reduces to the BM25
+// score directly. Putting BM25 on the doc side means the weights need
+// recomputing whenever the corpus DF or avg_dl changes — operators trigger
+// that with `assistant memory v2 reembed` after major content shifts.
+
+import { readFile } from "node:fs/promises";
+
+import type { SparseEmbedding } from "../embedding-types.js";
+import {
+  SPARSE_VOCAB_SIZE,
+  tokenHash,
+  tokenizeStemmed,
+} from "../sparse-tokenize.js";
+import { listPages } from "./page-store.js";
+
+/**
+ * Aggregate corpus statistics used to weight a BM25 document vector. Held in
+ * memory after a startup walk over `memory/concepts/`.
+ */
+export interface CorpusStats {
+  /** Total document count over which DF was accumulated. */
+  totalDocs: number;
+  /** hashedTokenIndex (in `[0, SPARSE_VOCAB_SIZE)`) → distinct-doc count. */
+  df: Map<number, number>;
+  /** Average document length in tokens, post-tokenize. */
+  avgDl: number;
+  /** Wall-clock millis at build time — used by diagnostics, not the formula. */
+  builtAt: number;
+}
+
+/** BM25 hyperparameters. Standard Lucene/Elasticsearch defaults. */
+export interface Bm25Params {
+  /** TF saturation curve. ~1.2 is standard. */
+  k1: number;
+  /** Length normalization. 0 = none, 1 = full. ~0.75 is standard. */
+  b: number;
+}
+
+let _conceptPageStats: CorpusStats | null = null;
+
+/**
+ * Latest in-memory corpus stats for `memory/concepts/`, or `null` if a build
+ * has not yet completed. Callers must handle `null` and fall back to legacy
+ * TF-only behavior so the daemon remains usable during the brief startup
+ * window before {@link rebuildConceptPageCorpusStats} finishes.
+ */
+export function getConceptPageCorpusStats(): CorpusStats | null {
+  return _conceptPageStats;
+}
+
+/**
+ * Walk every concept page on disk, accumulate document frequency per hashed
+ * token bucket, and average document length. Atomically swaps the result into
+ * the module-level cache when the walk succeeds. On error the previous stats
+ * stay live.
+ *
+ * Reads bodies via `readPage`-equivalent direct file reads to avoid paying for
+ * frontmatter parsing on every page (we only need the body for sparse).
+ */
+export async function rebuildConceptPageCorpusStats(
+  workspaceDir: string,
+): Promise<void> {
+  const slugs = await listPages(workspaceDir);
+  if (slugs.length === 0) {
+    _conceptPageStats = {
+      totalDocs: 0,
+      df: new Map(),
+      avgDl: 0,
+      builtAt: Date.now(),
+    };
+    return;
+  }
+
+  const df = new Map<number, number>();
+  let totalTokens = 0;
+  let docsCounted = 0;
+
+  for (const slug of slugs) {
+    const body = await readPageBodyForStats(workspaceDir, slug);
+    if (body === null) continue;
+    const tokens = tokenizeStemmed(body);
+    if (tokens.length === 0) continue;
+    totalTokens += tokens.length;
+    docsCounted += 1;
+    const seen = new Set<number>();
+    for (const token of tokens) {
+      const idx = tokenHash(token, SPARSE_VOCAB_SIZE);
+      if (seen.has(idx)) continue;
+      seen.add(idx);
+      df.set(idx, (df.get(idx) ?? 0) + 1);
+    }
+  }
+
+  _conceptPageStats = {
+    totalDocs: docsCounted,
+    df,
+    avgDl: docsCounted > 0 ? totalTokens / docsCounted : 0,
+    builtAt: Date.now(),
+  };
+}
+
+/**
+ * Read just the body of a page for stats accumulation. Skips the YAML
+ * frontmatter without invoking the schema-validating `readPage` parser, since
+ * any parse failure surfaced there would abort the whole rebuild — and we
+ * only need the prose half for tokenization.
+ */
+async function readPageBodyForStats(
+  workspaceDir: string,
+  slug: string,
+): Promise<string | null> {
+  const path = `${workspaceDir}/memory/concepts/${slug}.md`;
+  let raw: string;
+  try {
+    raw = await readFile(path, "utf-8");
+  } catch {
+    return null;
+  }
+  // Strip a leading `---\n...\n---\n` block if present; otherwise return raw.
+  if (raw.startsWith("---")) {
+    const closing = raw.indexOf("\n---", 3);
+    if (closing !== -1) {
+      const after = raw.indexOf("\n", closing + 4);
+      if (after !== -1) return raw.slice(after + 1);
+    }
+  }
+  return raw;
+}
+
+/**
+ * Compute the BM25 IDF weight for a hashed token bucket. Returns `0` when the
+ * token's df equals the corpus size (a token in every document carries no
+ * discriminating power).
+ */
+function computeIdf(stats: CorpusStats, hashIdx: number): number {
+  const df = stats.df.get(hashIdx) ?? 0;
+  const numerator = stats.totalDocs - df + 0.5;
+  const denominator = df + 0.5;
+  return Math.log(numerator / denominator + 1);
+}
+
+/**
+ * Document-side BM25-weighted sparse vector. Each emitted value is
+ * `IDF(t) · TF_sat(d, t)` so the dot product against a binary query vector
+ * (see {@link generateBm25QueryEmbedding}) yields the BM25 score.
+ *
+ * Returns an empty embedding for empty input or when the corpus is empty
+ * (every IDF would be zero anyway).
+ */
+export function generateBm25DocEmbedding(
+  text: string,
+  stats: CorpusStats,
+  params: Bm25Params,
+): SparseEmbedding {
+  const tokens = tokenizeStemmed(text);
+  if (tokens.length === 0 || stats.totalDocs === 0) {
+    return { indices: [], values: [] };
+  }
+
+  // Per-document term frequencies, keyed by hashed bucket.
+  const tf = new Map<number, number>();
+  for (const token of tokens) {
+    const idx = tokenHash(token, SPARSE_VOCAB_SIZE);
+    tf.set(idx, (tf.get(idx) ?? 0) + 1);
+  }
+
+  const docLen = tokens.length;
+  // avg_dl can be 0 only when totalDocs is 0, which we already short-circuited.
+  const lengthFactor = 1 - params.b + (params.b * docLen) / stats.avgDl;
+  const indices: number[] = [];
+  const values: number[] = [];
+
+  for (const [idx, freq] of tf) {
+    const idf = computeIdf(stats, idx);
+    if (idf === 0) continue; // Skip tokens that contribute nothing to scores.
+    const saturated =
+      (freq * (params.k1 + 1)) / (freq + params.k1 * lengthFactor);
+    const weight = idf * saturated;
+    if (weight === 0) continue;
+    indices.push(idx);
+    values.push(weight);
+  }
+
+  return { indices, values };
+}
+
+/**
+ * Query-side sparse vector — binary occurrence per distinct query token. The
+ * dot product `Σ_t v_q(t) · v_d(t)` against a BM25-weighted document vector
+ * is exactly the BM25 score, since `v_q(t) = 1` for tokens in the query and
+ * `0` otherwise.
+ *
+ * Stateless — does not need corpus stats, so callers can use this on every
+ * turn without coordinating with {@link rebuildConceptPageCorpusStats}.
+ */
+export function generateBm25QueryEmbedding(text: string): SparseEmbedding {
+  const tokens = tokenizeStemmed(text);
+  if (tokens.length === 0) {
+    return { indices: [], values: [] };
+  }
+
+  const seen = new Set<number>();
+  const indices: number[] = [];
+  const values: number[] = [];
+  for (const token of tokens) {
+    const idx = tokenHash(token, SPARSE_VOCAB_SIZE);
+    if (seen.has(idx)) continue;
+    seen.add(idx);
+    indices.push(idx);
+    values.push(1);
+  }
+
+  return { indices, values };
+}
+
+/** @internal Test-only: reset module-level singletons. */
+export function _resetCorpusStatsForTests(): void {
+  _conceptPageStats = null;
+}
+
+/** @internal Test-only: install a fixture stats table without disk I/O. */
+export function _setCorpusStatsForTests(stats: CorpusStats | null): void {
+  _conceptPageStats = stats;
+}

package/src/memory/v2/static-context.ts

@@ -6,11 +6,12 @@
 // and returns a concatenated, header-wrapped block ready to splice into the
 // current user message via the injector chain.
 //
-// Pairs with the v2 per-turn activation block (`prependMemoryV2Block` in
-// `conversation-graph-memory.ts`) — that block carries activated concept
-// pages selected by the activation pipeline; this static block carries the
-// always-relevant aggregate views written by consolidation and the user.
-// Both land on the user message so the system prompt stays cache-stable.
+// Pairs with the v2 per-turn activation block (`maybeRouteV2Injection` in
+// `conversation-graph-memory.ts`, which threads through `injectTextBlock`)
+// — that block carries activated concept pages selected by the activation
+// pipeline; this static block carries the always-relevant aggregate views
+// written by consolidation and the user. Both land on the user message so
+// the system prompt stays cache-stable.
 //
 // Refresh cadence is owned by the caller: the agent loop only passes the
 // content through when `mode === "full"` (first turn / post-compaction),