@vellumai/assistant 0.10.12 → 0.11.0-staging.2

package/src/plugins/defaults/memory/{v3/substrate → substrate}/qdrant.ts

@@ -1,7 +1,10 @@
 // ---------------------------------------------------------------------------
-// Memory v2 — Qdrant collection for concept pages
+// Memory substrate — Qdrant collection for concept pages
 // ---------------------------------------------------------------------------
 //
+// Shared by the v2 injection engine and memory-v3; active whenever
+// `usesConceptPageMemory()` holds.
+//
 // Owns a dedicated Qdrant collection, keyed by concept-page slug, that holds
 // dense + sparse embeddings of every page under `memory/concepts/`. The
 // collection is separate from the v1 `memory` collection so v2 retrieval can
@@ -15,8 +18,8 @@
 //
 // Per-channel queries: `hybridQueryConceptPages` runs separate dense and
 // sparse queries (no Qdrant-side RRF). Callers do their own weighted-sum
-// fusion using `dense_weight` / `sparse_weight` from `config.memory.v2`,
-// which RRF fusion would discard.
+// fusion using the `dense_weight` / `sparse_weight` substrate tunables
+// (`resolveSubstrateTuning`), which RRF fusion would discard.
 
 import { existsSync } from "node:fs";
 import { mkdir, unlink, writeFile } from "node:fs/promises";
@@ -25,15 +28,16 @@ import { dirname, join } from "node:path";
 import { QdrantClient as QdrantRestClient } from "@qdrant/js-client-rest";
 import { v5 as uuidv5 } from "uuid";
 
-import { getConfig } from "../../../../../config/loader.js";
-import type { SparseEmbedding } from "../../../../../persistence/embeddings/embedding-types.js";
-import { resolveQdrantUrl } from "../../embeddings.js";
-import { getLogger } from "../../logging.js";
-import { getDataDir } from "../../paths.js";
+import { getConfig } from "../../../../config/loader.js";
+import type { SparseEmbedding } from "../../../../persistence/embeddings/embedding-types.js";
+import { resolveQdrantUrl } from "../embeddings.js";
+import { getLogger } from "../logging.js";
+import { getDataDir } from "../paths.js";
 
 const log = getLogger("memory-v2-qdrant");
 
 /** Name of the dedicated Qdrant collection holding concept-page embeddings. */
+// FROZEN: persisted Qdrant collection name — never rename the value.
 export const MEMORY_V2_COLLECTION = "memory_v2_concept_pages";
 
 /**
@@ -106,6 +110,7 @@ const REQUIRED_SPARSE_VECTORS = ["sparse", "summary_sparse"] as const;
  * returns `migrated: true` until the lifecycle hook enqueues the reembed
  * and clears the sentinel.
  */
+// FROZEN: on-disk sentinel filename — never rename the value.
 const REEMBED_SENTINEL_FILENAME = ".memory-v2-reembed-required";
 
 function reembedSentinelPath(): string {
@@ -130,13 +135,17 @@ export async function clearReembedSentinel(): Promise<void> {
   try {
     await unlink(reembedSentinelPath());
   } catch (err) {
-    if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
+    if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
+      throw err;
+    }
   }
 }
 
 /** Lazily create a Qdrant REST client bound to the resolved URL. */
 function getClient(): QdrantRestClient {
-  if (_client) return _client;
+  if (_client) {
+    return _client;
+  }
   _client = new QdrantRestClient({
     url: resolveQdrantUrl(),
     checkCompatibility: false,
@@ -157,8 +166,12 @@ function getClient(): QdrantRestClient {
 export async function ensureConceptPageCollection(): Promise<{
   migrated: boolean;
 }> {
-  if (_collectionReady) return { migrated: false };
-  if (_collectionReadyPromise) return _collectionReadyPromise;
+  if (_collectionReady) {
+    return { migrated: false };
+  }
+  if (_collectionReadyPromise) {
+    return _collectionReadyPromise;
+  }
 
   _collectionReadyPromise = ensureConceptPageCollectionOnce().finally(() => {
     _collectionReadyPromise = null;
@@ -248,7 +261,9 @@ async function ensureConceptPageCollectionOnce(): Promise<{
     }
   } catch (err) {
     // Treat "not found"-shaped errors as "needs creation" and fall through.
-    if (!isCollectionMissing(err)) throw err;
+    if (!isCollectionMissing(err)) {
+      throw err;
+    }
   }
 
   log.info(
@@ -341,7 +356,9 @@ async function ensurePayloadIndexes(): Promise<void> {
       try {
         await client.createPayloadIndex(MEMORY_V2_COLLECTION, index);
       } catch (err) {
-        if (isPayloadIndexAlreadyExists(err)) return;
+        if (isPayloadIndexAlreadyExists(err)) {
+          return;
+        }
         throw err;
       }
     }),
@@ -373,7 +390,9 @@ async function reconcileSparseIndexOnDisk(
         }
       | undefined
   )?.sparse_vectors;
-  if (!sparseParams) return;
+  if (!sparseParams) {
+    return;
+  }
 
   // Only touch channels that exist in the collection and whose placement
   // drifts from the target — an update naming an absent sparse vector fails.
@@ -382,7 +401,9 @@ async function reconcileSparseIndexOnDisk(
       name in sparseParams &&
       (sparseParams[name]?.index?.on_disk ?? false) !== onDisk,
   );
-  if (drifted.length === 0) return;
+  if (drifted.length === 0) {
+    return;
+  }
 
   try {
     await getClient().updateCollection(MEMORY_V2_COLLECTION, {
@@ -441,10 +462,14 @@ function missingNamedVectors(
 
   const missing: string[] = [];
   for (const name of REQUIRED_DENSE_VECTORS) {
-    if (!denseNames.has(name)) missing.push(name);
+    if (!denseNames.has(name)) {
+      missing.push(name);
+    }
   }
   for (const name of REQUIRED_SPARSE_VECTORS) {
-    if (!sparseNames.has(name)) missing.push(name);
+    if (!sparseNames.has(name)) {
+      missing.push(name);
+    }
   }
   return missing;
 }
@@ -470,14 +495,18 @@ function wrongSizeNamedVectors(
   // Named-vectors map only — mirror the `!("size" in dense)` guard in
   // `missingNamedVectors`. A single unnamed vector exposes `size` at the top
   // level; its dimension is the collection's and is handled by the caller.
-  if (!dense || typeof dense !== "object" || "size" in dense) return [];
+  if (!dense || typeof dense !== "object" || "size" in dense) {
+    return [];
+  }
   const map = dense as Record<string, unknown>;
   const wrong: string[] = [];
   for (const name of REQUIRED_DENSE_VECTORS) {
     const entry = map[name];
     if (entry && typeof entry === "object" && "size" in entry) {
       const size = (entry as { size?: unknown }).size;
-      if (typeof size === "number" && size !== expectedSize) wrong.push(name);
+      if (typeof size === "number" && size !== expectedSize) {
+        wrong.push(name);
+      }
     }
   }
   return wrong;
@@ -525,7 +554,9 @@ export async function upsertConceptPageEmbedding(params: {
   }
 
   const payload: Record<string, unknown> = { slug, updated_at: updatedAt };
-  if (kind !== undefined) payload.kind = kind;
+  if (kind !== undefined) {
+    payload.kind = kind;
+  }
 
   const upsertOnce = () =>
     client.upsert(MEMORY_V2_COLLECTION, {
@@ -624,21 +655,30 @@ export async function pruneSlugsWithPrefixExcept(
           kind?: unknown;
         } | null;
         const slug = payload?.slug;
-        if (typeof slug !== "string") continue;
-        if (!slug.startsWith(prefix)) continue;
-        if (requiredKind !== undefined && payload?.kind !== requiredKind)
+        if (typeof slug !== "string") {
+          continue;
+        }
+        if (!slug.startsWith(prefix)) {
+          continue;
+        }
+        if (requiredKind !== undefined && payload?.kind !== requiredKind) {
           continue;
+        }
         const suffix = slug.slice(prefix.length);
         if (!activeSet.has(suffix)) {
           stalePointIds.push(point.id);
         }
       }
       const next = result.next_page_offset;
-      if (next == null) break;
+      if (next == null) {
+        break;
+      }
       offset = typeof next === "string" ? next : (next as number);
     }
 
-    if (stalePointIds.length === 0) return;
+    if (stalePointIds.length === 0) {
+      return;
+    }
 
     await client.delete(MEMORY_V2_COLLECTION, {
       wait: true,
@@ -683,7 +723,9 @@ export async function backfillKindOnPointsWithPrefix(
   kind: string,
   allowedSuffixes: ReadonlySet<string>,
 ): Promise<number> {
-  if (allowedSuffixes.size === 0) return 0;
+  if (allowedSuffixes.size === 0) {
+    return 0;
+  }
   await ensureConceptPageCollection();
 
   const client = getClient();
@@ -703,18 +745,28 @@ export async function backfillKindOnPointsWithPrefix(
       });
       for (const point of result.points) {
         const slug = (point.payload as { slug?: unknown } | null)?.slug;
-        if (typeof slug !== "string") continue;
-        if (!slug.startsWith(prefix)) continue;
+        if (typeof slug !== "string") {
+          continue;
+        }
+        if (!slug.startsWith(prefix)) {
+          continue;
+        }
         const suffix = slug.slice(prefix.length);
-        if (!allowedSuffixes.has(suffix)) continue;
+        if (!allowedSuffixes.has(suffix)) {
+          continue;
+        }
         pointIds.push(point.id);
       }
       const next = result.next_page_offset;
-      if (next == null) break;
+      if (next == null) {
+        break;
+      }
       offset = typeof next === "string" ? next : (next as number);
     }
 
-    if (pointIds.length === 0) return 0;
+    if (pointIds.length === 0) {
+      return 0;
+    }
 
     await client.setPayload(MEMORY_V2_COLLECTION, {
       payload: { kind },
@@ -778,7 +830,9 @@ export async function recreateConceptPageCollection(): Promise<void> {
   try {
     await client.deleteCollection(MEMORY_V2_COLLECTION);
   } catch (err) {
-    if (!isCollectionMissing(err)) throw err;
+    if (!isCollectionMissing(err)) {
+      throw err;
+    }
   }
   _collectionReady = false;
   _collectionReadyPromise = null;
@@ -796,7 +850,9 @@ export async function conceptPageCollectionExists(): Promise<boolean> {
     const result = await client.collectionExists(MEMORY_V2_COLLECTION);
     return result.exists;
   } catch (err) {
-    if (isCollectionMissing(err)) return false;
+    if (isCollectionMissing(err)) {
+      return false;
+    }
     throw err;
   }
 }
@@ -813,7 +869,9 @@ export async function dropLegacySkillsCollection(): Promise<void> {
   try {
     const client = getClient();
     const exists = await client.collectionExists("memory_v2_skills");
-    if (!exists.exists) return;
+    if (!exists.exists) {
+      return;
+    }
     await client.deleteCollection("memory_v2_skills");
     log.info("Deleted legacy memory_v2_skills Qdrant collection");
   } catch (err) {
@@ -943,7 +1001,9 @@ export async function hybridQueryConceptPages(
   ): void => {
     for (const point of points ?? []) {
       const slug = (point.payload as { slug?: unknown } | null)?.slug;
-      if (typeof slug !== "string") continue;
+      if (typeof slug !== "string") {
+        continue;
+      }
       const existing = merged.get(slug) ?? { slug };
       set(existing, point.score ?? 0);
       merged.set(slug, existing);

package/src/plugins/defaults/memory/substrate/reembed-job.ts

@@ -0,0 +1,54 @@
+/**
+ * Memory substrate — `memory_v2_reembed` job handler.
+ *
+ * Shared by the v2 injection engine and memory-v3; active whenever
+ * `usesConceptPageMemory()` holds.
+ *
+ * Fans out an `embed_concept_page` job per concept-page slug. Enqueued by the
+ * consolidation job's follow-ups, boot maintenance, the embedding reconciler,
+ * and workspace migrations that need every page re-indexed.
+ */
+
+import type { AssistantConfig } from "../../../../config/types.js";
+import type { MemoryJob } from "../../../../persistence/jobs-store.js";
+import { enqueueEmbedConceptPageJob } from "../jobs/embed-concept-page.js";
+import { getLogger } from "../logging.js";
+import { getWorkspaceDir } from "../paths.js";
+import { listPages } from "./page-store.js";
+
+const log = getLogger("memory-v2-reembed");
+
+/**
+ * Job handler: enqueue an `embed_concept_page` job per concept-page slug.
+ * Each enqueue coalesces with an already-pending job for the same slug, so
+ * stacked reembed runs converge on at most one pending embed per page.
+ *
+ * Returns the number of pages fanned out (enqueue attempts, not necessarily
+ * fresh rows). Callers (and tests) use the return value to assert progress
+ * without inspecting the job table directly.
+ *
+ * Note on meta files: `essentials.md` / `threads.md` / `recent.md` /
+ * `buffer.md` are direct-injected into the system prompt every turn via
+ * `_autoinject.md`. They are NOT enqueued for embedding here — their slugs
+ * (`__essentials__` etc.) contain underscores that the concept-page slug
+ * validator rejects (`[a-z0-9][a-z0-9-]*`), and they live at `memory/<name>.md`
+ * rather than `memory/concepts/<name>.md`, so path resolution would also miss.
+ * Embedding them would be redundant with the direct injection regardless.
+ */
+export async function memoryV2ReembedJob(
+  _job: MemoryJob,
+  _config: AssistantConfig,
+): Promise<number> {
+  const workspaceDir = getWorkspaceDir();
+  const slugs = await listPages(workspaceDir);
+
+  for (const slug of slugs) {
+    enqueueEmbedConceptPageJob({ slug });
+  }
+
+  log.info(
+    { conceptPages: slugs.length, total: slugs.length },
+    "Memory v2 reembed enqueued",
+  );
+  return slugs.length;
+}

package/src/plugins/defaults/memory/{v3/substrate → substrate}/sim.ts

@@ -1,7 +1,10 @@
 // ---------------------------------------------------------------------------
-// Memory v2 — Hybrid dense + sparse similarity
+// Memory substrate — Hybrid dense + sparse similarity
 // ---------------------------------------------------------------------------
 //
+// Shared by the v2 injection engine and memory-v3; active whenever
+// `usesConceptPageMemory()` holds.
+//
 // Computes per-page similarity scores against a query text by fusing dense
 // (cosine) and sparse (BM25-style) channels via a normalized weighted sum.
 // This is the building block the per-turn activation formula (`A_o`) uses to
@@ -32,13 +35,14 @@
 //   the score is consumed only as a per-turn ordering signal, not compared
 //   across turns.
 
-import type { AssistantConfig } from "../../../../../config/types.js";
-import { isEmbeddingDimensionAvailable } from "../../../../../persistence/embeddings/embedding-backend.js";
-import { applyCorrectionIfCalibrated } from "../../anisotropy.js";
-import { embedWithBackend } from "../../embeddings.js";
-import { clampUnitInterval } from "../../validation.js";
+import type { AssistantConfig } from "../../../../config/types.js";
+import { isEmbeddingDimensionAvailable } from "../../../../persistence/embeddings/embedding-backend.js";
+import { applyCorrectionIfCalibrated } from "../anisotropy.js";
+import { embedWithBackend } from "../embeddings.js";
+import { clampUnitInterval } from "../validation.js";
 import { hybridQueryConceptPages } from "./qdrant.js";
 import { generateBm25QueryEmbedding } from "./sparse-bm25.js";
+import { resolveSubstrateTuning } from "./tuning.js";
 
 /**
  * Clamp a value into the closed unit interval [0, 1]. Re-exported under the
@@ -89,10 +93,16 @@ export function effectiveWeights(
   let max = -Infinity;
   let count = 0;
   for (const h of hits) {
-    if (h.sparseScore === undefined) continue;
+    if (h.sparseScore === undefined) {
+      continue;
+    }
     const norm = h.sparseScore / maxSparse;
-    if (norm < min) min = norm;
-    if (norm > max) max = norm;
+    if (norm < min) {
+      min = norm;
+    }
+    if (norm > max) {
+      max = norm;
+    }
     count++;
   }
   // With < 2 sparse-bearing hits the spread is undefined — fall back to base
@@ -103,10 +113,9 @@ export function effectiveWeights(
   }
   const spread = max - min;
 
-  const minSpread =
-    config.memory.v2.min_sparse_spread ?? ADAPTIVE_SPARSE_MIN_SPREAD;
-  const fullSpread =
-    config.memory.v2.full_sparse_spread ?? ADAPTIVE_SPARSE_FULL_SPREAD;
+  const tuning = resolveSubstrateTuning(config.memory);
+  const minSpread = tuning.min_sparse_spread ?? ADAPTIVE_SPARSE_MIN_SPREAD;
+  const fullSpread = tuning.full_sparse_spread ?? ADAPTIVE_SPARSE_FULL_SPREAD;
   // Degenerate config (full <= min): no interpolation range. Don't try to
   // adapt; trust the operator's base weights and report the measured spread
   // for diagnostics.
@@ -212,7 +221,7 @@ export async function simBatch(
   const maxBodySparse = computeMaxSparse(hits, (h) => h.sparseScore);
   const maxSummarySparse = computeMaxSparse(hits, (h) => h.summarySparseScore);
   const { dense_weight: baseDense, sparse_weight: baseSparse } =
-    config.memory.v2;
+    resolveSubstrateTuning(config.memory);
   const { dense: bodyDenseWeight, sparse: bodySparseWeight } = effectiveWeights(
     hits.map((h) => ({ sparseScore: h.sparseScore })),
     maxBodySparse,
@@ -300,7 +309,9 @@ export function fuseHalf(
   denseWeight: number,
   sparseWeight: number,
 ): number | undefined {
-  if (denseScore === undefined && sparseScore === undefined) return undefined;
+  if (denseScore === undefined && sparseScore === undefined) {
+    return undefined;
+  }
   const dense = denseScore !== undefined ? Math.max(0, denseScore) : 0;
   const sparseNormalized =
     sparseScore !== undefined && maxSparse > 0 ? sparseScore / maxSparse : 0;

package/src/plugins/defaults/memory/{v3/substrate → substrate}/skill-content.ts

@@ -1,4 +1,4 @@
-import { getConfig } from "../../../../../config/loader.js";
+import { getConfig } from "../../../../config/loader.js";
 
 /**
  * Generic input for building capability statements: the capability fields of

package/src/plugins/defaults/memory/{v3/substrate → substrate}/skill-store.ts

@@ -1,7 +1,10 @@
 // ---------------------------------------------------------------------------
-// Memory v2 — Skill catalog → embedded skill entries
+// Memory substrate — Skill catalog → embedded skill entries
 // ---------------------------------------------------------------------------
 //
+// Shared by the v2 injection engine and memory-v3; active whenever
+// `usesConceptPageMemory()` holds.
+//
 // Enumerate the enabled-skill catalog AND uninstalled catalog skills, render
 // each skill's prose statement via `buildSkillContent`, embed dense + sparse,
 // and upsert into `memory_v2_concept_pages` under the slug `skills/<id>`.
@@ -24,11 +27,11 @@
 
 import { listCatalogSkills, listInstalledSkills } from "@vellumai/plugin-api";
 
-import { getConfig } from "../../../../../config/loader.js";
-import { generateSparseEmbedding } from "../../../../../persistence/embeddings/embedding-backend.js";
-import { applyCorrectionIfCalibrated } from "../../anisotropy.js";
-import { embedWithBackend } from "../../embeddings.js";
-import { getLogger } from "../../logging.js";
+import { getConfig } from "../../../../config/loader.js";
+import { generateSparseEmbedding } from "../../../../persistence/embeddings/embedding-backend.js";
+import { applyCorrectionIfCalibrated } from "../anisotropy.js";
+import { embedWithBackend } from "../embeddings.js";
+import { getLogger } from "../logging.js";
 import { invalidatePageIndex } from "./page-index.js";
 import {
   backfillKindOnPointsWithPrefix,
@@ -44,6 +47,7 @@ import {
   generateBm25DocEmbedding,
   getConceptPageCorpusStats,
 } from "./sparse-bm25.js";
+import { resolveSubstrateTuning } from "./tuning.js";
 import type { SkillEntry } from "./types.js";
 
 const log = getLogger("memory-v2-skill-store");
@@ -256,11 +260,12 @@ async function runSeedV2SkillEntries(generation: number): Promise<void> {
       // entirely. Fall back to the legacy TF encoder only during the cold-start
       // window before corpus stats finish building.
       const corpusStats = getConceptPageCorpusStats();
+      const { bm25_k1, bm25_b } = resolveSubstrateTuning(config.memory);
       encodeSparse = (input: string) =>
         corpusStats
           ? generateBm25DocEmbedding(input, corpusStats, {
-              k1: config.memory.v2.bm25_k1,
-              b: config.memory.v2.bm25_b,
+              k1: bm25_k1,
+              b: bm25_b,
             })
           : generateSparseEmbedding(input);
       try {

package/src/plugins/defaults/memory/{v3/substrate → substrate}/sparse-bm25.ts

@@ -1,7 +1,10 @@
 // ---------------------------------------------------------------------------
-// Memory v2 — BM25 sparse channel
+// Memory substrate — BM25 sparse channel
 // ---------------------------------------------------------------------------
 //
+// Shared by the v2 injection engine and memory-v3; active whenever
+// `usesConceptPageMemory()` holds.
+//
 // Replaces the legacy TF-only sparse embedding (`generateSparseEmbedding` in
 // `../embedding-backend.ts`) with a real Okapi BM25 implementation. Common
 // words like "i", "am", "the" no longer dominate sparse matching the way they
@@ -26,12 +29,12 @@
 
 import { readFile } from "node:fs/promises";
 
-import type { SparseEmbedding } from "../../../../../persistence/embeddings/embedding-types.js";
+import type { SparseEmbedding } from "../../../../persistence/embeddings/embedding-types.js";
 import {
   SPARSE_VOCAB_SIZE,
   tokenHash,
   tokenizeStemmed,
-} from "../../../../../persistence/embeddings/sparse-tokenize.js";
+} from "../../../../persistence/embeddings/sparse-tokenize.js";
 import { listPages } from "./page-store.js";
 
 /**
@@ -98,15 +101,21 @@ export async function rebuildConceptPageCorpusStats(
 
   for (const slug of slugs) {
     const body = await readPageBodyForStats(workspaceDir, slug);
-    if (body === null) continue;
+    if (body === null) {
+      continue;
+    }
     const tokens = tokenizeStemmed(body);
-    if (tokens.length === 0) continue;
+    if (tokens.length === 0) {
+      continue;
+    }
     totalTokens += tokens.length;
     docsCounted += 1;
     const seen = new Set<number>();
     for (const token of tokens) {
       const idx = tokenHash(token, SPARSE_VOCAB_SIZE);
-      if (seen.has(idx)) continue;
+      if (seen.has(idx)) {
+        continue;
+      }
       seen.add(idx);
       df.set(idx, (df.get(idx) ?? 0) + 1);
     }
@@ -142,7 +151,9 @@ async function readPageBodyForStats(
     const closing = raw.indexOf("\n---", 3);
     if (closing !== -1) {
       const after = raw.indexOf("\n", closing + 4);
-      if (after !== -1) return raw.slice(after + 1);
+      if (after !== -1) {
+        return raw.slice(after + 1);
+      }
     }
   }
   return raw;
@@ -193,11 +204,15 @@ export function generateBm25DocEmbedding(
 
   for (const [idx, freq] of tf) {
     const idf = computeIdf(stats, idx);
-    if (idf === 0) continue; // Skip tokens that contribute nothing to scores.
+    if (idf === 0) {
+      continue;
+    } // Skip tokens that contribute nothing to scores.
     const saturated =
       (freq * (params.k1 + 1)) / (freq + params.k1 * lengthFactor);
     const weight = idf * saturated;
-    if (weight === 0) continue;
+    if (weight === 0) {
+      continue;
+    }
     indices.push(idx);
     values.push(weight);
   }
@@ -225,7 +240,9 @@ export function generateBm25QueryEmbedding(text: string): SparseEmbedding {
   const values: number[] = [];
   for (const token of tokens) {
     const idx = tokenHash(token, SPARSE_VOCAB_SIZE);
-    if (seen.has(idx)) continue;
+    if (seen.has(idx)) {
+      continue;
+    }
     seen.add(idx);
     indices.push(idx);
     values.push(1);