@vellumai/assistant 0.7.0 → 0.7.2

package/src/memory/v2/qdrant.ts

@@ -236,6 +236,7 @@ export async function hybridQueryConceptPages(
   sparse: SparseEmbedding,
   limit: number,
   restrictToSlugs?: readonly string[],
+  options?: { skipSparse?: boolean },
 ): Promise<ConceptPageQueryResult[]> {
   if (restrictToSlugs && restrictToSlugs.length === 0) {
     // An empty restriction means "no candidates"; skip the round-trip.
@@ -249,6 +250,13 @@ export async function hybridQueryConceptPages(
     ? { must: [{ key: "slug", match: { any: [...restrictToSlugs] } }] }
     : undefined;
 
+  // When the caller weighted sparse to zero, skip the round-trip entirely.
+  // The downstream fuser (`fuseHit` in `sim.ts`) already treats a missing
+  // sparse score as a 0 contribution, so omitting the query is a pure
+  // optimization — and it's also the kill switch operators use to dodge a
+  // Qdrant 1.13.x sparse-index crash that we've reproduced in the wild.
+  const skipSparse = options?.skipSparse ?? false;
+
   const denseQuery = () =>
     client.query(MEMORY_V2_COLLECTION, {
       query: dense,
@@ -267,7 +275,14 @@ export async function hybridQueryConceptPages(
     });
 
   // Run both queries concurrently — they hit independent named vectors.
-  const runQueries = async () => Promise.all([denseQuery(), sparseQuery()]);
+  // When sparse is gated off we still resolve a Promise so the destructuring
+  // below stays uniform; the empty `points: []` matches the shape of a
+  // no-hit Qdrant response.
+  const emptyResult = {
+    points: [] as Array<{ payload?: unknown; score?: number }>,
+  };
+  const runQueries = async () =>
+    Promise.all([denseQuery(), skipSparse ? emptyResult : sparseQuery()]);
 
   let denseResults;
   let sparseResults;
@@ -305,6 +320,89 @@ export async function hybridQueryConceptPages(
   return Array.from(merged.values());
 }
 
+/**
+ * Page through the v2 concept-page collection and return up to `maxSamples`
+ * stored dense vectors. Used by the anisotropy-fit pipeline to compute a
+ * corpus mean + top-k principal components without re-embedding every page.
+ *
+ * Sparse vectors are skipped — anisotropy is a dense-embedding phenomenon, and
+ * pulling the sparse side would just inflate the response. Payload is also
+ * skipped because the fit doesn't need slug identity.
+ *
+ * Returns an empty array when the collection is empty or missing. Caller
+ * decides what to do (typically: surface a "no vectors to fit" error).
+ */
+export async function sampleConceptPageDenseVectors(
+  maxSamples: number,
+): Promise<number[][]> {
+  if (maxSamples <= 0) return [];
+  await ensureConceptPageCollection();
+
+  const client = getClient();
+  const out: number[][] = [];
+  let offset: string | number | undefined = undefined;
+  // Same pagination guard pattern as the rest of the file — bounds the loop
+  // even if Qdrant somehow keeps handing back a non-null offset.
+  const maxIterations = 10_000;
+  const batchSize = Math.min(256, maxSamples);
+
+  for (let i = 0; i < maxIterations; i++) {
+    if (out.length >= maxSamples) break;
+    const remaining = maxSamples - out.length;
+    let result;
+    try {
+      result = await client.scroll(MEMORY_V2_COLLECTION, {
+        limit: Math.min(batchSize, remaining),
+        with_payload: false,
+        // Fetch only the dense named vector — sparse is irrelevant for
+        // anisotropy correction.
+        with_vector: ["dense"],
+        ...(offset !== undefined ? { offset } : {}),
+      });
+    } catch (err) {
+      if (isCollectionMissing(err)) {
+        _collectionReady = false;
+        return out;
+      }
+      throw err;
+    }
+
+    for (const point of result.points) {
+      const v = extractDenseVector(point.vector);
+      if (v) out.push(v);
+      if (out.length >= maxSamples) break;
+    }
+
+    const next = result.next_page_offset;
+    if (next == null) break;
+    offset = typeof next === "string" ? next : (next as number);
+  }
+
+  return out;
+}
+
+/**
+ * Pull the `dense` named-vector payload out of a Qdrant point. Defensively
+ * handles both the named-vector shape (`{ dense: [...] }`) and the legacy
+ * unnamed-vector shape (`number[]`) so older collection layouts don't trip
+ * the sampler. Returns `null` for shapes we don't recognise.
+ */
+function extractDenseVector(vector: unknown): number[] | null {
+  if (Array.isArray(vector)) {
+    if (vector.every((n) => typeof n === "number")) {
+      return vector as number[];
+    }
+    return null;
+  }
+  if (vector && typeof vector === "object") {
+    const dense = (vector as { dense?: unknown }).dense;
+    if (Array.isArray(dense) && dense.every((n) => typeof n === "number")) {
+      return dense as number[];
+    }
+  }
+  return null;
+}
+
 /**
  * Detect "collection not found" errors so callers can reset readiness and
  * retry after an external deletion (e.g. workspace reset).

package/src/memory/v2/sim.ts

@@ -26,13 +26,12 @@
 //   only as a per-turn ordering signal, not compared across turns.
 
 import type { AssistantConfig } from "../../config/types.js";
-import {
-  embedWithBackend,
-  generateSparseEmbedding,
-} from "../embedding-backend.js";
+import { applyCorrectionIfCalibrated } from "../anisotropy.js";
+import { embedWithBackend } from "../embedding-backend.js";
 import { clampUnitInterval } from "../validation.js";
 import { hybridQueryConceptPages } from "./qdrant.js";
 import { hybridQuerySkills } from "./skill-qdrant.js";
+import { generateBm25QueryEmbedding } from "./sparse-bm25.js";
 
 /**
  * Clamp a value into the closed unit interval [0, 1]. Re-exported under the
@@ -40,6 +39,79 @@ import { hybridQuerySkills } from "./skill-qdrant.js";
  */
 export const clamp01 = clampUnitInterval;
 
+/**
+ * Built-in defaults for adaptive sparse weighting. Live here (not in the
+ * config schema) so operators don't see two new knobs in their config until
+ * they actually want to tune them.
+ *
+ * Below `MIN_SPREAD`, the sparse channel is treated as no-signal (its scores
+ * are uniform across the candidate set, so it can't rank anything) and the
+ * sparse weight collapses to 0. At or above `FULL_SPREAD`, sparse weight
+ * stays at its configured value. Linear interpolation between.
+ */
+const ADAPTIVE_SPARSE_MIN_SPREAD = 0.2;
+const ADAPTIVE_SPARSE_FULL_SPREAD = 0.5;
+
+/**
+ * Per-query effective dense + sparse weights, derived from the configured
+ * base weights and the spread of normalized sparse scores across the hit
+ * set. When the sparse channel can't discriminate (low spread or fewer
+ * than two sparse-bearing candidates), its weight collapses and dense
+ * weight is boosted to compensate so `dense + sparse` still equals
+ * `baseDense + baseSparse` and `fused` stays interpretable as a [0, 1]
+ * similarity.
+ *
+ * Pure function — exported so the diagnostic surface in
+ * `memory-v2-routes.explain-similarity` can show the effective weights and
+ * the measured spread alongside per-channel score statistics.
+ */
+export function effectiveWeights(
+  hits: ReadonlyArray<{ sparseScore?: number }>,
+  maxSparse: number,
+  baseDense: number,
+  baseSparse: number,
+  config: AssistantConfig,
+): { dense: number; sparse: number; spread: number } {
+  // Short-circuit when the channel is already disabled or unscored. Returning
+  // base weights here keeps `fused` numerically identical to today's output
+  // for the no-sparse-signal cases the existing tests assume.
+  if (baseSparse === 0 || maxSparse === 0) {
+    return { dense: baseDense, sparse: baseSparse, spread: 0 };
+  }
+  let min = Infinity;
+  let max = -Infinity;
+  let count = 0;
+  for (const h of hits) {
+    if (h.sparseScore === undefined) continue;
+    const norm = h.sparseScore / maxSparse;
+    if (norm < min) min = norm;
+    if (norm > max) max = norm;
+    count++;
+  }
+  // With < 2 sparse-bearing hits the spread is undefined — fall back to base
+  // weights so single-hit retrievals still surface their sparse contribution
+  // (and the existing fusion-math tests stay green).
+  if (count < 2) {
+    return { dense: baseDense, sparse: baseSparse, spread: 0 };
+  }
+  const spread = max - min;
+
+  const minSpread =
+    config.memory.v2.min_sparse_spread ?? ADAPTIVE_SPARSE_MIN_SPREAD;
+  const fullSpread =
+    config.memory.v2.full_sparse_spread ?? ADAPTIVE_SPARSE_FULL_SPREAD;
+  // Degenerate config (full <= min): no interpolation range. Don't try to
+  // adapt; trust the operator's base weights and report the measured spread
+  // for diagnostics.
+  if (fullSpread <= minSpread) {
+    return { dense: baseDense, sparse: baseSparse, spread };
+  }
+  const factor = clamp01((spread - minSpread) / (fullSpread - minSpread));
+  const sparse = baseSparse * factor;
+  const dense = baseDense + (baseSparse - sparse);
+  return { dense, sparse, spread };
+}
+
 /**
  * Compute hybrid (dense + sparse) similarity scores between a query text and
  * a fixed set of candidate concept-page slugs.
@@ -63,8 +135,13 @@ export const clamp01 = clampUnitInterval;
  * Edge cases:
  *   - Empty `candidateSlugs` → returns an empty map without touching Qdrant
  *     or the embedding backend.
- *   - Empty query text or all-zero sparse vector → still queries (dense may
- *     still hit), and the sparse contribution to fusion is zero.
+ *   - Empty / whitespace-only `text` → returns an empty map without touching
+ *     Qdrant or the embedding backend. The Gemini embedding API rejects empty
+ *     content with HTTP 400, and short-circuiting here prevents the failure
+ *     from cascading through `Promise.all` in `computeOwnActivation` (e.g.
+ *     turn 1 has no prior assistant message, so its `simBatch` channel is
+ *     called with `""`). Treating the channel's contribution as 0 is the
+ *     same outcome a no-hit query would produce.
  */
 export async function simBatch(
   text: string,
@@ -74,12 +151,20 @@ export async function simBatch(
   if (candidateSlugs.length === 0) {
     return new Map();
   }
+  if (text.trim().length === 0) {
+    return new Map();
+  }
 
-  // Sparse uses the shared TF-IDF encoder so the query and stored vectors
-  // share a vocabulary with PKB indexing.
+  // Sparse uses BM25: the query side encodes binary occurrences per token,
+  // and the stored doc vectors carry the IDF · TF-saturated weights — Qdrant
+  // dot product then yields the BM25 score directly.
   const denseResult = await embedWithBackend(config, [text]);
-  const denseVector = denseResult.vectors[0];
-  const sparseVector = generateSparseEmbedding(text);
+  const denseVector = await applyCorrectionIfCalibrated(
+    denseResult.vectors[0],
+    denseResult.provider,
+    denseResult.model,
+  );
+  const sparseVector = generateBm25QueryEmbedding(text);
 
   const hits = await hybridQueryConceptPages(
     denseVector,
@@ -93,8 +178,15 @@ export async function simBatch(
   }
 
   const maxSparse = computeMaxSparse(hits);
-  const { dense_weight: denseWeight, sparse_weight: sparseWeight } =
+  const { dense_weight: baseDense, sparse_weight: baseSparse } =
     config.memory.v2;
+  const { dense: denseWeight, sparse: sparseWeight } = effectiveWeights(
+    hits,
+    maxSparse,
+    baseDense,
+    baseSparse,
+    config,
+  );
 
   const scores = new Map<string, number>();
   for (const hit of hits) {
@@ -122,8 +214,12 @@ export async function simBatch(
  * Edge cases:
  *   - Empty `ids` → returns an empty map without touching Qdrant or the
  *     embedding backend.
- *   - Empty query text → still queries (dense may still hit), and the sparse
- *     contribution is zero.
+ *   - Empty / whitespace-only `text` → returns an empty map without touching
+ *     Qdrant or the embedding backend. Same rationale as {@link simBatch}:
+ *     Gemini rejects empty content with HTTP 400, so the activation pipeline
+ *     would otherwise fail on turn 1 (where the assistant-text channel is
+ *     `""`). Treating the channel's contribution as 0 matches a no-hit
+ *     query.
  */
 export async function simSkillBatch(
   text: string,
@@ -133,10 +229,17 @@ export async function simSkillBatch(
   if (ids.length === 0) {
     return new Map();
   }
+  if (text.trim().length === 0) {
+    return new Map();
+  }
 
   const denseResult = await embedWithBackend(config, [text]);
-  const denseVector = denseResult.vectors[0];
-  const sparseVector = generateSparseEmbedding(text);
+  const denseVector = await applyCorrectionIfCalibrated(
+    denseResult.vectors[0],
+    denseResult.provider,
+    denseResult.model,
+  );
+  const sparseVector = generateBm25QueryEmbedding(text);
 
   const hits = await hybridQuerySkills(
     denseVector,
@@ -160,8 +263,15 @@ export async function simSkillBatch(
   }
 
   const maxSparse = computeMaxSparse(filtered);
-  const { dense_weight: denseWeight, sparse_weight: sparseWeight } =
+  const { dense_weight: baseDense, sparse_weight: baseSparse } =
     config.memory.v2;
+  const { dense: denseWeight, sparse: sparseWeight } = effectiveWeights(
+    filtered,
+    maxSparse,
+    baseDense,
+    baseSparse,
+    config,
+  );
 
   const scores = new Map<string, number>();
   for (const hit of filtered) {

package/src/memory/v2/skill-qdrant.ts

@@ -278,14 +278,14 @@ export async function pruneSkillsExcept(
  * candidate set is already known so the activation scorer gets scores for
  * exactly those ids rather than Qdrant's global top-`limit`. An empty list
  * short-circuits to no results — the caller is asking for "nothing", not
- * "everything". Undefined queries the full collection (used by
- * `selectSkillCandidates` to discover candidates from the global top-K).
+ * "everything". Undefined queries the full collection.
  */
 export async function hybridQuerySkills(
   dense: number[],
   sparse: SparseEmbedding,
   limit: number,
   restrictToIds?: readonly string[],
+  options?: { skipSparse?: boolean },
 ): Promise<SkillQueryResult[]> {
   if (restrictToIds && restrictToIds.length === 0) {
     // An empty restriction means "no candidates"; skip the round-trip.
@@ -299,6 +299,11 @@ export async function hybridQuerySkills(
     ? { must: [{ key: "id", match: { any: [...restrictToIds] } }] }
     : undefined;
 
+  // Same opt-in short-circuit as `hybridQueryConceptPages`: skip the sparse
+  // round-trip entirely so we sidestep the Qdrant 1.13.x sparse-index OOM
+  // crash when operators flip sparse off via `sparse_weight: 0`.
+  const skipSparse = options?.skipSparse ?? false;
+
   const denseQuery = () =>
     client.query(MEMORY_V2_SKILLS_COLLECTION, {
       query: dense,
@@ -317,7 +322,11 @@ export async function hybridQuerySkills(
     });
 
   // Run both queries concurrently — they hit independent named vectors.
-  const runQueries = async () => Promise.all([denseQuery(), sparseQuery()]);
+  const emptyResult = {
+    points: [] as Array<{ payload?: unknown; score?: number }>,
+  };
+  const runQueries = async () =>
+    Promise.all([denseQuery(), skipSparse ? emptyResult : sparseQuery()]);
 
   let denseResults;
   let sparseResults;

package/src/memory/v2/skill-store.ts

@@ -2,11 +2,13 @@
 // Memory v2 — Skill catalog → embedded skill entries
 // ---------------------------------------------------------------------------
 //
-// Mirrors v1's `seedSkillGraphNodes` (capability-seed.ts) for the v2 pipeline:
-// enumerate the enabled-skill catalog, render each skill's prose statement via
-// `buildSkillContent`, embed dense + sparse, upsert into the dedicated
+// Mirrors v1's `seedSkillGraphNodes` + `seedUninstalledCatalogSkillMemories`
+// (capability-seed.ts) for the v2 pipeline: enumerate the enabled-skill
+// catalog AND uninstalled catalog skills, render each skill's prose statement
+// via `buildSkillContent`, embed dense + sparse, upsert into the dedicated
 // `memory_v2_skills` Qdrant collection, and prune stale points from prior
-// catalog state.
+// catalog state. Including uninstalled catalog skills ensures their activation
+// hints are discoverable by intent so the model can auto-install them.
 //
 // Unlike v1, skill entries are kept in a small in-process cache so the render
 // path can fetch a `SkillEntry` synchronously by id without round-tripping to
@@ -17,8 +19,13 @@ import { isAssistantFeatureFlagEnabled } from "../../config/assistant-feature-fl
 import { getConfig } from "../../config/loader.js";
 import { resolveSkillStates } from "../../config/skill-state.js";
 import { loadSkillCatalog } from "../../config/skills.js";
-import { fromSkillSummary } from "../../skills/skill-memory.js";
+import { getCatalog } from "../../skills/catalog-cache.js";
+import {
+  fromCatalogSkill,
+  fromSkillSummary,
+} from "../../skills/skill-memory.js";
 import { getLogger } from "../../util/logger.js";
+import { applyCorrectionIfCalibrated } from "../anisotropy.js";
 import {
   embedWithBackend,
   generateSparseEmbedding,
@@ -55,6 +62,9 @@ let entries: Map<string, SkillEntry> | null = null;
  *      already enforces this, but we mirror v1's enforcement point so the v2
  *      collection never holds an embedding for a flag-gated skill if the two
  *      ever drift.
+ *   3b. Fetch the full remote catalog and seed any uninstalled skills so their
+ *      activation hints are discoverable by semantic search. Best-effort: if
+ *      the catalog fetch fails, only installed skills are seeded.
  *   4. Embed all `content` strings in a single dense `embedWithBackend` call,
  *      and a per-skill synchronous `generateSparseEmbedding`.
  *   5. Upsert one Qdrant point per skill via `upsertSkillEmbedding` (keyed
@@ -70,6 +80,13 @@ export async function seedV2SkillEntries(): Promise<void> {
     const resolved = resolveSkillStates(catalog, config);
     const enabled = resolved.filter((r) => r.state === "enabled");
 
+    // Track every locally-installed skill id (regardless of enabled/disabled
+    // state) so the catalog-seeding loop below treats them all as "installed"
+    // and never re-seeds a disabled skill from `getCatalog()` as if it were
+    // uninstalled. Mirrors v1's `seedUninstalledCatalogSkillMemories`, which
+    // keys off `loadSkillCatalog()` (the installed set) for the same reason.
+    const installedIds = new Set<string>(catalog.map((s) => s.id));
+
     // Build the input list, applying the mcp-setup description augmentation
     // and the defense-in-depth feature-flag filter.
     const seeds: SkillEntry[] = [];
@@ -82,12 +99,39 @@ export async function seedV2SkillEntries(): Promise<void> {
       seeds.push({ id: summary.id, content });
     }
 
+    // Seed uninstalled catalog skills so their activation hints are
+    // discoverable by intent (mirrors v1's seedUninstalledCatalogSkillMemories).
+    // Track whether the catalog was available so we can guard pruning below.
+    let catalogAvailable = false;
+    try {
+      const fullCatalog = await getCatalog();
+      catalogAvailable = fullCatalog.length > 0;
+      for (const entry of fullCatalog) {
+        if (installedIds.has(entry.id)) continue;
+        const flagKey = entry.metadata?.vellum?.["feature-flag"];
+        if (flagKey && !isAssistantFeatureFlagEnabled(flagKey, config))
+          continue;
+        const content = buildSkillContent(fromCatalogSkill(entry));
+        seeds.push({ id: entry.id, content });
+      }
+    } catch (err) {
+      log.warn(
+        { err },
+        "Failed to fetch catalog for uninstalled skill seeding — continuing with installed skills only",
+      );
+    }
+
     // Embed all content strings in one batched call. Sparse vectors are
     // computed in-process (no network).
-    const { vectors: denseVectors } = await embedWithBackend(
+    const embedded = await embedWithBackend(
       config,
       seeds.map((s) => s.content),
     );
+    const denseVectors = await Promise.all(
+      embedded.vectors.map((v) =>
+        applyCorrectionIfCalibrated(v, embedded.provider, embedded.model),
+      ),
+    );
 
     const now = Date.now();
     const nextEntries = new Map<string, SkillEntry>();
@@ -102,8 +146,18 @@ export async function seedV2SkillEntries(): Promise<void> {
       nextEntries.set(seed.id, seed);
     }
 
-    // Prune any points whose id is no longer in the active set.
-    await pruneSkillsExcept(seeds.map((s) => s.id));
+    // Prune stale points. When the catalog is unavailable (empty array from
+    // network failure or cold cache), we cannot enumerate which uninstalled
+    // catalog skills should exist, so skip pruning entirely to avoid
+    // aggressively removing previously-seeded catalog skill embeddings.
+    // Mirrors v1's safeguard in capability-seed.ts (lines 124–143).
+    if (catalogAvailable) {
+      await pruneSkillsExcept(seeds.map((s) => s.id));
+    } else {
+      log.info(
+        "Catalog unavailable — skipping skill pruning to preserve prior catalog embeddings",
+      );
+    }
 
     // Atomically replace the cache only after every step above succeeds.
     entries = nextEntries;
@@ -122,6 +176,15 @@ export function getSkillCapability(id: string): SkillEntry | null {
   return entries?.get(id) ?? null;
 }
 
+/**
+ * Every skill id in the cache — both installed-and-enabled skills and
+ * uninstalled-catalog skills. Empty before the first `seedV2SkillEntries`
+ * run completes.
+ */
+export function getAllSkillIds(): string[] {
+  return entries ? [...entries.keys()] : [];
+}
+
 /** @internal Test-only: clear the module-level cache. */
 export function _resetSkillStoreForTests(): void {
   entries = null;