@vellumai/assistant 0.10.0 → 0.10.1-dev.202606240317.ea25efe

package/src/memory/v2/prompts/router.ts

@@ -22,22 +22,14 @@
  * same placeholder substitution applies to overrides.
  */
 
-import { lstatSync, readFileSync } from "node:fs";
-import { homedir } from "node:os";
-import { isAbsolute, join } from "node:path";
-
 import { getLogger } from "../../../util/logger.js";
+import {
+  loadPromptOverride,
+  MAX_PROMPT_OVERRIDE_BYTES,
+} from "../../prompt-override.js";
 
 const log = getLogger("memory-v2-router-prompt");
 
-/**
- * Hard upper bound on the override file size. The bundled prompt is well
- * under 4 KiB; 1 MiB is generous-enough for any reasonable hand-edit while
- * still preventing pathological inputs from being slurped into memory on
- * every router call.
- */
-const MAX_PROMPT_BYTES = 1 * 1024 * 1024;
-
 /** Sentinel substituted with the assistant's display name at runtime. */
 const ASSISTANT_NAME_PLACEHOLDER = "{{ASSISTANT_NAME}}";
 
@@ -102,14 +94,12 @@ export function renderRouterPrompt(opts: RenderRouterPromptOpts): string {
 /**
  * Load the router prompt template, optionally overridden from the file
  * referenced by `memory.v2.router.router_prompt_path`, then substitute the
- * standard placeholders. Path-resolution rules mirror the consolidation
- * prompt override: absolute paths used as-is, leading `~/` expanded to home,
- * relative paths resolved under `workspaceDir`.
+ * standard placeholders. File loading (path resolution, size guard, and the
+ * permissive fall-back to the bundled prompt on a missing/unreadable/empty/
+ * oversized override) is handled by the shared {@link loadPromptOverride}.
  *
- * Failure handling is intentionally permissive — missing file, read error,
- * oversized file, or empty/whitespace-only body all log a warning and fall
- * back to the bundled prompt. Router selection must never break because of
- * a bad override.
+ * An `inlineOverride` (e.g. the simulator playground) takes precedence over the
+ * configured file path; same placeholder substitution and size guard apply.
  */
 export function resolveRouterPrompt(
   overridePath: string | null,
@@ -117,17 +107,15 @@ export function resolveRouterPrompt(
   opts: RenderRouterPromptOpts,
   inlineOverride?: string | null,
 ): string {
-  // Inline override (e.g. simulator playground) takes precedence over the
-  // configured file path and the bundled prompt. Same placeholder
-  // substitution + size guard as the file-path branch; empty/whitespace
-  // bodies fall through to file/bundled resolution so a "cleared" textarea
-  // is treated as no override.
+  // Inline override takes precedence over the configured file path and the
+  // bundled prompt. Empty/whitespace bodies fall through to file/bundled
+  // resolution so a "cleared" textarea is treated as no override.
   if (inlineOverride !== undefined && inlineOverride !== null) {
-    if (inlineOverride.length > MAX_PROMPT_BYTES) {
+    if (inlineOverride.length > MAX_PROMPT_OVERRIDE_BYTES) {
       log.warn(
         {
           size: inlineOverride.length,
-          limit: MAX_PROMPT_BYTES,
+          limit: MAX_PROMPT_OVERRIDE_BYTES,
           reason: "oversized_inline_override",
           fallback: "path_or_bundled",
         },
@@ -138,62 +126,13 @@ export function resolveRouterPrompt(
     }
   }
 
-  if (overridePath === null) return renderRouterPrompt(opts);
-
-  const resolvedPath = resolveOverridePath(overridePath, workspaceDir);
-  let contents: string;
-  try {
-    const stat = lstatSync(resolvedPath);
-    if (!stat.isFile()) {
-      log.warn(
-        {
-          configuredPath: overridePath,
-          resolvedPath,
-          reason: "not_regular_file",
-          fallback: "bundled",
-        },
-        "router prompt override is not a regular file; using bundled prompt",
-      );
-      return renderRouterPrompt(opts);
-    }
-    if (stat.size > MAX_PROMPT_BYTES) {
-      log.warn(
-        {
-          configuredPath: overridePath,
-          resolvedPath,
-          size: stat.size,
-          limit: MAX_PROMPT_BYTES,
-          reason: "oversized_override",
-          fallback: "bundled",
-        },
-        "router prompt override exceeds size limit; using bundled prompt",
-      );
-      return renderRouterPrompt(opts);
-    }
-    contents = readFileSync(resolvedPath, "utf-8");
-  } catch (err) {
-    const code = (err as NodeJS.ErrnoException).code;
-    log.warn(
-      { configuredPath: overridePath, resolvedPath, code, fallback: "bundled" },
-      "router prompt override unreadable; using bundled prompt",
-    );
-    return renderRouterPrompt(opts);
-  }
-
-  if (contents.trim().length === 0) {
-    log.warn(
-      {
-        configuredPath: overridePath,
-        resolvedPath,
-        reason: "empty_override",
-        fallback: "bundled",
-      },
-      "router prompt override is empty; using bundled prompt",
-    );
-    return renderRouterPrompt(opts);
-  }
-
-  return substitutePlaceholders(contents, opts);
+  const override = loadPromptOverride({
+    overridePath,
+    workspaceDir,
+    log,
+    label: "router prompt",
+  });
+  return substitutePlaceholders(override ?? ROUTER_PROMPT, opts);
 }
 
 function substitutePlaceholders(
@@ -207,14 +146,3 @@ function substitutePlaceholders(
     .replaceAll(USER_NAME_PLACEHOLDER, () => user)
     .replaceAll(PAGE_INDEX_PLACEHOLDER, () => opts.pageIndexBlock);
 }
-
-function resolveOverridePath(
-  overridePath: string,
-  workspaceDir: string,
-): string {
-  if (overridePath.startsWith("~/")) {
-    return join(homedir(), overridePath.slice(2));
-  }
-  if (isAbsolute(overridePath)) return overridePath;
-  return join(workspaceDir, overridePath);
-}

package/src/memory/v2/skill-store.ts

@@ -16,8 +16,11 @@
 //
 // Skill entries are kept in a small in-process cache so the render path can
 // fetch a `SkillEntry` synchronously by id without round-tripping to Qdrant.
-// The cache is replaced atomically at the end of a successful seed run; on
-// error the prior cache stays intact (skills are best-effort).
+// The cache is replaced atomically from the local catalog at the end of a seed
+// run — including when the dense embedding backend is unavailable, in which
+// case only the dense Qdrant write is deferred so the cache (and the v3 needle
+// lane it feeds) still reflects the current skills. An unexpected error in
+// another step leaves the prior cache intact (skills are best-effort).
 
 import { isAssistantFeatureFlagEnabled } from "../../config/assistant-feature-flags.js";
 import { getConfig } from "../../config/loader.js";
@@ -228,26 +231,26 @@ async function runSeedV2SkillEntries(generation: number): Promise<void> {
       );
     }
 
-    // Embed all content strings in one batched call when there is anything to
-    // embed. Skipping the call when `seeds` is empty avoids throwing on an
-    // unavailable embedding backend in the all-disabled case, so pruning and
-    // cache replacement still run and clear stale state.
+    // Build the dense + sparse vectors for the Qdrant write. Sparse (BM25/TF)
+    // encoding is computed locally and needs no backend; only the dense vectors
+    // require `embedWithBackend`, which is unconfigured during the cold-start
+    // window before a managed-proxy embedding credential is provisioned.
+    //
+    // A dense-embed failure is non-fatal to the in-memory cache: the v3 needle
+    // finder lane and always-candidate skill pinning read skills from `entries`
+    // / the page index, NOT from Qdrant, so the cache is populated from the
+    // local catalog regardless of backend state and skills stay discoverable
+    // from first boot. Only the dense Qdrant upsert is skipped; the managed-
+    // credential reseed (`maybeReseedCapabilitiesAfterManagedCredential`) and
+    // the v3 maintain pass backfill the dense vectors once the backend recovers.
     const nextEntries = new Map<string, SkillEntry>();
     let denseVectors: number[][] = [];
+    let denseAvailable = false;
+    let denseError: unknown = null;
     let encodeSparse: (
       input: string,
     ) => ReturnType<typeof generateSparseEmbedding> = generateSparseEmbedding;
     if (seeds.length > 0) {
-      const embedded = await embedWithBackend(
-        config,
-        seeds.map((s) => s.content),
-      );
-      denseVectors = await Promise.all(
-        embedded.vectors.map((v) =>
-          applyCorrectionIfCalibrated(v, embedded.provider, embedded.model),
-        ),
-      );
-
       // Skills share the concept-page Qdrant collection, so the sparse vector
       // must use the same stemmed BM25 encoding the concept-page documents
       // carry — otherwise the stemmed BM25 query vectors used by callers (see
@@ -263,6 +266,24 @@ async function runSeedV2SkillEntries(generation: number): Promise<void> {
               b: config.memory.v2.bm25_b,
             })
           : generateSparseEmbedding(input);
+      try {
+        const embedded = await embedWithBackend(
+          config,
+          seeds.map((s) => s.content),
+        );
+        denseVectors = await Promise.all(
+          embedded.vectors.map((v) =>
+            applyCorrectionIfCalibrated(v, embedded.provider, embedded.model),
+          ),
+        );
+        denseAvailable = true;
+      } catch (err) {
+        denseError = err;
+        log.warn(
+          { err },
+          "Embedding backend unavailable — seeding skill cache without dense Qdrant vectors; the needle lane surfaces skills from the cache and the dense lane backfills when the backend recovers",
+        );
+      }
     }
 
     if (generation !== requestedSeedGeneration) {
@@ -274,7 +295,17 @@ async function runSeedV2SkillEntries(generation: number): Promise<void> {
       return;
     }
 
-    if (seeds.length > 0) {
+    // Populate the in-memory cache (and therefore the page index / needle lane)
+    // from the local catalog regardless of dense availability.
+    for (const seed of seeds) {
+      nextEntries.set(seed.id, seed);
+    }
+
+    // Write Qdrant points only when dense vectors were produced. In the
+    // degraded (backend-unavailable) path we skip Qdrant mutation entirely —
+    // both the upsert and the prune below — so we never write half-formed
+    // points or reconcile the collection against a set we did not persist.
+    if (seeds.length > 0 && denseAvailable) {
       const now = Date.now();
       await Promise.all(
         seeds.map((seed, i) =>
@@ -287,16 +318,15 @@ async function runSeedV2SkillEntries(generation: number): Promise<void> {
           }),
         ),
       );
-      for (const seed of seeds) {
-        nextEntries.set(seed.id, seed);
-      }
     }
 
-    // Prune stale skill slugs. When the catalog is unavailable (empty array
-    // from network failure or cold cache), we cannot enumerate which
-    // uninstalled catalog skills should exist, so skip pruning entirely to
-    // avoid aggressively removing previously-seeded catalog skill embeddings.
-    if (catalogAvailable) {
+    // Prune stale skill slugs. Skip when the catalog is unavailable (empty array
+    // from network failure or cold cache — we cannot enumerate which uninstalled
+    // catalog skills should exist) OR when dense vectors were not written this
+    // run (don't reconcile Qdrant against points we did not refresh). The
+    // `seeds.length === 0` branch still prunes under an available catalog so the
+    // all-disabled case clears stale rows.
+    if (catalogAvailable && (denseAvailable || seeds.length === 0)) {
       // Tag legacy skill points missing `payload.kind` before pruning so the
       // kind-scoped prune can see them. Once-per-process; the backfill is
       // idempotent (server-side `is_empty` filter), so a partial failure
@@ -321,19 +351,26 @@ async function runSeedV2SkillEntries(generation: number): Promise<void> {
         seeds.map((s) => s.id),
         { kind: SKILL_PAYLOAD_KIND },
       );
-    } else {
+    } else if (!catalogAvailable) {
       log.info(
         "Catalog unavailable — skipping skill pruning to preserve prior catalog embeddings",
       );
     }
 
-    // Atomically replace the cache only after every step above succeeds.
-    entries = nextEntries;
-    // Drop the page-index cache so the next router invocation observes the
-    // freshly seeded skill set (skill entries share the unified concept-page
+    // Atomically replace the cache from the freshly enumerated skills. The local
+    // resolution (`resolveSkillStates`) is authoritative, so a skill the config
+    // just disabled or removed drops out here even when the remote catalog is
+    // unavailable. Drop the page-index cache so the next router invocation
+    // observes the new skill set (skill entries share the unified concept-page
     // collection and surface in the same index).
+    entries = nextEntries;
     invalidatePageIndex();
-    lastSeedError = null;
+
+    // Surface a dense-embed failure to `throwOnError` callers (the managed-
+    // credential reseed and the operator reembed route) so the existing retry +
+    // maintain machinery backfills the dense lane. The in-memory cache is
+    // already updated above, so the needle lane is fixed regardless.
+    lastSeedError = denseError;
   } catch (err) {
     lastSeedError = err;
     log.warn({ err }, "Failed to seed v2 skill entries");

package/src/memory/v3-eval/__tests__/eval-packets.test.ts

@@ -11,6 +11,8 @@ import {
   type RawMsgRow,
   renderMemorySet,
   resolveDir,
+  selectPinnedTurns,
+  turnConversationId,
 } from "../eval-packets.js";
 
 /** Build a stored content-block JSON string the way the DB holds it. */
@@ -246,3 +248,39 @@ describe("resolveDir", () => {
     );
   });
 });
+
+describe("turn pinning (reproducible re-runs)", () => {
+  const mk = (turn: string, createdAt: number) => ({
+    turn,
+    conversationId: turnConversationId(turn),
+    userText: `u-${turn}`,
+    replyText: `r-${turn}`,
+    context: "",
+    createdAt,
+  });
+  const all = [mk("c1:100", 100), mk("c1:102", 102), mk("c2:200", 200)];
+
+  test("turnConversationId splits on the LAST colon (ids keep colons)", () => {
+    expect(turnConversationId("c1:100")).toBe("c1");
+    // A UUID conversation id has no colons; the createdAt is the final segment.
+    expect(turnConversationId("019d1e04-cb20-719a:1781916529975")).toBe(
+      "019d1e04-cb20-719a",
+    );
+  });
+
+  test("keeps exactly the pinned turns, in pinned order", () => {
+    const picked = selectPinnedTurns(all, ["c2:200", "c1:100"]);
+    expect(picked.map((t) => t.turn)).toEqual(["c2:200", "c1:100"]);
+  });
+
+  test("drops pinned ids that no longer exist (count surfaces the loss)", () => {
+    const picked = selectPinnedTurns(all, ["c1:100", "c9:999", "c1:102"]);
+    // c9:999 was deleted since the ids were captured — dropped, not faked.
+    expect(picked.map((t) => t.turn)).toEqual(["c1:100", "c1:102"]);
+    expect(picked).toHaveLength(2);
+  });
+
+  test("an empty pin set selects nothing", () => {
+    expect(selectPinnedTurns(all, [])).toEqual([]);
+  });
+});

package/src/memory/v3-eval/__tests__/eval-tally.test.ts

@@ -0,0 +1,139 @@
+import { describe, expect, test } from "bun:test";
+
+import type { EvalKeyEntry } from "../eval-packets.js";
+import {
+  binomialTwoSidedSignP,
+  type JudgeVerdict,
+  tallyVerdicts,
+} from "../eval-tally.js";
+
+/**
+ * Build N single-vote turns where set A is always the snapshot, so the mapped
+ * tally is exactly (snapWins snapshot, stageWins staging, tieN ties). Used for
+ * the count-driven verdict tests; per-turn shuffle is exercised separately.
+ */
+function buildAllASnapshot(snapWins: number, stageWins: number, tieN: number) {
+  const verdicts: JudgeVerdict[] = [];
+  const key: EvalKeyEntry[] = [];
+  let i = 0;
+  const push = (winner: "A" | "B" | "tie", sa: number, sb: number) => {
+    const turn = `t${i++}`;
+    key.push({ turn, a: "snapshot", b: "staging" });
+    verdicts.push({ turn, winner, scoreA: sa, scoreB: sb });
+  };
+  for (let n = 0; n < snapWins; n++) push("A", 7, 6); // snapshot (A) wins by 1
+  for (let n = 0; n < stageWins; n++) push("B", 6, 7); // staging (B) wins by 1
+  for (let n = 0; n < tieN; n++) push("tie", 5, 5);
+  return { verdicts, key };
+}
+
+describe("binomialTwoSidedSignP", () => {
+  test("no decided turns → 1", () => {
+    expect(binomialTwoSidedSignP(0, 0)).toBe(1);
+  });
+  test("a near coin-flip split is not significant", () => {
+    expect(binomialTwoSidedSignP(12, 11)).toBeGreaterThan(0.5);
+  });
+  test("a lopsided split is significant", () => {
+    expect(binomialTwoSidedSignP(25, 4)).toBeLessThan(0.001);
+  });
+  test("is symmetric in its arguments", () => {
+    expect(binomialTwoSidedSignP(25, 4)).toBeCloseTo(
+      binomialTwoSidedSignP(4, 25),
+    );
+  });
+  test("a clean sweep is significant", () => {
+    expect(binomialTwoSidedSignP(15, 0)).toBeLessThan(0.05);
+  });
+});
+
+describe("tallyVerdicts — verdict gating", () => {
+  test("a 12-11 split is a TIE (within noise), not a loss — passes the gate", () => {
+    // The field-run shape that a hand tally mis-read as a decisive loss.
+    const { verdicts, key } = buildAllASnapshot(12, 11, 7);
+    const r = tallyVerdicts(verdicts, key);
+    expect(r.snapshotWins).toBe(12);
+    expect(r.stagingWins).toBe(11);
+    expect(r.ties).toBe(7);
+    expect(r.signTestP).toBeGreaterThan(0.05);
+    expect(r.verdict).toBe("tie");
+    expect(r.gate).toBe("pass");
+    // Single-vote panel → not confident; the note nudges toward a real panel.
+    expect(r.confident).toBe(false);
+    expect(r.notes.some((n) => /panel/i.test(n))).toBe(true);
+  });
+
+  test("a significant snapshot lead is a LOSS — fails the gate", () => {
+    const { verdicts, key } = buildAllASnapshot(25, 4, 1);
+    const r = tallyVerdicts(verdicts, key);
+    expect(r.signTestP).toBeLessThan(0.05);
+    expect(r.verdict).toBe("wiki-loses");
+    expect(r.gate).toBe("fail");
+  });
+
+  test("a significant staging lead is a WIN — passes the gate", () => {
+    const { verdicts, key } = buildAllASnapshot(3, 20, 7);
+    const r = tallyVerdicts(verdicts, key);
+    expect(r.verdict).toBe("wiki-wins");
+    expect(r.gate).toBe("pass");
+  });
+});
+
+describe("tallyVerdicts — unblinding + panels", () => {
+  test("maps the winner through the per-turn key (A is not always snapshot)", () => {
+    const key: EvalKeyEntry[] = [
+      { turn: "s1", a: "snapshot", b: "staging" },
+      { turn: "s2", a: "staging", b: "snapshot" }, // shuffled: A is staging here
+    ];
+    const verdicts: JudgeVerdict[] = [
+      { turn: "s1", winner: "A", scoreA: 8, scoreB: 3 }, // A=snapshot wins
+      { turn: "s2", winner: "A", scoreA: 8, scoreB: 3 }, // A=staging wins
+    ];
+    const r = tallyVerdicts(verdicts, key, { minDecided: 1, minPanel: 1 });
+    expect(r.snapshotWins).toBe(1);
+    expect(r.stagingWins).toBe(1);
+    // s1: snap=8/stage=3; s2: staging=8/snapshot=3 → both per-corpus means = 5.5
+    expect(r.meanSnapshot).toBeCloseTo(5.5);
+    expect(r.meanStaging).toBeCloseTo(5.5);
+  });
+
+  test("a panel decides each turn by majority before the set tally", () => {
+    const key: EvalKeyEntry[] = [{ turn: "p1", a: "snapshot", b: "staging" }];
+    const verdicts: JudgeVerdict[] = [
+      { turn: "p1", winner: "B", scoreA: 5, scoreB: 8 }, // staging
+      { turn: "p1", winner: "B", scoreA: 4, scoreB: 7 }, // staging
+      { turn: "p1", winner: "A", scoreA: 7, scoreB: 6 }, // snapshot
+    ];
+    const r = tallyVerdicts(verdicts, key, { minDecided: 1, minPanel: 3 });
+    expect(r.turns).toBe(1);
+    expect(r.stagingWins).toBe(1); // 2 of 3 judges → staging
+    expect(r.snapshotWins).toBe(0);
+    expect(r.panel.mean).toBe(3);
+  });
+
+  test("ignores verdicts with no matching key entry", () => {
+    const key: EvalKeyEntry[] = [{ turn: "k1", a: "snapshot", b: "staging" }];
+    const verdicts: JudgeVerdict[] = [
+      { turn: "k1", winner: "A", scoreA: 7, scoreB: 5 },
+      { turn: "ghost", winner: "B", scoreA: 1, scoreB: 9 },
+    ];
+    const r = tallyVerdicts(verdicts, key, { minDecided: 1, minPanel: 1 });
+    expect(r.unmatchedVerdicts).toBe(1);
+    expect(r.turns).toBe(1);
+    expect(r.snapshotWins).toBe(1);
+  });
+
+  test("derives the winner from scores when `winner` is absent", () => {
+    const key: EvalKeyEntry[] = [{ turn: "d1", a: "snapshot", b: "staging" }];
+    const verdicts = [{ turn: "d1", scoreA: 9, scoreB: 2 }] as JudgeVerdict[];
+    const r = tallyVerdicts(verdicts, key, { minDecided: 1, minPanel: 1 });
+    expect(r.snapshotWins).toBe(1);
+  });
+
+  test("no matched verdicts → empty tally with an explanatory note", () => {
+    const r = tallyVerdicts([], [{ turn: "x", a: "snapshot", b: "staging" }]);
+    expect(r.turns).toBe(0);
+    expect(r.gate).toBe("pass"); // nothing decided → not a loss
+    expect(r.notes.some((n) => /no verdicts matched/i.test(n))).toBe(true);
+  });
+});