@vellumai/assistant 0.12.3-staging.1 → 0.12.3-staging.2

package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts

@@ -164,6 +164,23 @@ const PAGE_2_LEAD: Section = {
   text: "y",
   ordinal: 0,
 };
+const DEFAULT_FINDER = [
+  {
+    slug: "page-1",
+    section: PAGE_1_LEAD,
+    descriptor: "",
+    lane: "needle" as const,
+  },
+  {
+    slug: "page-2",
+    section: PAGE_2_LEAD,
+    descriptor: "",
+    lane: "dense" as const,
+  },
+  { slug: "page-3", descriptor: "", lane: "edge" as const },
+  { slug: "page-4", descriptor: "", lane: "reply" as const },
+  { slug: "page-5", descriptor: "", lane: "learned" as const },
+];
 const orchestrateSpy = mock(
   async (): Promise<OrchestrateResult> => ({
     selections: [
@@ -181,20 +198,18 @@ const orchestrateSpy = mock(
       hot: ["page-hot"],
       fresh: ["page-fresh"],
       always: [CAPABILITY_SLUG],
-      finder: [
-        {
-          slug: "page-1",
-          section: PAGE_1_LEAD,
-          descriptor: "",
-          lane: "needle",
-        },
-        { slug: "page-2", section: PAGE_2_LEAD, descriptor: "", lane: "dense" },
-        { slug: "page-3", descriptor: "", lane: "edge" },
-        { slug: "page-4", descriptor: "", lane: "reply" },
-        { slug: "page-5", descriptor: "", lane: "learned" },
-      ],
+      finder: DEFAULT_FINDER,
     },
     selectorRan: true,
+    pool: {
+      stable: [
+        { slug: "page-core", card: "core card", lane: "core" },
+        { slug: "page-hot", card: "hot card", lane: "hot" },
+        { slug: "page-fresh", card: "fresh card", lane: "fresh" },
+        { slug: CAPABILITY_SLUG, card: "skill card", lane: "always" },
+      ],
+      finder: DEFAULT_FINDER,
+    },
   }),
 );
 
@@ -665,6 +680,11 @@ function chosenBySlug(): Record<string, boolean> {
  *  needle lane, page-2 via dense, page-3 via edge) whose selector kept exactly
  *  `kept`. */
 function poolOf(kept: string[]): OrchestrateResult {
+  const finder = [
+    { slug: "page-1", descriptor: "", lane: "needle" as const },
+    { slug: "page-2", descriptor: "", lane: "dense" as const },
+    { slug: "page-3", descriptor: "", lane: "edge" as const },
+  ];
   return {
     selections: kept.map((slug) => ({ slug, sections: [] })),
     lanes: {
@@ -672,13 +692,10 @@ function poolOf(kept: string[]): OrchestrateResult {
       hot: [],
       fresh: [],
       always: [],
-      finder: [
-        { slug: "page-1", descriptor: "", lane: "needle" },
-        { slug: "page-2", descriptor: "", lane: "dense" },
-        { slug: "page-3", descriptor: "", lane: "edge" },
-      ],
+      finder,
     },
     selectorRan: true,
+    pool: { stable: [], finder },
   };
 }
 
@@ -893,6 +910,15 @@ describe("memory-v3 engine", () => {
       text: "x",
       ordinal: 1,
     };
+    const finder = [
+      {
+        slug: "page-rare",
+        section: inventory,
+        terms: ["turnip"],
+        descriptor: "",
+        lane: "rare" as const,
+      },
+    ];
     orchestrateSpy.mockImplementationOnce(async () => ({
       selections: [{ slug: "page-rare", sections: [inventory] }],
       lanes: {
@@ -900,17 +926,10 @@ describe("memory-v3 engine", () => {
         hot: [],
         fresh: [],
         always: [],
-        finder: [
-          {
-            slug: "page-rare",
-            section: inventory,
-            terms: ["turnip"],
-            descriptor: "",
-            lane: "rare",
-          },
-        ],
+        finder,
       },
       selectorRan: true,
+      pool: { stable: [], finder },
     }));
 
     await observeTurn("conv-1", 2);
@@ -1023,7 +1042,7 @@ describe("memory-v3 engine", () => {
       keptAll: false,
       gateReason: "dense_pass",
       pool: {
-        stable: [{ slug: "page-core", card: "core card" }],
+        stable: [{ slug: "page-core", card: "core card", lane: "core" }],
         finder,
       },
     }));

package/src/plugins/defaults/memory/v3/orchestrate.ts

@@ -67,7 +67,9 @@
  *   3. A SINGLE forced-tool select (`selectPool`) over the whole pool. The
  *      result is this turn's selections — current turn only. Cross-turn
  *      persistence is the injector's job (net-new blocks frozen into history),
- *      not a per-turn re-rendered carry set.
+ *      not a per-turn re-rendered carry set. When the call site resolves to
+ *      TypeSafe, `selectPool` asks one System One noul per numbered candidate
+ *      instead of forcing `select_pages`.
  */
 
 import type { AssistantConfig } from "../../../../config/schema.js";
@@ -338,30 +340,34 @@ export interface OrchestrateResult {
    *  carried-forward set unioned in. */
   selections: SelectedPage[];
   /** The candidate lanes in cache order; see {@link OrchestrateLanes}. Consumed
-   *  by the selection telemetry (lane attribution), the per-turn pool record
-   *  (`pool-log-store.ts`), and the downstream selector rendering. */
+   *  by selection telemetry for lane attribution and retained as the complete
+   *  retrieval diagnostics even when selector budgeting trims `pool`. */
   lanes: OrchestrateLanes;
   /** Whether the selector LLM judged a non-empty pool this turn (the
    *  `selector_ran` telemetry field). False when the pool was empty, when the
    *  disabled-selector passthrough kept every candidate, when a closed
    *  injection gate hard-skipped selection, and when the selector's provider
-   *  failed and the stable prefix was kept unjudged (`selectorFailure`). On
-   *  the gate path `lanes` still carries the stable prefix as computed, but
-   *  no pool was ever assembled, so a false value with empty `selections`
-   *  means the selector was given nothing. */
+   *  failed and the retained stable candidates were kept unjudged
+   *  (`selectorFailure`). A false value with empty `selections` means the
+   *  selector was given nothing only when `pool` is absent; a failed attempt
+   *  can carry a finder-only pool with no fallback selections. */
   selectorRan: boolean;
   /** Set when the selector could not be run this turn (provider unavailable,
    *  or no usable tool call after the re-prompt retries). `selections` then
-   *  holds the stable prefix unjudged; finder candidates are dropped because
-   *  they are evidence for a judge, not evidence to inject on their own. The
+   *  holds the retained stable candidates unjudged; finder candidates are
+   *  dropped because they are evidence for a judge, not evidence to inject on
+   *  their own. The
    *  live injector queues a notice so the person knows this turn drew on
    *  core memories only. */
   selectorFailure?: MemoryV3RetrievalUnavailableError;
   /** The pool as the selector was given it, in one numbering: the
    *  stable-prefix cards, then the finder lines. Absent when no pool was
-   *  assembled this turn (a closed gate's hard skip). Read by the pool input
-   *  capture (`buildPoolInput` in `pool-log-store.ts`). */
+   *  assembled this turn (a closed gate's hard skip). Read by the pool record
+   *  and input capture (`pool-log-store.ts`). */
   pool?: SelectorPool;
+  /** The exact context strings rendered for the selector request. Absent when
+   *  no selector pool was assembled. */
+  selectorTurn?: MemoryRoutingTurn;
   /** Whether the selector's recall-safe keep-all fallback fired
    *  (`selectPool`'s `keptAll`). Absent when no pool was assembled. */
   keptAll?: boolean;
@@ -404,7 +410,12 @@ export async function orchestrate(
   // the byte-stable-prefix contract). Built lazily so the gate's bypass path can
   // reuse the exact same construction as the normal step-2 pool assembly.
   const buildStable = (): StableCandidate[] =>
-    [...core, ...hot, ...fresh, ...always].map((slug) => {
+    [
+      ...core.map((slug) => ({ slug, lane: "core" as const })),
+      ...hot.map((slug) => ({ slug, lane: "hot" as const })),
+      ...fresh.map((slug) => ({ slug, lane: "fresh" as const })),
+      ...always.map((slug) => ({ slug, lane: "always" as const })),
+    ].map(({ slug, lane }) => {
       const card = deps.prefixCards.get(slug);
       if (card === undefined) {
         // Lane init renders a card for every core/hot slug; a hole here means
@@ -415,7 +426,7 @@ export async function orchestrate(
           `memory-v3: no pre-rendered card for stable-prefix slug "${slug}"`,
         );
       }
-      return { slug, card };
+      return { slug, card, lane };
     });
 
   // Run the selector over a pool, or pass its candidates straight through when
@@ -438,39 +449,51 @@ export async function orchestrate(
   ): Promise<{
     selections: SelectedPage[];
     keptAll: boolean;
+    pool: SelectorPool;
+    selectorTurn: MemoryRoutingTurn;
     failure?: MemoryV3RetrievalUnavailableError;
   }> =>
     timeLatencySubSpan("v3_selection", "Memory selection", async () => {
       if (deps.selectorEnabled === false) {
-        return { selections: selectAllPoolCandidates(pool), keptAll: false };
+        return {
+          selections: selectAllPoolCandidates(pool),
+          keptAll: false,
+          pool,
+          selectorTurn: turn,
+        };
       }
       try {
-        const { pages, keptAll } = await selectPool(
-          pool,
-          turn,
-          deps.selectorPrompt,
-        );
-        return { selections: pages, keptAll };
+        const selection = await selectPool(pool, turn, deps.selectorPrompt);
+        return {
+          selections: selection.pages,
+          keptAll: selection.keptAll,
+          pool: selection.pool,
+          selectorTurn: selection.turn,
+        };
       } catch (err) {
         if (!(err instanceof MemoryV3RetrievalUnavailableError)) {
           throw err;
         }
+        const attemptedPool = err.pool ?? pool;
+        const attemptedTurn = err.turn ?? turn;
         log.warn(
           {
             conversationId: turn.conversationId,
             turnNumber: turn.turnNumber,
-            stableCount: pool.stable.length,
-            finderCount: pool.finder.length,
+            stableCount: attemptedPool.stable.length,
+            finderCount: attemptedPool.finder.length,
             err: err.message,
           },
           "memory-v3 selector unavailable; keeping the stable prefix unjudged",
         );
         return {
           selections: selectAllPoolCandidates({
-            stable: pool.stable,
+            stable: attemptedPool.stable,
             finder: [],
           }),
           keptAll: false,
+          pool: attemptedPool,
+          selectorTurn: attemptedTurn,
           failure: err,
         };
       }
@@ -910,13 +933,23 @@ export async function orchestrate(
             selections: SelectedPage[],
             selectorRan: boolean,
             selectorFailure?: MemoryV3RetrievalUnavailableError,
-            judged?: { pool: SelectorPool; keptAll: boolean },
+            judged?: {
+              pool: SelectorPool;
+              selectorTurn: MemoryRoutingTurn;
+              keptAll: boolean;
+            },
           ): OrchestrateResult => ({
             selections,
             lanes: { core, hot, fresh, always, finder: [] },
             selectorRan,
             gateReason: gate.reason,
-            ...(judged ? { pool: judged.pool, keptAll: judged.keptAll } : {}),
+            ...(judged
+              ? {
+                  pool: judged.pool,
+                  selectorTurn: judged.selectorTurn,
+                  keptAll: judged.keptAll,
+                }
+              : {}),
             ...(selectorFailure ? { selectorFailure } : {}),
           });
           if (deps.gateConfig.bypassForCore) {
@@ -936,19 +969,18 @@ export async function orchestrate(
             const {
               selections: bypassed,
               keptAll,
+              pool: selectedPool,
+              selectorTurn,
               failure,
             } = await runSelection(stableOnly);
-            recordSelection(
-              bypassed,
-              stableOnly.stable.length,
-              keptAll,
-              failure,
-            );
+            const selectedPoolSize =
+              selectedPool.stable.length + selectedPool.finder.length;
+            recordSelection(bypassed, selectedPoolSize, keptAll, failure);
             return closed(
               bypassed,
-              selectorRanOver(stableOnly.stable.length, failure),
+              selectorRanOver(selectedPoolSize, failure),
               failure,
-              { pool: stableOnly, keptAll },
+              { pool: selectedPool, selectorTurn, keptAll },
             );
           }
           // Hard skip: the selector is never consulted, so this is a zero
@@ -1054,15 +1086,23 @@ export async function orchestrate(
     "Gate & edge expansion",
     Date.now() - expandStartedAt,
   );
-  const poolSize = stable.length + finderTail.length;
-  const { selections, keptAll, failure } = await runSelection(pool);
-  recordSelection(selections, poolSize, keptAll, failure);
+  const {
+    selections,
+    keptAll,
+    pool: selectedPool,
+    selectorTurn,
+    failure,
+  } = await runSelection(pool);
+  const selectedPoolSize =
+    selectedPool.stable.length + selectedPool.finder.length;
+  recordSelection(selections, selectedPoolSize, keptAll, failure);
 
   return {
     selections,
     lanes: { core, hot, fresh, always, finder },
-    selectorRan: selectorRanOver(poolSize, failure),
-    pool,
+    selectorRan: selectorRanOver(selectedPoolSize, failure),
+    pool: selectedPool,
+    selectorTurn,
     keptAll,
     ...(gateOutcome ? { gateReason: gateOutcome.reason } : {}),
     ...(failure ? { selectorFailure: failure } : {}),

package/src/plugins/defaults/memory/v3/pool-log-store.ts

@@ -106,30 +106,39 @@ interface StoredPool {
 }
 
 /**
- * Build the turn's pool record from an orchestrate result. The stable prefix
- * comes first in cache order (core, hot, fresh, always-candidate) as
- * whole-page cards with no section; the finder tail follows in surfacing
- * order, each line tagged with the lane that surfaced it and the section it
- * carries. A finder hit on a stable-prefix page therefore appears twice, and
- * a page hit on several sections once per section, exactly as the selector
- * saw it. A card or a section-less line reads chosen when its page was kept;
+ * Build the turn's pool record from the exact selector pool carried by the
+ * orchestrate result. The stable prefix comes first in retained cache order as
+ * whole-page cards with no section; the finder tail follows in retained
+ * surfacing order, each line tagged with the lane that surfaced it and the
+ * section it carries. A finder hit on a stable-prefix page therefore appears
+ * twice, and a page hit on several sections once per section, exactly as the
+ * selector saw it. A card or a section-less line reads chosen when its page
+ * was kept;
  * a line carrying a section reads chosen when that section was selected.
  *
- * When the selector did not run and nothing was selected, no pool reached it:
- * either the injection gate hard-skipped selection (the result's lanes still
- * carry the stable prefix, which the selector never saw) or the pool was
- * empty. The record is then empty rather than a list of candidates marked
- * unchosen, which would read as a rejection the selector never made. The
- * disabled-selector passthrough keeps every pooled candidate as a selection,
- * so it still records its pool.
+ * A result without `pool` records no candidates: the injection gate either
+ * hard-skipped selection or no pool was assembled. A failed selector attempt
+ * still carries the exact pool it received, with retained stable cards chosen
+ * by the unjudged fallback and finder lines unchosen. The disabled-selector
+ * passthrough also carries and records its pool.
  */
 export function buildPoolRecord(result: OrchestrateResult): PoolRecord {
-  if (!result.selectorRan && result.selections.length === 0) {
+  const pool = result.pool;
+  if (pool === undefined) {
+    if (result.selections.length > 0) {
+      log.warn(
+        {
+          selectorRan: result.selectorRan,
+          selectedCount: result.selections.length,
+        },
+        "memory-v3 pool record: result has selections but no selector pool",
+      );
+    }
     return {
       candidates: [],
       pool_size: 0,
-      selected_count: 0,
-      selector_ran: false,
+      selected_count: result.selections.length,
+      selector_ran: result.selectorRan,
     };
   }
   const selected = new Map<Slug, Set<string>>(
@@ -138,30 +147,27 @@ export function buildPoolRecord(result: OrchestrateResult): PoolRecord {
       new Set(s.sections.map((section) => sectionKey(section))),
     ]),
   );
-  const card = (slug: Slug, lane: PoolLane): PoolCandidateRecord => ({
-    slug,
-    lane,
-    section_title: null,
-    section_key: null,
-    chosen: selected.has(slug),
-  });
-  const { core, hot, fresh, always, finder } = result.lanes;
   const candidates: PoolCandidateRecord[] = [
-    ...core.map((slug) => card(slug, "core")),
-    ...hot.map((slug) => card(slug, "hot")),
-    ...fresh.map((slug) => card(slug, "fresh")),
-    ...always.map((slug) => card(slug, "always")),
-    ...finder.map(({ slug, lane, section }): PoolCandidateRecord => {
-      const key = section ? sectionKey(section) : null;
+    ...pool.stable.map(
+      (candidate): PoolCandidateRecord => ({
+        slug: candidate.slug,
+        lane: candidate.lane,
+        section_title: null,
+        section_key: null,
+        chosen: selected.has(candidate.slug),
+      }),
+    ),
+    ...pool.finder.map((candidate): PoolCandidateRecord => {
+      const key = candidate.section ? sectionKey(candidate.section) : null;
       return {
-        slug,
-        lane,
-        section_title: section?.title ?? null,
+        slug: candidate.slug,
+        lane: candidate.lane,
+        section_title: candidate.section?.title ?? null,
         section_key: key,
         chosen:
           key === null
-            ? selected.has(slug)
-            : (selected.get(slug)?.has(key) ?? false),
+            ? selected.has(candidate.slug)
+            : (selected.get(candidate.slug)?.has(key) ?? false),
       };
     }),
   ];
@@ -348,15 +354,15 @@ export interface PoolInputCapture {
 
 /**
  * Build the turn's input capture from an orchestrate result and the turn it
- * ran on. Each candidate text is exactly what the selector was shown minus
- * its pool number: a stable-prefix candidate's pre-rendered card, a finder
- * candidate's rendered line ({@link renderFinderLine}), in the pool
- * record's order ({@link buildPoolRecord}). A turn whose pool record is
- * empty (the selector never judged a pool and nothing was selected)
- * captures the context strings and no candidate texts, so the hashes stay
- * aligned with that record; a result that carries no selector pool, or one
- * that disagrees with its lanes on the candidate count, captures no texts
- * either and logs the mismatch.
+ * ran on. When selector budgeting trimmed the context, the result's
+ * `selectorTurn` is the source of truth. Each candidate text is exactly what
+ * the selector was shown minus its pool number: a stable-prefix candidate's
+ * pre-rendered card, a finder candidate's rendered line
+ * ({@link renderFinderLine}), in the pool
+ * record's order ({@link buildPoolRecord}). A result that carries no selector
+ * pool captures the turn context and no candidate texts, so the hashes stay
+ * aligned with the empty record. Failed attempts carry their exact attempted
+ * pool and trimmed context even though `selectorRan` is false.
  */
 export function buildPoolInput(
   result: OrchestrateResult,
@@ -365,44 +371,24 @@ export function buildPoolInput(
 ): PoolInputCapture {
   const texts = new Map<string, string>();
   const hashes: string[] = [];
-  const recordsPool = result.selectorRan || result.selections.length > 0;
-  if (recordsPool) {
-    const { core, hot, fresh, always, finder } = result.lanes;
-    const expected =
-      core.length + hot.length + fresh.length + always.length + finder.length;
-    const rendered =
-      result.pool === undefined
-        ? undefined
-        : [
-            ...result.pool.stable.map((candidate) => candidate.card),
-            ...result.pool.finder.map((candidate) =>
-              renderFinderLine(candidate),
-            ),
-          ];
-    if (rendered === undefined || rendered.length !== expected) {
-      log.warn(
-        {
-          conversationId: turn.conversationId,
-          turnNumber: turn.turnNumber,
-          rendered: rendered?.length ?? null,
-          expected,
-        },
-        "memory-v3 pool input capture: the result's pool does not match its lanes; capturing no candidate texts",
-      );
-    } else {
-      for (const text of rendered) {
-        const hash = hashPoolText(text);
-        hashes.push(hash);
-        texts.set(hash, text);
-      }
+  if (result.pool !== undefined) {
+    const rendered = [
+      ...result.pool.stable.map((candidate) => candidate.card),
+      ...result.pool.finder.map((candidate) => renderFinderLine(candidate)),
+    ];
+    for (const text of rendered) {
+      const hash = hashPoolText(text);
+      hashes.push(hash);
+      texts.set(hash, text);
     }
   }
+  const selectorTurn = result.selectorTurn ?? turn;
   return {
     input: {
-      situational_context: turn.situationalContext ?? null,
-      recent_context: turn.recentContext,
-      current_message: turn.currentMessage,
-      previous_assistant_message: turn.previousAssistantMessage ?? null,
+      situational_context: selectorTurn.situationalContext ?? null,
+      recent_context: selectorTurn.recentContext,
+      current_message: selectorTurn.currentMessage,
+      previous_assistant_message: selectorTurn.previousAssistantMessage ?? null,
       selector_prompt_hash:
         result.selectorRan && selectorPrompt !== undefined
           ? hashPoolText(selectorPrompt)

package/src/plugins/defaults/memory/v3/pool-select.test.ts

@@ -72,9 +72,11 @@ const turn: MemoryRoutingTurn = {
   recentContext: "",
 };
 
-const pool = {
+const pool: Parameters<typeof selectPool>[0] = {
   stable: [],
-  finder: [{ slug: "page-a" as Slug, descriptor: "a descriptor" }],
+  finder: [
+    { slug: "page-a" as Slug, descriptor: "a descriptor", lane: "needle" },
+  ],
 };
 
 describe("selectPool", () => {
@@ -133,6 +135,8 @@ describe("selectPool", () => {
     expect(await selectPool(pool, turn)).toEqual({
       pages: [{ slug: "page-a", sections: [] }],
       keptAll: false,
+      pool,
+      turn,
     });
   });
 
@@ -144,6 +148,8 @@ describe("selectPool", () => {
     expect(await selectPool(pool, turn)).toEqual({
       pages: [{ slug: "page-a", sections: [] }],
       keptAll: true,
+      pool,
+      turn,
     });
   });
 
@@ -157,14 +163,14 @@ describe("selectPool", () => {
           input: { ids: [] },
         },
       ]);
-    const candidate = {
+    const candidate: Parameters<typeof renderFinderLine>[0] = {
       slug: "page-a" as Slug,
       descriptor: "a descriptor",
       lane: "needle",
     };
     await selectPool(
       {
-        stable: [{ slug: "page-s" as Slug, card: "card s" }],
+        stable: [{ slug: "page-s" as Slug, card: "card s", lane: "core" }],
         finder: [candidate],
       },
       turn,
@@ -177,9 +183,12 @@ describe("selectPool", () => {
   });
 
   test("an empty candidate pool returns no selections", async () => {
-    expect(await selectPool({ stable: [], finder: [] }, turn)).toEqual({
+    const emptyPool = { stable: [], finder: [] };
+    expect(await selectPool(emptyPool, turn)).toEqual({
       pages: [],
       keptAll: false,
+      pool: emptyPool,
+      turn,
     });
   });
 });