@vellumai/assistant 0.8.10 → 0.8.11-staging.1

package/src/plugins/defaults/memory-v3-shadow/pool-select.ts

@@ -1,18 +1,34 @@
 /**
  * Memory v3 — single pool selector.
  *
- * Runs a SINGLE forced-tool call over one unified candidate pool. Each candidate
- * is a page slug paired with a per-candidate `descriptor` the caller supplies —
- * the matched section text for a needle/dense hit, or a curated link description
- * for an edge page. The caller assembles the pool by unioning the section lanes;
- * the pool selector only decides which of those candidates the reply will draw
- * on.
+ * Runs a SINGLE forced-tool call over one unified candidate pool rendered in
+ * two segments that share one numbering:
  *
- * No cache breakpoint. The pool is recomputed from per-turn section matches and
- * is therefore dynamic per turn, so it carries no `cache_control` breakpoint:
- * caching a prefix that changes every turn would never hit. Carry-forward (the
- * working set unioned in by the orchestrator) is the cache mechanism here, not a
- * static prefix breakpoint.
+ *   1. STABLE PREFIX — the core+hot lane pages as FULL CARDS (head section +
+ *      section TOC, see `card.ts`), numbered `[1]…[m]` in pool order. The
+ *      cards are query- and conversation-state-independent, so the rendered
+ *      segment is byte-identical across turns while the lanes are unchanged
+ *      (lane invalidation at consolidation is the recompute cadence). It is
+ *      emitted as its OWN content block carrying a `cache_control`
+ *      breakpoint, so it rides the provider KV cache — block-level
+ *      `cache_control` is preserved through `toAnthropicBlockSafe` (PR
+ *      #33258; covered by the caller-stamped-block tests in
+ *      `anthropic-provider.test.ts`).
+ *   2. DYNAMIC TAIL — the per-turn finder candidates as compact numbered
+ *      lines (`[m+1] <slug> — <matched-section snippet>`), then the
+ *      situational/recent/current conversation context. Re-rendered every
+ *      turn; never cached. `disableTurnStartCache` suppresses the provider's
+ *      auto-applied 1h turn-start breakpoint, which would otherwise land on
+ *      this per-turn-varying block and pay cache_creation with no future hit
+ *      (same rationale as the v2 router).
+ *
+ * Candidate composition is deliberately conversation-state-independent:
+ * already-injected pages are NOT filtered out of the pool — that would change
+ * the stable prefix per conversation state and bust the cache. Re-selecting
+ * an injected page is harmless (injection dedup happens downstream) and feeds
+ * hot-set frecency + spotlight eligibility. A page may appear BOTH as a
+ * stable-prefix card and as a finder line (its current matched section);
+ * selections are deduped by slug.
  *
  * Failure handling is recall-safe by construction:
  *   - explicit `ids` → select exactly those candidates,
@@ -27,11 +43,16 @@
 
 import { z } from "zod";
 
+import { cachedTextBlock } from "../../../providers/cache-control.js";
 import {
   extractToolUse,
   getConfiguredProvider,
 } from "../../../providers/provider-send-message.js";
-import type { Message, ToolDefinition } from "../../../providers/types.js";
+import type {
+  ContentBlock,
+  Message,
+  ToolDefinition,
+} from "../../../providers/types.js";
 import { getLogger } from "../../../util/logger.js";
 import { truncate } from "../../../util/truncate.js";
 import { retryForResult } from "./llm-retry.js";
@@ -39,22 +60,40 @@ import type { MemoryRoutingTurn, SelectedPage, Slug } from "./types.js";
 
 const log = getLogger("memory-v3-pool-select");
 
-/** A candidate page in the unified pool, with the descriptor that justifies it. */
+/** A dynamic-tail (finder) candidate: the slug plus the descriptor that
+ *  justifies it — a matched section for a needle/dense hit, or a curated link
+ *  description for an edge page. Rendered as a one-line snippet. */
 export interface PoolCandidate {
   slug: Slug;
-  /**
-   * The text that justifies this candidate: a matched section for a
-   * needle/dense hit, or a curated link description for an edge page. Rendered
-   * (truncated) in the numbered candidate list the selector judges against.
-   */
   descriptor: string;
 }
 
+/** A stable-prefix candidate: the slug plus its pre-rendered FULL card
+ *  (`renderCard` output — head section + TOC). Cards must be byte-stable
+ *  across turns for the prefix to ride the provider KV cache, which is why
+ *  callers pre-render them at lane init rather than per turn. */
+export interface StableCandidate {
+  slug: Slug;
+  card: string;
+}
+
+/**
+ * The selector's unified candidate pool in cache order: the stable prefix
+ * (core+hot cards) then the dynamic finder tail. The two segments share one
+ * numbering — `[1]…[m]` cards, `[m+1]…` finder lines — and MAY repeat a slug
+ * (a finder hit on a core/hot page keeps its matched-section line so the
+ * page's CURRENT relevance stays visible); selections are deduped by slug.
+ */
+export interface SelectorPool {
+  stable: StableCandidate[];
+  finder: PoolCandidate[];
+}
+
 /** Tool name forced via `tool_choice`. Shared constant so tests can match it. */
 const SELECT_PAGES_TOOL_NAME = "select_pages";
 
-/** Descriptors are truncated to keep the candidate list compact. */
-const DESCRIPTOR_MAX_CHARS = 400;
+/** Finder-line snippets are truncated to keep the dynamic tail compact. */
+const SNIPPET_MAX_CHARS = 300;
 
 const SelectPagesSchema = z.object({
   // Optional: an omitted `ids` field is the recall-safe "keep everything"
@@ -66,8 +105,8 @@ const SelectPagesSchema = z.object({
 const SELECT_PAGES_TOOL: ToolDefinition = {
   name: SELECT_PAGES_TOOL_NAME,
   description:
-    "Select the candidate pages whose content the reply would directly draw " +
-    "on. Lean inclusive — when in doubt, keep a candidate; for a list or " +
+    "Select the candidate pages whose content the reply would draw on. Lean " +
+    "inclusive — when in doubt, keep a candidate; for a list or " +
     '"all of X" request keep every candidate that belongs. Pass `pinned_ids` ' +
     "for pages the conversation is centrally about. Omit `ids` only as a " +
     "recall-safe fallback when you cannot judge the pool (keeps every " +
@@ -88,34 +127,63 @@ const SELECT_PAGES_TOOL: ToolDefinition = {
   },
 };
 
-const SYSTEM_PROMPT = `You are given a pool of candidate memory pages, each surfaced because some part of it matched the conversation. Select every page whose specific content the reply to THIS message would directly draw on.
+const SYSTEM_PROMPT = `You are given the candidate memory pages for an assistant's next reply, in two segments that share one numbering: full page cards first (the curated core and recently-recurring pages, shown every turn), then this turn's search hits as one-line snippets.
+
+Select EVERY candidate whose content the upcoming reply would draw on. That includes facts the reply needs, but equally register, established framing, calibration rules, and relationship/person/project texture — pages that shape HOW to reply, not only what to say. There is no limit on how many you may select; recall matters more than precision, so when a candidate could plausibly inform the reply, keep it. For a list or an "all of X" request, keep EVERY candidate that belongs to X rather than guessing a representative subset.
 
-Lean inclusive: recall matters more than precision here, so when a candidate could plausibly inform the reply, keep it. For a list or an "all of X" request, keep EVERY candidate that belongs to X rather than guessing a representative subset.
+Pages you select persist in the conversation automatically, and re-selecting a page that is already in context is harmless (duplicates are removed downstream) — so never withhold a candidate because it might have been selected before. Judge each candidate only by whether the upcoming reply would draw on it.
 
 A page can be relevant because of the current situation — the date or the live scratchpad — not only the message: keep a page the situation makes pertinent (e.g. a person whose anniversary is today).
 
 If the conversation is centrally ABOUT a page (rather than only peripherally relevant to it), mark that page as pinned. Call \`select_pages\` with the chosen IDs. Omit \`ids\` only as a recall-safe fallback when you cannot judge the pool (keeps every candidate); return \`[]\` when candidates are present but none are relevant.`;
 
-/** Collapse a descriptor to one line and cap its length for the candidate list. */
-function renderDescriptor(descriptor: string): string {
-  return truncate(descriptor.replace(/\s+/g, " ").trim(), DESCRIPTOR_MAX_CHARS);
+/** Collapse a descriptor to one line and cap its length for a finder line. */
+function renderSnippet(descriptor: string): string {
+  return truncate(descriptor.replace(/\s+/g, " ").trim(), SNIPPET_MAX_CHARS);
+}
+
+/**
+ * Render the stable prefix: each core/hot card prefixed with its pool number.
+ * Pure concatenation of pre-rendered cards — byte-identical across turns for
+ * identical `stable` input, which is the cache contract.
+ */
+function renderCardSegment(stable: StableCandidate[]): string {
+  const cards = stable.map((c, i) => `[${i + 1}] ${c.card}`);
+  return `<candidate_cards>\n${cards.join("\n\n")}\n</candidate_cards>`;
 }
 
 /**
- * Render the numbered candidate list: one `[i] slug — descriptor` line per
- * candidate, descriptors collapsed to one line and length-capped. The pool is
- * dynamic per turn, so this block is NOT cached (see the module doc).
+ * Render the finder tail: one `[m+i] slug — snippet` line per candidate,
+ * numbered continuing after the `offset` stable-prefix cards. A candidate
+ * with an empty descriptor renders without the dash.
  */
-function renderCandidateList(pool: PoolCandidate[]): string {
-  const lines = pool.map(
-    (c, i) => `[${i + 1}] ${c.slug} — ${renderDescriptor(c.descriptor)}`,
-  );
+function renderFinderSegment(finder: PoolCandidate[], offset: number): string {
+  const lines = finder.map((c, i) => {
+    const snippet = renderSnippet(c.descriptor);
+    const id = offset + i + 1;
+    return snippet.length > 0
+      ? `[${id}] ${c.slug} — ${snippet}`
+      : `[${id}] ${c.slug}`;
+  });
   return `<candidates>\n${lines.join("\n")}\n</candidates>`;
 }
 
+/** Dedupe selections by slug, preserving first-seen order and ORing pinned
+ *  flags (a page can be selected as both a card and a finder line). */
+function dedupeBySlug(
+  entries: Array<{ slug: Slug; pinned: boolean }>,
+): SelectedPage[] {
+  const bySlug = new Map<Slug, boolean>();
+  for (const entry of entries) {
+    bySlug.set(entry.slug, (bySlug.get(entry.slug) ?? false) || entry.pinned);
+  }
+  return [...bySlug].map(([slug, pinned]) => ({ slug, pinned }));
+}
+
 /**
  * Run the single forced-tool selector over the unified candidate pool. Returns
- * the pages to inject.
+ * the pages to inject, deduped by slug (a page that appeared as both a card
+ * and a finder line yields one entry, pinned flags ORed).
  *
  * An omitted `ids` keeps ALL candidates (the recall-safe "all of these are
  * relevant" signal); an explicit `[]` keeps none; an infrastructure failure
@@ -123,41 +191,48 @@ function renderCandidateList(pool: PoolCandidate[]): string {
  * recall lanes the orchestrator unions in.
  */
 export async function selectPool(
-  pool: PoolCandidate[],
+  pool: SelectorPool,
   turn: MemoryRoutingTurn,
 ): Promise<SelectedPage[]> {
-  if (pool.length === 0) return [];
+  // The concatenated numbering: ids 1…m are the stable-prefix cards, ids
+  // m+1… are the finder lines.
+  const ordered: Slug[] = [
+    ...pool.stable.map((c) => c.slug),
+    ...pool.finder.map((c) => c.slug),
+  ];
+  if (ordered.length === 0) return [];
 
   const keepAll = (): SelectedPage[] =>
-    pool.map((c) => ({ slug: c.slug, pinned: false }));
+    dedupeBySlug(ordered.map((slug) => ({ slug, pinned: false })));
 
   const provider = await getConfiguredProvider("memoryV3SelectL2");
   if (!provider) {
     log.warn(
-      { candidateCount: pool.length },
+      { candidateCount: ordered.length },
       "pool selector provider unavailable; degrading to deterministic lanes",
     );
     return [];
   }
 
-  // The pool is dynamic per turn, so there is no static cache breakpoint here;
-  // carry-forward is the cache mechanism. The candidate list and the per-turn
-  // context go in a single plain text block.
-  const userMsg: Message = {
-    role: "user",
-    content: [
-      {
-        type: "text",
-        text:
-          `${renderCandidateList(pool)}\n` +
-          (turn.situationalContext
-            ? `<situation>${turn.situationalContext}</situation>\n`
-            : "") +
-          `<recent_context>${turn.recentContext}</recent_context>\n` +
-          `<current_message>${turn.currentMessage}</current_message>`,
-      },
-    ],
-  };
+  // Two content blocks: the stable prefix (cards) carries the cache
+  // breakpoint; the dynamic tail (finder lines + per-turn context) does not.
+  // See the module doc for the cache contract.
+  const content: ContentBlock[] = [];
+  if (pool.stable.length > 0) {
+    content.push(cachedTextBlock(renderCardSegment(pool.stable)));
+  }
+  const tailParts: string[] = [];
+  if (pool.finder.length > 0) {
+    tailParts.push(renderFinderSegment(pool.finder, pool.stable.length));
+  }
+  if (turn.situationalContext) {
+    tailParts.push(`<situation>${turn.situationalContext}</situation>`);
+  }
+  tailParts.push(`<recent_context>${turn.recentContext}</recent_context>`);
+  tailParts.push(`<current_message>${turn.currentMessage}</current_message>`);
+  content.push({ type: "text", text: tailParts.join("\n") });
+
+  const userMsg: Message = { role: "user", content };
 
   // One forced-tool call, retried a few times so a transient malformed response
   // (no usable tool_use, or tool input that fails the schema) re-prompts before
@@ -170,6 +245,11 @@ export async function selectPool(
       config: {
         callSite: "memoryV3SelectL2" as const,
         tool_choice: { type: "tool" as const, name: SELECT_PAGES_TOOL_NAME },
+        // The last block of this one-shot message varies every turn; the
+        // provider's auto-applied turn-start breakpoint would land on it and
+        // pay cache_creation with no future hit. The stable-prefix block
+        // above carries its own breakpoint instead.
+        disableTurnStartCache: true,
       },
     });
     const toolBlock = extractToolUse(response);
@@ -180,7 +260,7 @@ export async function selectPool(
 
   if (parsed === null) {
     log.warn(
-      { candidateCount: pool.length },
+      { candidateCount: ordered.length },
       "pool selector could not obtain a selection after retries; degrading to deterministic lanes",
     );
     return [];
@@ -191,14 +271,12 @@ export async function selectPool(
 
   const pinned = new Set(parsed.pinned_ids ?? []);
 
-  // Map 1-based IDs back to candidate slugs, dropping out-of-range IDs without
-  // throwing. De-duplicate while preserving model-returned order.
-  const seen = new Set<number>();
-  const selected: SelectedPage[] = [];
+  // Map 1-based IDs over the concatenated numbering, dropping out-of-range
+  // IDs without throwing, then dedupe by slug (pinned flags ORed).
+  const selected: Array<{ slug: Slug; pinned: boolean }> = [];
   for (const id of parsed.ids) {
-    if (id < 1 || id > pool.length || seen.has(id)) continue;
-    seen.add(id);
-    selected.push({ slug: pool[id - 1]!.slug, pinned: pinned.has(id) });
+    if (id < 1 || id > ordered.length) continue;
+    selected.push({ slug: ordered[id - 1]!, pinned: pinned.has(id) });
   }
-  return selected;
+  return dedupeBySlug(selected);
 }