@vellumai/assistant 0.12.2 → 0.12.3-staging.2

package/src/plugins/defaults/memory/v3/pool-log-store.ts

@@ -27,17 +27,39 @@
  * Rows are per-turn diagnostics (roughly 10KB each) with no retention job;
  * a conversation delete purges them with the other conversation-keyed memory
  * tables (`conversation-memory-purge.ts`).
+ *
+ * Under `memory.v3.poolLog.captureInput` (off by default) the turn's row is
+ * joined by the selector's exact input, for offline selector evaluation and
+ * training data: `memory_v3_pool_inputs` (one row per `(conversation,
+ * turn)`: the context strings the selector was given, the selector prompt's
+ * content hash, the gate reason, the keep-all flag, and the content hash of
+ * every pooled candidate's rendered text in pool order, aligned with
+ * `candidates_json`) and `memory_v3_pool_texts` (each rendered text once,
+ * keyed by that hash). `buildPoolInput` derives both from the same
+ * orchestrate result and turn as the pool record, and `writePoolInput` runs
+ * in `writeTurnLog`'s transaction with the pool and selection rows. The
+ * input row is purged with the conversation; the texts are page-derived and
+ * content-keyed, so they are not.
  */
 
+import { createHash } from "node:crypto";
+
 import { getLogger } from "../logging.js";
 import type { MemorySqlite } from "../memory-db.js";
 import type { OrchestrateResult } from "./orchestrate.js";
 import {
   ensuredMemorySqlite,
+  ensureMemoryV3PoolInputsSchemaOnce,
   ensureMemoryV3PoolsSchemaOnce,
   memoryReader,
 } from "./plugin-schema.js";
-import { type FinderLane, sectionKey, type Slug } from "./types.js";
+import { renderFinderLine } from "./pool-select.js";
+import {
+  type FinderLane,
+  type MemoryRoutingTurn,
+  sectionKey,
+  type Slug,
+} from "./types.js";
 
 const log = getLogger("memory-v3-pool-log");
 
@@ -84,30 +106,39 @@ interface StoredPool {
 }
 
 /**
- * Build the turn's pool record from an orchestrate result. The stable prefix
- * comes first in cache order (core, hot, fresh, always-candidate) as
- * whole-page cards with no section; the finder tail follows in surfacing
- * order, each line tagged with the lane that surfaced it and the section it
- * carries. A finder hit on a stable-prefix page therefore appears twice, and
- * a page hit on several sections once per section, exactly as the selector
- * saw it. A card or a section-less line reads chosen when its page was kept;
+ * Build the turn's pool record from the exact selector pool carried by the
+ * orchestrate result. The stable prefix comes first in retained cache order as
+ * whole-page cards with no section; the finder tail follows in retained
+ * surfacing order, each line tagged with the lane that surfaced it and the
+ * section it carries. A finder hit on a stable-prefix page therefore appears
+ * twice, and a page hit on several sections once per section, exactly as the
+ * selector saw it. A card or a section-less line reads chosen when its page
+ * was kept;
  * a line carrying a section reads chosen when that section was selected.
  *
- * When the selector did not run and nothing was selected, no pool reached it:
- * either the injection gate hard-skipped selection (the result's lanes still
- * carry the stable prefix, which the selector never saw) or the pool was
- * empty. The record is then empty rather than a list of candidates marked
- * unchosen, which would read as a rejection the selector never made. The
- * disabled-selector passthrough keeps every pooled candidate as a selection,
- * so it still records its pool.
+ * A result without `pool` records no candidates: the injection gate either
+ * hard-skipped selection or no pool was assembled. A failed selector attempt
+ * still carries the exact pool it received, with retained stable cards chosen
+ * by the unjudged fallback and finder lines unchosen. The disabled-selector
+ * passthrough also carries and records its pool.
  */
 export function buildPoolRecord(result: OrchestrateResult): PoolRecord {
-  if (!result.selectorRan && result.selections.length === 0) {
+  const pool = result.pool;
+  if (pool === undefined) {
+    if (result.selections.length > 0) {
+      log.warn(
+        {
+          selectorRan: result.selectorRan,
+          selectedCount: result.selections.length,
+        },
+        "memory-v3 pool record: result has selections but no selector pool",
+      );
+    }
     return {
       candidates: [],
       pool_size: 0,
-      selected_count: 0,
-      selector_ran: false,
+      selected_count: result.selections.length,
+      selector_ran: result.selectorRan,
     };
   }
   const selected = new Map<Slug, Set<string>>(
@@ -116,30 +147,27 @@ export function buildPoolRecord(result: OrchestrateResult): PoolRecord {
       new Set(s.sections.map((section) => sectionKey(section))),
     ]),
   );
-  const card = (slug: Slug, lane: PoolLane): PoolCandidateRecord => ({
-    slug,
-    lane,
-    section_title: null,
-    section_key: null,
-    chosen: selected.has(slug),
-  });
-  const { core, hot, fresh, always, finder } = result.lanes;
   const candidates: PoolCandidateRecord[] = [
-    ...core.map((slug) => card(slug, "core")),
-    ...hot.map((slug) => card(slug, "hot")),
-    ...fresh.map((slug) => card(slug, "fresh")),
-    ...always.map((slug) => card(slug, "always")),
-    ...finder.map(({ slug, lane, section }): PoolCandidateRecord => {
-      const key = section ? sectionKey(section) : null;
+    ...pool.stable.map(
+      (candidate): PoolCandidateRecord => ({
+        slug: candidate.slug,
+        lane: candidate.lane,
+        section_title: null,
+        section_key: null,
+        chosen: selected.has(candidate.slug),
+      }),
+    ),
+    ...pool.finder.map((candidate): PoolCandidateRecord => {
+      const key = candidate.section ? sectionKey(candidate.section) : null;
       return {
-        slug,
-        lane,
-        section_title: section?.title ?? null,
+        slug: candidate.slug,
+        lane: candidate.lane,
+        section_title: candidate.section?.title ?? null,
         section_key: key,
         chosen:
           key === null
-            ? selected.has(slug)
-            : (selected.get(slug)?.has(key) ?? false),
+            ? selected.has(candidate.slug)
+            : (selected.get(candidate.slug)?.has(key) ?? false),
       };
     }),
   ];
@@ -289,3 +317,213 @@ export function readPoolForMessageIds(messageIds: string[]): StoredPool | null {
         .get(...messageIds) as PoolRow | null,
   );
 }
+
+/** Content hash of a rendered candidate text or a selector prompt: the key
+ *  of `memory_v3_pool_texts`. */
+export function hashPoolText(text: string): string {
+  return createHash("sha256").update(text).digest("hex");
+}
+
+/** The selector's exact input for one turn, persisted under
+ *  `memory.v3.poolLog.captureInput` beside the turn's pool record. */
+export interface PoolInputRecord {
+  situational_context: string | null;
+  recent_context: string;
+  current_message: string;
+  previous_assistant_message: string | null;
+  /** Content hash of the selector system prompt the turn ran with; null
+   *  when the selector was not run. */
+  selector_prompt_hash: string | null;
+  /** Whether the selector's recall-safe keep-all fallback fired. */
+  kept_all: boolean;
+  /** The injection gate's reason code; null when the gate did not run. */
+  gate_reason: string | null;
+  /** Content hash of each pooled candidate's rendered text, in pool order:
+   *  index `i` describes the turn's pool record candidate `i`, which the
+   *  selector saw under pool number `i + 1`. Empty when the record is, and
+   *  when the texts could not be captured (a consumer zips by index and
+   *  treats a length mismatch as texts unavailable). */
+  candidate_text_hashes: string[];
+}
+
+/** A turn's input record with the rendered texts its hashes name. */
+export interface PoolInputCapture {
+  input: PoolInputRecord;
+  texts: Map<string, string>;
+}
+
+/**
+ * Build the turn's input capture from an orchestrate result and the turn it
+ * ran on. When selector budgeting trimmed the context, the result's
+ * `selectorTurn` is the source of truth. Each candidate text is exactly what
+ * the selector was shown minus its pool number: a stable-prefix candidate's
+ * pre-rendered card, a finder candidate's rendered line
+ * ({@link renderFinderLine}), in the pool
+ * record's order ({@link buildPoolRecord}). A result that carries no selector
+ * pool captures the turn context and no candidate texts, so the hashes stay
+ * aligned with the empty record. Failed attempts carry their exact attempted
+ * pool and trimmed context even though `selectorRan` is false.
+ */
+export function buildPoolInput(
+  result: OrchestrateResult,
+  turn: MemoryRoutingTurn,
+  selectorPrompt: string | undefined,
+): PoolInputCapture {
+  const texts = new Map<string, string>();
+  const hashes: string[] = [];
+  if (result.pool !== undefined) {
+    const rendered = [
+      ...result.pool.stable.map((candidate) => candidate.card),
+      ...result.pool.finder.map((candidate) => renderFinderLine(candidate)),
+    ];
+    for (const text of rendered) {
+      const hash = hashPoolText(text);
+      hashes.push(hash);
+      texts.set(hash, text);
+    }
+  }
+  const selectorTurn = result.selectorTurn ?? turn;
+  return {
+    input: {
+      situational_context: selectorTurn.situationalContext ?? null,
+      recent_context: selectorTurn.recentContext,
+      current_message: selectorTurn.currentMessage,
+      previous_assistant_message: selectorTurn.previousAssistantMessage ?? null,
+      selector_prompt_hash:
+        result.selectorRan && selectorPrompt !== undefined
+          ? hashPoolText(selectorPrompt)
+          : null,
+      kept_all: result.keptAll ?? false,
+      gate_reason: result.gateReason ?? null,
+      candidate_text_hashes: hashes,
+    },
+    texts,
+  };
+}
+
+/**
+ * Write the turn's input row and its candidate texts on `raw`, their tables
+ * ensured first (`plugin-schema.ts`). The row's PK is `(conversation_id,
+ * turn)`, so a re-observed turn overwrites it in step with its pool row;
+ * each text is inserted by hash and ignored when already stored. Throws on
+ * a failed statement: `writeTurnLog` in `shadow-plugin.ts` runs this inside
+ * the transaction that writes the turn's pool and selection rows, and owns
+ * the best-effort boundary around it.
+ */
+export function writePoolInput(
+  raw: MemorySqlite,
+  conversationId: string,
+  turn: number,
+  capture: PoolInputCapture,
+): void {
+  ensureMemoryV3PoolInputsSchemaOnce(raw);
+  const now = Date.now();
+  const { input, texts } = capture;
+  raw
+    .query(
+      /*sql*/ `
+      INSERT OR REPLACE INTO memory_v3_pool_inputs (
+        conversation_id, turn, created_at, situational_context, recent_context,
+        current_message, previous_assistant_message, selector_prompt_hash,
+        kept_all, gate_reason, candidate_text_hashes_json
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `,
+    )
+    .run(
+      conversationId,
+      turn,
+      now,
+      input.situational_context,
+      input.recent_context,
+      input.current_message,
+      input.previous_assistant_message,
+      input.selector_prompt_hash,
+      input.kept_all ? 1 : 0,
+      input.gate_reason,
+      JSON.stringify(input.candidate_text_hashes),
+    );
+  const insertText = raw.query(/*sql*/ `
+      INSERT OR IGNORE INTO memory_v3_pool_texts (text_hash, text, created_at)
+      VALUES (?, ?, ?)
+    `);
+  for (const [hash, text] of texts) {
+    insertText.run(hash, text, now);
+  }
+}
+
+interface PoolInputRow {
+  situational_context: string | null;
+  recent_context: string;
+  current_message: string;
+  previous_assistant_message: string | null;
+  selector_prompt_hash: string | null;
+  kept_all: number;
+  gate_reason: string | null;
+  candidate_text_hashes_json: string;
+}
+
+/** Best-effort read over the capture tables, degrading to the fallback when
+ *  the memory connection is unavailable or a statement fails
+ *  (`plugin-schema.ts`). */
+const readPoolInputOr = memoryReader(
+  (context) => ensuredMemorySqlite(context, ensureMemoryV3PoolInputsSchemaOnce),
+  (err, context) =>
+    log.warn(
+      { err, context },
+      "failed to read memory-v3 pool input; treating it as unrecorded",
+    ),
+);
+
+/**
+ * Read the input record for an exact `(conversation, turn)`. `null` when the
+ * turn has none (it ran with the capture off or predates it) or the read
+ * degraded.
+ */
+export function readPoolInputForTurn(
+  conversationId: string,
+  turn: number,
+): PoolInputRecord | null {
+  return readPoolInputOr("readPoolInputForTurn", null, (raw) => {
+    const row = raw
+      .query(
+        /*sql*/ `
+        SELECT situational_context, recent_context, current_message,
+               previous_assistant_message, selector_prompt_hash, kept_all,
+               gate_reason, candidate_text_hashes_json
+        FROM memory_v3_pool_inputs
+        WHERE conversation_id = ? AND turn = ?
+      `,
+      )
+      .get(conversationId, turn) as PoolInputRow | null;
+    if (!row) {
+      return null;
+    }
+    const hashes: unknown = JSON.parse(row.candidate_text_hashes_json);
+    if (!Array.isArray(hashes)) {
+      throw new Error("candidate_text_hashes_json is not an array");
+    }
+    return {
+      situational_context: row.situational_context,
+      recent_context: row.recent_context,
+      current_message: row.current_message,
+      previous_assistant_message: row.previous_assistant_message,
+      selector_prompt_hash: row.selector_prompt_hash,
+      kept_all: row.kept_all === 1,
+      gate_reason: row.gate_reason,
+      candidate_text_hashes: hashes as string[],
+    };
+  });
+}
+
+/** The rendered text stored under `textHash`, or `null` when none is or the
+ *  read degraded. */
+export function readPoolText(textHash: string): string | null {
+  return readPoolInputOr("readPoolText", null, (raw) => {
+    const row = raw
+      .query(
+        /*sql*/ `SELECT text FROM memory_v3_pool_texts WHERE text_hash = ?`,
+      )
+      .get(textHash) as { text: string } | null;
+    return row?.text ?? null;
+  });
+}

package/src/plugins/defaults/memory/v3/pool-select.test.ts

@@ -29,10 +29,14 @@ const realPluginApi = await import("@vellumai/plugin-api");
 
 let selectMockActive = false;
 let sendMessageImpl: (() => Promise<ProviderResponse>) | null = null;
+// The messages of the most recent selector call, for asserting the rendered
+// pool.
+let lastMessages: unknown = null;
 
 const mockProvider = {
   name: "mock-memory-v3-selector",
-  async sendMessage(): Promise<ProviderResponse> {
+  async sendMessage(messages: unknown): Promise<ProviderResponse> {
+    lastMessages = messages;
     if (!sendMessageImpl) {
       throw new Error("sendMessageImpl not configured");
     }
@@ -50,7 +54,7 @@ mock.module("@vellumai/plugin-api", () => ({
       : realPluginApi.getConfiguredProvider(...args),
 }));
 
-const { MemoryV3RetrievalUnavailableError, selectPool } =
+const { MemoryV3RetrievalUnavailableError, renderFinderLine, selectPool } =
   await import("./pool-select.js");
 
 function response(content: ContentBlock[]): ProviderResponse {
@@ -68,9 +72,11 @@ const turn: MemoryRoutingTurn = {
   recentContext: "",
 };
 
-const pool = {
+const pool: Parameters<typeof selectPool>[0] = {
   stable: [],
-  finder: [{ slug: "page-a" as Slug, descriptor: "a descriptor" }],
+  finder: [
+    { slug: "page-a" as Slug, descriptor: "a descriptor", lane: "needle" },
+  ],
 };
 
 describe("selectPool", () => {
@@ -129,6 +135,8 @@ describe("selectPool", () => {
     expect(await selectPool(pool, turn)).toEqual({
       pages: [{ slug: "page-a", sections: [] }],
       keptAll: false,
+      pool,
+      turn,
     });
   });
 
@@ -140,13 +148,47 @@ describe("selectPool", () => {
     expect(await selectPool(pool, turn)).toEqual({
       pages: [{ slug: "page-a", sections: [] }],
       keptAll: true,
+      pool,
+      turn,
     });
   });
 
+  test("a finder line is shown as its pool number and renderFinderLine", async () => {
+    sendMessageImpl = async () =>
+      response([
+        {
+          type: "tool_use",
+          id: "call-1",
+          name: "select_pages",
+          input: { ids: [] },
+        },
+      ]);
+    const candidate: Parameters<typeof renderFinderLine>[0] = {
+      slug: "page-a" as Slug,
+      descriptor: "a descriptor",
+      lane: "needle",
+    };
+    await selectPool(
+      {
+        stable: [{ slug: "page-s" as Slug, card: "card s", lane: "core" }],
+        finder: [candidate],
+      },
+      turn,
+    );
+    const sent = JSON.stringify(lastMessages);
+    expect(sent).toContain(
+      JSON.stringify(`[2] ${renderFinderLine(candidate)}`).slice(1, -1),
+    );
+    expect(sent).toContain(JSON.stringify("[1] card s").slice(1, -1));
+  });
+
   test("an empty candidate pool returns no selections", async () => {
-    expect(await selectPool({ stable: [], finder: [] }, turn)).toEqual({
+    const emptyPool = { stable: [], finder: [] };
+    expect(await selectPool(emptyPool, turn)).toEqual({
       pages: [],
       keptAll: false,
+      pool: emptyPool,
+      turn,
     });
   });
 });