@vellumai/assistant 0.8.6 → 0.8.7-dev.202606052118.34cd356

package/src/memory/v3/filter.ts

@@ -1,286 +0,0 @@
-/**
- * Memory v3 — fast dense-hit filter.
- *
- * The dense scout lane surfaces embedding-similarity candidates that span
- * subtrees: some are meaningful cross-domain associations worth carrying into
- * the gate, others are spurious near-neighbors that only crowd the slate. This
- * module makes **one cheap LLM call** to keep the meaningful associations and
- * drop the noise, *before* the more expensive selection gate runs.
- *
- * What it judges. Only the bounded dense candidate set (the scout lane is
- * already capped at ~50–200 by quota/MMR — the filter never sees the whole
- * corpus). Hot pages and near-exact sparse hits arrive via the scouts'
- * `sticky` / `bypass` sets and are **never judged**: a literal keyword hit or a
- * page the user has been touching is a strong enough signal that we shouldn't
- * make it earn its place through a fallible cheap judgment, and the downstream
- * gate force-injects every sticky slug regardless — judging it could not change
- * its fate. The `bypass` subset is additionally unioned straight into `kept`.
- *
- * Fail-open. If no provider is configured or the call errors / returns an
- * unusable response, the filter keeps *all* judged dense candidates and surfaces
- * a `failureReason` so the loop can record that the filter was bypassed.
- * Dropping candidates on a model outage would silently starve retrieval; keeping
- * them is the safe degradation (the downstream gate still narrows the slate).
- *
- * No LLM call when there is nothing to judge. A dense set fully covered by
- * sticky short-circuits to `kept` = the bypass-relevant slugs (no judged
- * additions), with no provider round-trip.
- */
-
-import { z } from "zod";
-
-import {
-  extractToolUse,
-  getConfiguredProvider,
-} from "../../providers/provider-send-message.js";
-import type {
-  Message,
-  Provider,
-  ToolDefinition,
-} from "../../providers/types.js";
-import { getLogger } from "../../util/logger.js";
-import type { RetrievalInput } from "../v2/harness/retriever.js";
-import type { ScoutResult } from "../v2/harness/trace.js";
-import type { LlmCallSink } from "./llm-capture.js";
-import { renderConversationContext } from "./prompt-context.js";
-import {
-  FILTER_SYSTEM_PROMPT,
-  resolveV3SystemPrompt,
-} from "./prompts/system-prompts.js";
-
-const log = getLogger("memory-v3-filter");
-
-/** Tool name forced via `tool_choice`. Shared constant so tests can match it. */
-const FILTER_TOOL_NAME = "filter_dense_hits";
-
-/**
- * Arguments to one filter invocation.
- *
- * `dense` is the bounded dense scout result; only its slugs that are *not*
- * already in `sticky` are judged. `sticky` is the keep-in-the-running set (hot +
- * near-exact sparse) the downstream gate force-injects regardless of this
- * filter, so judging a sticky page wastes an LLM call that can never change its
- * fate. `bypass` is the subset of sticky strong enough to skip judgment that the
- * filter also unions straight into `kept`. Sticky slugs that also appear in the
- * dense lane are excluded from the judged set and never sent to the model.
- */
-export interface FilterDenseHitsArgs {
-  input: RetrievalInput;
-  dense: ScoutResult;
-  sticky: Set<string>;
-  bypass: Set<string>;
-  /** Optional debug sink — emits one record for the filter's LLM call. */
-  capture?: LlmCallSink;
-  /**
-   * Provider override seam for tests. Production leaves this unset and the
-   * filter resolves `getConfiguredProvider("memoryV3Filter")`. `null` is
-   * distinct from `undefined`: passing `null` simulates "no provider
-   * configured" and exercises the fail-open path without resolving the real
-   * registry.
-   */
-  provider?: Provider | null;
-}
-
-export interface FilterDenseHitsResult {
-  /** Final kept slugs: bypass ∪ judged-kept. */
-  kept: string[];
-  /** Inspection trace: which dense slugs were judged and which were dropped. */
-  trace: { judged: string[]; dropped: string[] };
-  /**
-   * Non-null when the filter could not judge (no provider, provider throw,
-   * missing tool_use, schema mismatch) and therefore failed open by keeping all
-   * dense candidates. The loop can surface this to flag a bypassed filter.
-   */
-  failureReason?: string;
-}
-
-/**
- * Build the forced tool definition. `keep_slugs` is the model's subset of the
- * judged candidate set to retain; everything judged-but-not-kept is dropped.
- * Mirrors the forced-tool pattern of v2's `select_pages_to_inject`.
- */
-function buildFilterTool(judgedSlugs: readonly string[]): ToolDefinition {
-  return {
-    name: FILTER_TOOL_NAME,
-    description:
-      "From the candidate concept pages surfaced by embedding similarity for " +
-      "the current turn, keep the ones that are meaningful associations worth " +
-      "surfacing and drop the spurious near-neighbors. Return keep_slugs as the " +
-      "subset to retain — choose only from the candidate set. Lean toward " +
-      "keeping a plausible cross-domain association over dropping it.",
-    input_schema: {
-      type: "object",
-      properties: {
-        keep_slugs: {
-          type: "array",
-          items: { type: "string", enum: [...judgedSlugs] },
-          description:
-            "The subset of candidate page slugs to keep. Choose only from the candidate set.",
-        },
-        reasoning: {
-          type: "string",
-          description:
-            "One short sentence: why these hits were kept and the rest dropped.",
-        },
-      },
-      required: ["keep_slugs"],
-    },
-  };
-}
-
-const FilterToolResultSchema = z.object({
-  keep_slugs: z.array(z.string()),
-  reasoning: z.string().optional(),
-});
-
-/**
- * Compose the final result. `kept` = bypass slugs ∪ judged-kept (de-duplicated,
- * bypass first then judged-kept in the model's returned order). `trace` records
- * exactly which dense slugs were judged and which the model dropped.
- */
-function buildResult(
-  bypass: Set<string>,
-  judged: readonly string[],
-  judgedKept: readonly string[],
-  failureReason?: string,
-): FilterDenseHitsResult {
-  const keptSet = new Set<string>(bypass);
-  const kept: string[] = [...bypass];
-  for (const slug of judgedKept) {
-    if (keptSet.has(slug)) continue;
-    keptSet.add(slug);
-    kept.push(slug);
-  }
-  const keptJudged = new Set(judgedKept);
-  const dropped = judged.filter((slug) => !keptJudged.has(slug));
-  return {
-    kept,
-    trace: { judged: [...judged], dropped },
-    ...(failureReason !== undefined ? { failureReason } : {}),
-  };
-}
-
-/**
- * Run the fast dense-hit filter for one pass.
- *
- * Makes at most one forced-tool LLM call over the *judged* set (dense slugs not
- * already in `sticky`). Sticky slugs are force-selected by the downstream gate
- * regardless of this filter, so they are excluded from judgment; bypass slugs
- * are additionally kept unconditionally here. On an empty judged set no call is
- * made. Any failure (no provider, provider throw, missing tool_use, schema
- * mismatch) fails open: every judged dense candidate is kept and a
- * `failureReason` is returned.
- */
-export async function filterDenseHits(
-  args: FilterDenseHitsArgs,
-): Promise<FilterDenseHitsResult> {
-  const { input, dense, sticky, bypass } = args;
-
-  // Sticky slugs (hot + near-exact sparse) are force-selected by the gate
-  // regardless of this filter, so judging them wastes an LLM call that can't
-  // change their fate. Exclude the full sticky set (a superset of bypass) from
-  // the judged set; only the remaining dense near-neighbors are judged.
-  const judged = dense.slugs.filter((slug) => !sticky.has(slug));
-
-  // Nothing to judge → no LLM call. Kept is just the bypass-relevant slugs.
-  if (judged.length === 0) {
-    return buildResult(bypass, judged, judged);
-  }
-
-  // Resolve the provider. A `provider` key in args (including explicit `null`)
-  // takes precedence so tests inject a stub; production omits it and resolves
-  // the configured `memoryV3Filter` call site.
-  const provider =
-    args.provider !== undefined
-      ? args.provider
-      : await getConfiguredProvider("memoryV3Filter");
-
-  if (!provider) {
-    log.warn(
-      "memoryV3Filter provider unavailable; failing open (keeping all dense)",
-    );
-    return buildResult(bypass, judged, judged, "no_provider");
-  }
-
-  const systemPrompt = resolveV3SystemPrompt(
-    FILTER_SYSTEM_PROMPT,
-    input.config.memory?.v3?.prompts?.filter,
-    input.workspaceDir,
-  );
-
-  const userMsg: Message = {
-    role: "user",
-    content: [
-      {
-        type: "text",
-        text: renderConversationContext(input),
-      },
-      {
-        type: "text",
-        text: `<candidate_slugs>\n${judged.join("\n")}\n</candidate_slugs>`,
-      },
-    ],
-  };
-
-  const filterTool = buildFilterTool(judged);
-
-  const startedAt = Date.now();
-  let response;
-  try {
-    response = await provider.sendMessage(
-      [userMsg],
-      [filterTool],
-      systemPrompt,
-      {
-        config: {
-          callSite: "memoryV3Filter" as const,
-          tool_choice: { type: "tool" as const, name: FILTER_TOOL_NAME },
-        },
-        ...(input.signal ? { signal: input.signal } : {}),
-      },
-    );
-  } catch (err) {
-    log.warn({ err }, "Filter provider call threw; failing open (keep all)");
-    return buildResult(bypass, judged, judged, "api_error");
-  }
-
-  args.capture?.({
-    lane: "filter",
-    callSite: "memoryV3Filter",
-    request: { systemPrompt, messages: [userMsg], tools: [filterTool] },
-    response,
-    ms: Date.now() - startedAt,
-  });
-
-  const toolBlock = extractToolUse(response);
-  if (!toolBlock || toolBlock.name !== FILTER_TOOL_NAME) {
-    log.warn(
-      { stopReason: response.stopReason },
-      "Filter model returned no filter_dense_hits tool_use; failing open (keep all)",
-    );
-    return buildResult(bypass, judged, judged, "tool_use_missing");
-  }
-
-  const parsed = FilterToolResultSchema.safeParse(toolBlock.input);
-  if (!parsed.success) {
-    log.warn(
-      { error: parsed.error.message },
-      "Filter tool input did not match schema; failing open (keep all)",
-    );
-    return buildResult(bypass, judged, judged, "schema_mismatch");
-  }
-
-  // Restrict the model's keep set to the judged candidates (it can only keep
-  // what it was shown) and preserve its returned order.
-  const judgedSet = new Set(judged);
-  const seen = new Set<string>();
-  const judgedKept: string[] = [];
-  for (const slug of parsed.data.keep_slugs) {
-    if (!judgedSet.has(slug)) continue;
-    if (seen.has(slug)) continue;
-    seen.add(slug);
-    judgedKept.push(slug);
-  }
-
-  return buildResult(bypass, judged, judgedKept);
-}

package/src/memory/v3/gate.ts

@@ -1,349 +0,0 @@
-/**
- * Memory v3 — selection gate.
- *
- * The gate is the final step of one retrieval pass. After the scouts, the tree
- * walk, the edge expansion, and the sticky carry-over have each contributed
- * candidate page slugs, the gate makes one capable LLM call over the *unioned*
- * candidate set and decides:
- *
- *   - **ready** — finalize the selection and inject for the next reply, or
- *   - **more**  — the candidates don't yet cover the turn; emit follow-up
- *     questions that seed the next pass. These questions are the gate's own
- *     *generated* queries (a refined sub-question), NOT a replay of the
- *     original user message — the loop feeds them back to the scouts/tree on
- *     the next iteration.
- *
- * The gate also returns the final ordered `selectedSlugs` (the order the model
- * returned, with sticky slugs guaranteed present). Sticky pages are never
- * dropped: they were injected on a prior turn and removing them mid-conversation
- * would silently amnesia the assistant, so we union them back in even when the
- * model omits them.
- *
- * Scope — brief generation is deferred. The full v3 design pairs the selection
- * with a ~1000-token voice brief, but that brief is only consumed when v3 is
- * actually injected (a later cutover). In shadow mode the harness injects v2
- * and only compares selections, so this module produces the selection +
- * `GateDecision` only — matching what the harness trace already models. The
- * brief-generation seam is marked below; do not build voice synthesis here.
- *
- * Fail-safe. If no provider is configured or the provider call errors/returns
- * an unusable response, the gate fails *open*: it returns
- * `decision: { decision: "ready" }` and selects every candidate. A retrieval
- * loop that can't reach the model should still inject what it found rather than
- * inject nothing.
- *
- * This module is currently unwired — a later PR composes it into the loop.
- */
-
-import { z } from "zod";
-
-import {
-  extractToolUse,
-  getConfiguredProvider,
-} from "../../providers/provider-send-message.js";
-import type {
-  Message,
-  Provider,
-  ToolDefinition,
-} from "../../providers/types.js";
-import { getLogger } from "../../util/logger.js";
-import type { RetrievalInput } from "../v2/harness/retriever.js";
-import type { GateDecision } from "../v2/harness/trace.js";
-import type { LlmCallSink } from "./llm-capture.js";
-import { renderConversationContext } from "./prompt-context.js";
-import {
-  GATE_SYSTEM_PROMPT,
-  resolveV3SystemPrompt,
-} from "./prompts/system-prompts.js";
-
-const log = getLogger("memory-v3-gate");
-
-/** Tool name forced via `tool_choice`. Shared constant so tests can match it. */
-const GATE_TOOL_NAME = "decide_selection";
-
-/**
- * Arguments to one gate invocation.
- *
- * `candidates` is the accumulated candidate set for this pass — the union of
- * scouts-kept, tree pages, edge-pulled, and sticky slugs. `sticky` is the
- * subset that was injected on a prior turn and must survive: it is always a
- * subset of `candidates` in practice, but the gate unions it back into both
- * the prompt and the final selection defensively.
- */
-export interface RunGateArgs {
-  input: RetrievalInput;
-  candidates: Set<string>;
-  sticky: Set<string>;
-  passNumber: number;
-  /**
-   * Per-candidate one-line summaries, keyed by slug. When present, candidates
-   * are rendered to the model as `slug — summary` so the gate can judge
-   * relevance on page content rather than the slug alone. Missing entries fall
-   * back to the bare slug; the model answers with the candidate's `[N]` number
-   * (`selected_ids`) regardless, so summaries never change the answer format. The
-   * loop passes this only when `memory.v3.gateCandidateSummaries` is set.
-   */
-  summaryBySlug?: ReadonlyMap<string, string>;
-  /** Optional debug sink — emits one record for the gate's LLM call. */
-  capture?: LlmCallSink;
-  /**
-   * Provider override seam for tests. Production leaves this unset and the
-   * gate resolves `getConfiguredProvider("memoryV3Gate")`. `null` is distinct
-   * from `undefined`: passing `null` simulates "no provider configured" and
-   * exercises the fail-safe path without resolving the real registry.
-   */
-  provider?: Provider | null;
-}
-
-export interface RunGateResult {
-  decision: GateDecision;
-  /** Final page slugs in the model's returned order; sticky guaranteed present. */
-  selectedSlugs: string[];
-}
-
-/**
- * Build the forced tool definition. `selected_ids` is the list of bracketed
- * `[N]` candidate numbers to keep; `decision` is the ready/more verdict;
- * `questions` carries the generated follow-up queries on "more" (ignored on
- * "ready"). The schema is candidate-independent — no per-turn enum — so it stays
- * byte-identical across turns (cache-friendly) and the model answers with the
- * line numbers rather than reproducing exact slug strings, mirroring the
- * integer-id output of v2's `select_pages_to_inject`.
- */
-function buildGateTool(): ToolDefinition {
-  return {
-    name: GATE_TOOL_NAME,
-    description:
-      "Decide whether the accumulated candidate pages are sufficient to answer " +
-      "the next turn. Return decision='ready' with the final selection " +
-      "when the candidates cover the turn; return decision='more' with one or " +
-      "more generated follow-up questions (NOT the original message) to seed " +
-      "another retrieval pass when coverage is incomplete.",
-    input_schema: {
-      type: "object",
-      properties: {
-        decision: { type: "string", enum: ["ready", "more"] },
-        selected_ids: {
-          type: "array",
-          items: { type: "integer" },
-          description:
-            "The bracketed `[N]` numbers of the candidate pages to keep. Prefer " +
-            "keeping a plausibly-relevant page over dropping it; for a list / " +
-            "'all of X' / breadth request, include every candidate that plausibly " +
-            "applies rather than trimming to the most prominent few.",
-        },
-        questions: {
-          type: "array",
-          items: { type: "string" },
-          description:
-            "When decision='more', the generated follow-up questions seeding the next pass.",
-        },
-        reasoning: {
-          type: "string",
-          description:
-            "One short sentence: why this ready/more verdict, and which " +
-            "candidates were kept or dropped.",
-        },
-      },
-      required: ["decision", "selected_ids"],
-    },
-  };
-}
-
-const GateToolResultSchema = z.object({
-  decision: z.enum(["ready", "more"]),
-  selected_ids: z.array(z.coerce.number()).optional(),
-  questions: z.array(z.string()).optional(),
-  reasoning: z.string().optional(),
-});
-
-/**
- * Render the candidate list for the prompt. Each line is `[N] slug — summary`
- * (or `[N] slug` without a summary), numbered from 1 in candidate order. The
- * model answers with the `[N]` numbers via `selected_ids`, which the caller maps
- * back to slugs by index — so the tool schema stays candidate-independent.
- */
-function renderCandidateLines(
-  slugs: readonly string[],
-  summaryBySlug: ReadonlyMap<string, string> | undefined,
-): string {
-  return slugs
-    .map((slug, i) => {
-      const summary = summaryBySlug?.get(slug);
-      const body = summary ? `${slug} — ${summary}` : slug;
-      return `[${i + 1}] ${body}`;
-    })
-    .join("\n");
-}
-
-/**
- * Order a slug selection: keep the model's returned order, restricted to the
- * candidate set, then append any sticky slugs the model omitted (sticky is
- * never dropped). De-duplicates while preserving first-seen order.
- */
-function orderSelection(
-  modelSlugs: readonly string[],
-  candidates: Set<string>,
-  sticky: Set<string>,
-): string[] {
-  const seen = new Set<string>();
-  const out: string[] = [];
-  for (const slug of modelSlugs) {
-    if (!candidates.has(slug)) continue; // model can only pick from candidates
-    if (seen.has(slug)) continue;
-    seen.add(slug);
-    out.push(slug);
-  }
-  for (const slug of sticky) {
-    if (seen.has(slug)) continue;
-    seen.add(slug);
-    out.push(slug);
-  }
-  return out;
-}
-
-/**
- * Fail-safe result: inject every candidate and declare the pass ready. Used
- * when the provider is unavailable or the call cannot produce a usable
- * decision. Ordering puts sticky last via `orderSelection` with an empty
- * model selection, so candidates come first then any sticky not already in
- * the set.
- */
-function failSafe(candidates: Set<string>, sticky: Set<string>): RunGateResult {
-  return {
-    decision: { decision: "ready" },
-    selectedSlugs: orderSelection([...candidates], candidates, sticky),
-  };
-}
-
-/**
- * Run the gate for one pass.
- *
- * Makes one forced-tool LLM call over the candidate set and maps the result to
- * a `GateDecision` plus the final ordered selection. Sticky slugs are always
- * present in the selection. Any failure (no provider, provider throw, missing
- * tool_use, schema mismatch) falls back to selecting all candidates with a
- * "ready" decision.
- */
-export async function runGate(args: RunGateArgs): Promise<RunGateResult> {
-  const { input, candidates, sticky, passNumber } = args;
-
-  const candidateSlugs = [...candidates];
-
-  // Resolve the provider. A `provider` key in args (including explicit `null`)
-  // takes precedence so tests inject a stub; production omits it and resolves
-  // the configured `memoryV3Gate` call site.
-  const provider =
-    args.provider !== undefined
-      ? args.provider
-      : await getConfiguredProvider("memoryV3Gate");
-
-  if (!provider) {
-    log.warn("memoryV3Gate provider unavailable; gate failing open (ready)");
-    return failSafe(candidates, sticky);
-  }
-
-  const systemPrompt = resolveV3SystemPrompt(
-    GATE_SYSTEM_PROMPT,
-    input.config.memory?.v3?.prompts?.gate,
-    input.workspaceDir,
-  );
-
-  const stickySlugs = [...sticky];
-  const userMsg: Message = {
-    role: "user",
-    content: [
-      {
-        type: "text",
-        text: renderConversationContext(input),
-      },
-      {
-        type: "text",
-        text:
-          `<pass_number>${passNumber}</pass_number>\n\n` +
-          `<sticky_slugs>\n${stickySlugs.join("\n")}\n</sticky_slugs>\n\n` +
-          `<candidates>\n` +
-          `${renderCandidateLines(candidateSlugs, args.summaryBySlug)}\n` +
-          `</candidates>`,
-      },
-    ],
-  };
-
-  const gateTool = buildGateTool();
-
-  const startedAt = Date.now();
-  let response;
-  try {
-    response = await provider.sendMessage([userMsg], [gateTool], systemPrompt, {
-      config: {
-        callSite: "memoryV3Gate" as const,
-        tool_choice: { type: "tool" as const, name: GATE_TOOL_NAME },
-      },
-      ...(input.signal ? { signal: input.signal } : {}),
-    });
-  } catch (err) {
-    log.warn({ err }, "Gate provider call threw; failing open (ready)");
-    return failSafe(candidates, sticky);
-  }
-
-  args.capture?.({
-    lane: "gate",
-    callSite: "memoryV3Gate",
-    request: { systemPrompt, messages: [userMsg], tools: [gateTool] },
-    response,
-    ms: Date.now() - startedAt,
-  });
-
-  const toolBlock = extractToolUse(response);
-  if (!toolBlock || toolBlock.name !== GATE_TOOL_NAME) {
-    log.warn(
-      { stopReason: response.stopReason },
-      "Gate model returned no decide_selection tool_use; failing open (ready)",
-    );
-    return failSafe(candidates, sticky);
-  }
-
-  const parsed = GateToolResultSchema.safeParse(toolBlock.input);
-  if (!parsed.success) {
-    log.warn(
-      { error: parsed.error.message },
-      "Gate tool input did not match schema; failing open (ready)",
-    );
-    return failSafe(candidates, sticky);
-  }
-
-  // Recall-safe fallback: an *omitted* `selected_ids` means the model gave no
-  // instruction, so keep everything that was surfaced — dropping all non-sticky
-  // context on a silent omission is the worse failure. An *explicit* `[]` is the
-  // model genuinely choosing nothing and is honored as-is. Ids are 1-based
-  // indices into the rendered candidate list; out-of-range ids are dropped.
-  const modelSlugs =
-    parsed.data.selected_ids === undefined
-      ? [...candidates]
-      : parsed.data.selected_ids
-          .map((id) => candidateSlugs[Math.trunc(id) - 1])
-          .filter((slug): slug is string => slug !== undefined);
-  const selectedSlugs = orderSelection(modelSlugs, candidates, sticky);
-
-  const reasoning = parsed.data.reasoning?.trim() || undefined;
-
-  if (parsed.data.decision === "more") {
-    const questions = (parsed.data.questions ?? []).filter(
-      (q) => q.trim().length > 0,
-    );
-    const decision: GateDecision = {
-      decision: "more",
-      ...(questions.length > 0 ? { questions } : {}),
-      ...(reasoning ? { reasoning } : {}),
-    };
-    return { decision, selectedSlugs };
-  }
-
-  // brief generation lands at cutover (P5) — shadow mode injects v2, so this
-  // gate produces only the selection + decision. Do NOT synthesize a voice
-  // brief here.
-  return {
-    decision: { decision: "ready", ...(reasoning ? { reasoning } : {}) },
-    selectedSlugs,
-  };
-}