@vellumai/assistant 0.8.6 → 0.8.7

package/src/memory/v3/shadow-diff.ts

@@ -1,287 +0,0 @@
-/**
- * Memory v3 shadow-diff — compare the v3 shadow selection against the live v2
- * router selection, turn-for-turn, from the `memory_v2_activation_logs` table.
- *
- * When v3 runs in shadow mode it logs its per-turn selection as a `v3_shadow`
- * row while the live v2 router logs its own selection as a `router` row on the
- * same turn. This module pairs the two and reports where they agree, what v3
- * surfaced that v2 did not, and what v2 had that v3 dropped — broken down by the
- * v3 provenance lane so a shadow run is analyzable by where its recall comes
- * from.
- *
- * Pairing is by timestamp, NOT by the `turn` column: the shadow middleware logs
- * the orchestrator's per-runtime turn counter while v2 logs the cumulative
- * conversation turn, so the two numbers diverge. The shadow row and its sibling
- * router row are written within the same turn (a second or two apart), so each
- * shadow row is matched to the nearest router row in the same conversation
- * within a tolerance window.
- *
- * The v2 comparand is the router's FRESH per-turn pick (`status: "injected"`),
- * not its full in-context set. v2 accumulates pages across turns (`in_context`
- * carry-over can reach the hundreds on a long conversation) whereas v3 selects
- * fresh each turn; comparing fresh-against-fresh keeps the diff meaningful. The
- * carried-over count is surfaced per turn as context, not folded into the sets.
- *
- * Pure and DB-free: the route handler reads the rows and hands them here.
- */
-
-import type { MemoryV2ConceptRowRecord } from "../memory-v2-activation-log-store.js";
-
-/** An activation-log row reduced to what the diff needs. */
-export interface ShadowDiffLogRow {
-  conversationId: string;
-  /** Epoch milliseconds. */
-  createdAt: number;
-  concepts: MemoryV2ConceptRowRecord[];
-}
-
-/** One paired (v2 router ↔ v3 shadow) turn. */
-export interface ShadowDiffTurn {
-  conversationId: string;
-  /** Epoch ms of the v3 shadow row. */
-  shadowAt: number;
-  /** Epoch ms of the paired v2 router row. */
-  routerAt: number;
-  /** `routerAt - shadowAt`; small (within tolerance) by construction. */
-  deltaMs: number;
-  /** Size of the v2 fresh pick (`status: "injected"`). */
-  v2Count: number;
-  /** Size of the v3 shadow selection. */
-  v3Count: number;
-  /** v2 pages carried over from prior turns (`in_context`); annotation only. */
-  v2CachedCount: number;
-  /** `|overlap| / |v2 ∪ v3|`; 0 when both sets are empty. */
-  jaccard: number;
-  /** Slugs both systems picked, sorted. */
-  overlap: string[];
-  /** Slugs v3 surfaced but v2 did not freshly inject, sorted. */
-  v3Only: string[];
-  /** Slugs v2 freshly injected but v3 missed, sorted. */
-  v2Only: string[];
-  /** Provenance lane for each v3 slug (overlap + v3-only). */
-  laneBySlug: Record<string, string>;
-}
-
-/** A shadow row with no router row inside the tolerance window. */
-export interface UnpairedShadowTurn {
-  conversationId: string;
-  shadowAt: number;
-  v3Count: number;
-}
-
-/** A slug with how many paired turns it appeared in. */
-export interface SlugFrequency {
-  slug: string;
-  count: number;
-}
-
-export interface ShadowDiffResult {
-  /** Pairing tolerance actually used (ms). */
-  toleranceMs: number;
-  /** Total v3 shadow rows in the read window. */
-  shadowRows: number;
-  /** Shadow rows that paired to a router row. */
-  turnsCompared: number;
-  /** Shadow rows that did not pair. */
-  unpaired: UnpairedShadowTurn[];
-  agg: {
-    meanV2: number;
-    meanV3: number;
-    meanOverlap: number;
-    meanJaccard: number;
-    totalOverlap: number;
-    totalV3Only: number;
-    totalV2Only: number;
-    /** v3-only slug count by the lane that surfaced it — v3's extra reach. */
-    v3OnlyByLane: Record<string, number>;
-    /** overlap slug count by the v3 lane that recovered v2's pick. */
-    overlapByLane: Record<string, number>;
-    /** Most frequently dropped v2 pages (recall-regression watchlist). */
-    v2OnlyTop: SlugFrequency[];
-    /** Most frequent v3 extras (associative reach beyond v2). */
-    v3OnlyTop: SlugFrequency[];
-  };
-  /** Per-turn detail, newest first, capped at the requested limit. */
-  turns: ShadowDiffTurn[];
-}
-
-/** Status on a v2 router row that counts as a fresh per-turn selection. */
-const V2_PICKED_STATUS = "injected";
-/** Status on a v2 router row that means carried-over from a prior turn. */
-const V2_CACHED_STATUS = "in_context";
-/** How many slugs to list in the top-frequency aggregates. */
-const TOP_FREQUENCY_LIMIT = 15;
-
-function selectedV2Slugs(concepts: MemoryV2ConceptRowRecord[]): Set<string> {
-  const slugs = new Set<string>();
-  for (const c of concepts) {
-    if (c.status === V2_PICKED_STATUS) slugs.add(c.slug);
-  }
-  return slugs;
-}
-
-function cachedV2Count(concepts: MemoryV2ConceptRowRecord[]): number {
-  let n = 0;
-  for (const c of concepts) {
-    if (c.status === V2_CACHED_STATUS) n += 1;
-  }
-  return n;
-}
-
-function v3LaneBySlug(
-  concepts: MemoryV2ConceptRowRecord[],
-): Map<string, string> {
-  const bySlug = new Map<string, string>();
-  for (const c of concepts) {
-    bySlug.set(c.slug, c.lane ?? "unknown");
-  }
-  return bySlug;
-}
-
-/** Increment a string-keyed tally in place. */
-function bump(tally: Record<string, number>, key: string): void {
-  tally[key] = (tally[key] ?? 0) + 1;
-}
-
-/** Sort a frequency map into the top-N slugs, ties broken by slug name. */
-function topSlugs(freq: Map<string, number>, limit: number): SlugFrequency[] {
-  return [...freq.entries()]
-    .map(([slug, count]) => ({ slug, count }))
-    .sort((a, b) => b.count - a.count || a.slug.localeCompare(b.slug))
-    .slice(0, limit);
-}
-
-/**
- * Pair each shadow row to the nearest unconsumed router row in the same
- * conversation within `toleranceMs`, then diff the two selections. Pairing is
- * greedy by absolute time delta; since real turns are minutes apart while a
- * shadow/router sibling pair lands a second or two apart, the greedy match is a
- * clean bijection in practice.
- */
-export function computeShadowDiff(
-  shadow: readonly ShadowDiffLogRow[],
-  router: readonly ShadowDiffLogRow[],
-  opts: { toleranceMs: number; detailLimit: number },
-): ShadowDiffResult {
-  const { toleranceMs, detailLimit } = opts;
-
-  // Index router rows by conversation, time-sorted, with a consumed flag so a
-  // router row pairs to at most one shadow row.
-  const routerByConv = new Map<
-    string,
-    Array<{ row: ShadowDiffLogRow; consumed: boolean }>
-  >();
-  for (const row of router) {
-    const bucket = routerByConv.get(row.conversationId) ?? [];
-    bucket.push({ row, consumed: false });
-    routerByConv.set(row.conversationId, bucket);
-  }
-  for (const bucket of routerByConv.values()) {
-    bucket.sort((a, b) => a.row.createdAt - b.row.createdAt);
-  }
-
-  const sortedShadow = [...shadow].sort((a, b) => a.createdAt - b.createdAt);
-
-  const turns: ShadowDiffTurn[] = [];
-  const unpaired: UnpairedShadowTurn[] = [];
-  const v3OnlyByLane: Record<string, number> = {};
-  const overlapByLane: Record<string, number> = {};
-  const v2OnlyFreq = new Map<string, number>();
-  const v3OnlyFreq = new Map<string, number>();
-
-  for (const sh of sortedShadow) {
-    const bucket = routerByConv.get(sh.conversationId);
-    let best: { row: ShadowDiffLogRow; consumed: boolean } | undefined;
-    let bestDelta = Number.POSITIVE_INFINITY;
-    for (const candidate of bucket ?? []) {
-      if (candidate.consumed) continue;
-      const delta = Math.abs(candidate.row.createdAt - sh.createdAt);
-      if (delta < bestDelta) {
-        bestDelta = delta;
-        best = candidate;
-      }
-    }
-
-    if (!best || bestDelta > toleranceMs) {
-      unpaired.push({
-        conversationId: sh.conversationId,
-        shadowAt: sh.createdAt,
-        v3Count: new Set(sh.concepts.map((c) => c.slug)).size,
-      });
-      continue;
-    }
-    best.consumed = true;
-
-    const v2Set = selectedV2Slugs(best.row.concepts);
-    const laneBySlug = v3LaneBySlug(sh.concepts);
-    const v3Set = new Set(laneBySlug.keys());
-
-    const overlap: string[] = [];
-    const v3Only: string[] = [];
-    for (const slug of v3Set) {
-      if (v2Set.has(slug)) {
-        overlap.push(slug);
-        bump(overlapByLane, laneBySlug.get(slug)!);
-      } else {
-        v3Only.push(slug);
-        bump(v3OnlyByLane, laneBySlug.get(slug)!);
-        v3OnlyFreq.set(slug, (v3OnlyFreq.get(slug) ?? 0) + 1);
-      }
-    }
-    const v2Only: string[] = [];
-    for (const slug of v2Set) {
-      if (!v3Set.has(slug)) {
-        v2Only.push(slug);
-        v2OnlyFreq.set(slug, (v2OnlyFreq.get(slug) ?? 0) + 1);
-      }
-    }
-
-    const unionSize = new Set([...v2Set, ...v3Set]).size;
-    turns.push({
-      conversationId: sh.conversationId,
-      shadowAt: sh.createdAt,
-      routerAt: best.row.createdAt,
-      deltaMs: best.row.createdAt - sh.createdAt,
-      v2Count: v2Set.size,
-      v3Count: v3Set.size,
-      v2CachedCount: cachedV2Count(best.row.concepts),
-      jaccard: unionSize === 0 ? 0 : overlap.length / unionSize,
-      overlap: overlap.sort(),
-      v3Only: v3Only.sort(),
-      v2Only: v2Only.sort(),
-      laneBySlug: Object.fromEntries(laneBySlug),
-    });
-  }
-
-  const n = turns.length;
-  const sum = (pick: (t: ShadowDiffTurn) => number): number =>
-    turns.reduce((acc, t) => acc + pick(t), 0);
-  const mean = (pick: (t: ShadowDiffTurn) => number): number =>
-    n === 0 ? 0 : sum(pick) / n;
-
-  // Newest-first for the detail listing; aggregates are order-independent.
-  const detail = [...turns]
-    .sort((a, b) => b.shadowAt - a.shadowAt)
-    .slice(0, detailLimit);
-
-  return {
-    toleranceMs,
-    shadowRows: shadow.length,
-    turnsCompared: n,
-    unpaired,
-    agg: {
-      meanV2: mean((t) => t.v2Count),
-      meanV3: mean((t) => t.v3Count),
-      meanOverlap: mean((t) => t.overlap.length),
-      meanJaccard: mean((t) => t.jaccard),
-      totalOverlap: sum((t) => t.overlap.length),
-      totalV3Only: sum((t) => t.v3Only.length),
-      totalV2Only: sum((t) => t.v2Only.length),
-      v3OnlyByLane,
-      overlapByLane,
-      v2OnlyTop: topSlugs(v2OnlyFreq, TOP_FREQUENCY_LIMIT),
-      v3OnlyTop: topSlugs(v3OnlyFreq, TOP_FREQUENCY_LIMIT),
-    },
-    turns: detail,
-  };
-}

package/src/memory/v3/shadow-middleware.ts

@@ -1,347 +0,0 @@
-/**
- * Memory v3 — live-shadow `memoryRetrieval` middleware.
- *
- * Registered unconditionally into the `memoryRetrieval` pipeline, but inert
- * unless BOTH `config.memory.v3.enabled` and `config.memory.v3.shadow` are on.
- * When inert it is a byte-for-byte pass-through: it returns `next(args)`
- * verbatim and performs zero extra work (no v3 call, no DB read, no log write).
- *
- * When active, it:
- *   1. Returns the real (v2/default) `MemoryResult` from `next(args)` promptly —
- *      the injected context is ALWAYS the v2 result, never v3.
- *   2. Kicks off the v3 retrieval loop DETACHED (not awaited on the path that
- *      returns the result), so the shadow run can never block or slow the turn.
- *   3. Logs v3's selection set to `memory_v2_activation_logs` with
- *      `mode = "v3_shadow"`. The harness oracle filters `mode='router'`, so
- *      shadow rows never pollute it; the inspector can still surface them.
- *
- * The shadow build mirrors the inputs the v2 router receives (recent turn
- * pairs, NOW context, prior-ever-injected slugs, config) so its recall is
- * measured against the same situational context the live path saw. Failures
- * are swallowed with a warn — the shadow is observational only and must never
- * affect the live turn.
- */
-
-import { desc, eq } from "drizzle-orm";
-
-import { getConfig } from "../../config/loader.js";
-import { registerPlugin } from "../../plugins/registry.js";
-import {
-  type MemoryArgs,
-  type MemoryResult,
-  type Middleware,
-  type Plugin,
-  PluginExecutionError,
-} from "../../plugins/types.js";
-import type { ContentBlock } from "../../providers/types.js";
-import { isUntrustedTrustClass } from "../../runtime/actor-trust-resolver.js";
-import { getLogger } from "../../util/logger.js";
-import { getWorkspaceDir } from "../../util/platform.js";
-import type { DrizzleDb } from "../db-connection.js";
-import { getDb } from "../db-connection.js";
-import {
-  type MemoryV2ConceptRowRecord,
-  type MemoryV2ConfigSnapshot,
-  recordMemoryV2ActivationLog,
-} from "../memory-v2-activation-log-store.js";
-import { messages } from "../schema.js";
-import { hydrate } from "../v2/activation-store.js";
-import type { RetrievalInput } from "../v2/harness/retriever.js";
-import { loadNowText } from "../v2/now-text.js";
-import type { RouterTurnPair } from "../v2/router.js";
-import type { EverInjectedEntry } from "../v2/types.js";
-import { runRetrievalLoop } from "./loop.js";
-
-const log = getLogger("memory-v3-shadow");
-
-/**
- * Extract the recent (assistant, user) turn pairs from a conversation's
- * message list, newest-pair-last, capped at `k`. Mirrors production
- * `extractRecentTurnPairs` in `conversation-graph-memory.ts` (and its harness
- * twin in `replay-input.ts`) so the shadow's `recentTurnPairs` matches what the
- * live router was fed.
- */
-function extractRecentTurnPairs(
-  msgs: ReadonlyArray<{ role: string; content: ContentBlock[] }>,
-  k: number,
-): RouterTurnPair[] {
-  const messageText = (content: ContentBlock[]): string =>
-    content
-      .filter(
-        (b): b is Extract<ContentBlock, { type: "text" }> => b.type === "text",
-      )
-      .map((b) => b.text)
-      .join(" ");
-
-  const pairs: RouterTurnPair[] = [];
-  let pendingUser: string | null = null;
-  for (let i = msgs.length - 1; i >= 0 && pairs.length < k; i--) {
-    const msg = msgs[i]!;
-    if (msg.role === "user" && pendingUser === null) {
-      pendingUser = messageText(msg.content);
-    } else if (msg.role === "assistant" && pendingUser !== null) {
-      pairs.unshift({
-        assistantMessage: messageText(msg.content),
-        userMessage: pendingUser,
-      });
-      pendingUser = null;
-    }
-  }
-  if (pendingUser !== null && pairs.length < k) {
-    pairs.unshift({ assistantMessage: "", userMessage: pendingUser });
-  }
-  if (pairs.length === 0) {
-    pairs.push({ assistantMessage: "", userMessage: "" });
-  }
-  return pairs;
-}
-
-/** Parse a persisted JSON content-block string; tolerate malformed rows. */
-function parseContent(raw: string): ContentBlock[] {
-  try {
-    const parsed = JSON.parse(raw);
-    return Array.isArray(parsed) ? (parsed as ContentBlock[]) : [];
-  } catch {
-    return [];
-  }
-}
-
-/**
- * Load the most recent messages for a conversation, oldest-first, bounded to a
- * small generous multiple of `historicalPairs`. Pair extraction only needs the
- * tail, so a bounded `LIMIT` query avoids loading an entire (potentially
- * multi-GB) conversation on every shadow turn — mirrors the harness's bounded
- * fetch in `replay-input.ts`.
- */
-function loadRecentMessages(
-  db: DrizzleDb,
-  conversationId: string,
-  historicalPairs: number,
-): Array<{ role: string; content: ContentBlock[] }> {
-  const fetchWindow = Math.max(20, historicalPairs * 12);
-  const rows = db
-    .select({ role: messages.role, content: messages.content })
-    .from(messages)
-    .where(eq(messages.conversationId, conversationId))
-    .orderBy(desc(messages.createdAt), desc(messages.id))
-    .limit(fetchWindow)
-    .all();
-  return rows
-    .reverse()
-    .map((r) => ({ role: r.role, content: parseContent(r.content) }));
-}
-
-/**
- * Empty config snapshot for shadow log rows. The activation-state values are
- * meaningless for a v3 selection (it computes no spreading-activation scores),
- * so they are zeroed — exactly as the v2 router-mode rows do.
- */
-const SHADOW_CONFIG_SNAPSHOT: MemoryV2ConfigSnapshot = {
-  d: 0,
-  c_user: 0,
-  c_assistant: 0,
-  c_now: 0,
-  k: 0,
-  hops: 0,
-  top_k: 0,
-  epsilon: 0,
-};
-
-/**
- * Build the concept rows logged for a v3 shadow selection. Each selected slug
- * becomes a zeroed concept row tagged `source: "router"` and
- * `status: "injected"` — the shadow has no activation scores to record, and the
- * `mode='v3_shadow'` row tag (not the concept source) is what distinguishes
- * shadow telemetry from live router selections. Each row also carries the
- * `lane` that surfaced the slug (from `sourceBySlug`) so a shadow run can be
- * analyzed by provenance.
- */
-function buildShadowConceptRows(
-  selectedSlugs: readonly string[],
-  sourceBySlug: ReadonlyMap<string, string>,
-): MemoryV2ConceptRowRecord[] {
-  return selectedSlugs.map((slug) => ({
-    slug,
-    finalActivation: 0,
-    ownActivation: 0,
-    priorActivation: 0,
-    simUser: 0,
-    simAssistant: 0,
-    simNow: 0,
-    simUserRerankBoost: 0,
-    simAssistantRerankBoost: 0,
-    inRerankPool: false,
-    spreadContribution: 0,
-    source: "router",
-    status: "injected",
-    ...(sourceBySlug.get(slug) ? { lane: sourceBySlug.get(slug) } : {}),
-  }));
-}
-
-/**
- * Run the v3 retrieval loop for the shadow and log its selection. Best-effort:
- * any failure is logged and swallowed. Honors `signal` so a cancelled turn
- * stops the shadow's lane work.
- */
-async function runShadowAndLog(
-  args: MemoryArgs,
-  signal: AbortSignal,
-): Promise<void> {
-  try {
-    if (signal.aborted) return;
-
-    const config = getConfig();
-    const workspaceDir = getWorkspaceDir();
-    const db = getDb();
-
-    const historicalPairs = config.memory.v2.router.historical_pairs;
-    const recentMessages = loadRecentMessages(
-      db,
-      args.conversationId,
-      historicalPairs,
-    );
-    const recentTurnPairs = extractRecentTurnPairs(
-      recentMessages,
-      historicalPairs,
-    );
-
-    const nowText = await loadNowText(workspaceDir);
-
-    let priorEverInjected: readonly EverInjectedEntry[] = [];
-    try {
-      const state = await hydrate(db, args.conversationId);
-      priorEverInjected = state?.everInjected ?? [];
-    } catch (err) {
-      log.warn(
-        { err, conversationId: args.conversationId },
-        "v3 shadow: failed to hydrate prior-ever-injected; continuing with empty set",
-      );
-    }
-
-    if (signal.aborted) return;
-
-    const input: RetrievalInput = {
-      workspaceDir,
-      recentTurnPairs,
-      nowText,
-      priorEverInjected,
-      config,
-      signal,
-    };
-
-    const output = await runRetrievalLoop(input, {
-      db,
-      conversationId: args.conversationId,
-      turn: args.turnIndex,
-    });
-
-    if (signal.aborted) return;
-
-    // Per-turn summary so a shadow run is analyzable from the logs — the pool
-    // shape, how many passes ran, and the gate's verdict + rationale. The
-    // selection set + per-slug lane land in the activation log below.
-    const passes = output.trace?.passes ?? [];
-    const lastGate = passes[passes.length - 1]?.gate;
-    const laneTally: Record<string, number> = {};
-    for (const lane of output.sourceBySlug.values()) {
-      laneTally[lane] = (laneTally[lane] ?? 0) + 1;
-    }
-    log.info(
-      {
-        conversationId: args.conversationId,
-        turn: args.turnIndex,
-        selected: output.selectedSlugs.length,
-        poolSize: output.sourceBySlug.size,
-        laneTally,
-        passes: passes.length,
-        gateDecision: lastGate?.decision,
-        gateReasoning: lastGate?.reasoning,
-      },
-      "v3 shadow selection",
-    );
-
-    recordMemoryV2ActivationLog({
-      conversationId: args.conversationId,
-      turn: args.turnIndex,
-      mode: "v3_shadow",
-      concepts: buildShadowConceptRows(
-        output.selectedSlugs,
-        output.sourceBySlug,
-      ),
-      config: SHADOW_CONFIG_SNAPSHOT,
-    });
-  } catch (err) {
-    log.warn(
-      { err, conversationId: args.conversationId, turn: args.turnIndex },
-      "v3 shadow retrieval failed; live turn unaffected",
-    );
-  }
-}
-
-/**
- * Live-shadow `memoryRetrieval` middleware.
- *
- * Flag-gated INSIDE the middleware (per-turn, live-toggle): when v3 shadow is
- * off it is a pure pass-through. When on, it fires the v3 loop detached and
- * returns the unchanged downstream (v2) result immediately.
- *
- * The shadow loop spends filter + gate LLM calls, so — like the other
- * guardian-trust background memory loops (`enqueueAutoAnalysisOnCompaction`,
- * `enqueueMemoryRetrospectiveOnCompaction`) — it is gated on actor trust: an
- * untrusted turn passes through without kicking off the v3 loop.
- */
-export const memoryV3ShadowMiddleware: Middleware<MemoryArgs, MemoryResult> =
-  async function memoryV3Shadow(args, next) {
-    const v3 = getConfig().memory.v3;
-    if (!v3?.enabled || !v3?.shadow) {
-      // Inert: byte-for-byte pass-through, zero extra work.
-      return next(args);
-    }
-
-    if (isUntrustedTrustClass(args.trustContext?.trustClass)) {
-      // Untrusted actor: don't spend shadow retrieval LLM calls — mirrors the
-      // live path's trust gate. Pure pass-through, no detached work.
-      return next(args);
-    }
-
-    // Detached — never awaited on the path that returns the result, so the
-    // shadow can neither block nor slow the live turn. Errors are swallowed
-    // inside `runShadowAndLog`.
-    void runShadowAndLog(args, args.signal);
-
-    return next(args);
-  };
-
-/**
- * First-party plugin contributing the live-shadow `memoryRetrieval`
- * middleware. Registered unconditionally by the plugin bootstrap (it is inert
- * unless both v3 flags are on), so the registration is always present but does
- * zero work in the default (flags-off) configuration.
- */
-export const memoryV3ShadowPlugin: Plugin = {
-  manifest: {
-    name: "memory-v3-shadow",
-    version: "0.0.1",
-  },
-  middleware: {
-    memoryRetrieval: memoryV3ShadowMiddleware,
-  },
-};
-
-// Module-load side effect: register the shadow plugin at import time so the
-// registry is populated even in tests that skip `bootstrapPlugins()`, matching
-// the first-party `default-*` plugins. Idempotent via the swallowed
-// duplicate-name check (the defaults aggregator also lists this plugin).
-try {
-  registerPlugin(memoryV3ShadowPlugin);
-} catch (err) {
-  if (
-    err instanceof PluginExecutionError &&
-    err.message.includes("already registered")
-  ) {
-    // already registered — expected when both the defaults aggregator and the
-    // direct module import run in the same process.
-  } else {
-    throw err;
-  }
-}