@vellumai/assistant 0.10.0 → 0.10.1-staging.1

package/src/memory/v3-eval/eval-packets.ts

@@ -25,7 +25,7 @@ import {
 } from "node:fs";
 import { join, resolve, sep } from "node:path";
 
-import { and, desc, eq, sql } from "drizzle-orm";
+import { and, desc, eq, inArray, notInArray, sql } from "drizzle-orm";
 
 import type { AssistantConfig } from "../../config/types.js";
 import { renderCard } from "../../plugins/defaults/memory-v3-shadow/card.js";
@@ -304,12 +304,67 @@ export function pairTurns(
   return capped.slice(0, opts.limit);
 }
 
-/** Read recent user→assistant turns from the live DB (read-only). */
+/** The conversation id embedded in a `${conversationId}:${createdAt}` turn id. */
+export function turnConversationId(turnId: string): string {
+  return turnId.slice(0, turnId.lastIndexOf(":"));
+}
+
+/**
+ * Keep exactly the pinned turn ids, in pinned order. Turns that no longer exist
+ * (deleted since the ids were captured) are dropped — the caller compares the
+ * returned count to the requested count to surface drops. Pure over `allTurns`
+ * so it is unit-testable.
+ */
+export function selectPinnedTurns(
+  allTurns: MinedTurn[],
+  pinnedTurnIds: string[],
+): MinedTurn[] {
+  const byId = new Map(allTurns.map((t) => [t.turn, t]));
+  const out: MinedTurn[] = [];
+  for (const id of pinnedTurnIds) {
+    const t = byId.get(id);
+    if (t) out.push(t);
+  }
+  return out;
+}
+
+/** Group by conversation, ascending by time, so `pairTurns` walks each in order. */
+function sortForPairing(rows: RawMsgRow[]): RawMsgRow[] {
+  return [...rows].sort(
+    (a, b) =>
+      a.conversationId.localeCompare(b.conversationId) ||
+      a.createdAt - b.createdAt ||
+      a.id.localeCompare(b.id),
+  );
+}
+
+/**
+ * Read user→assistant turns from the live DB (read-only).
+ *
+ * Default: the most recent `limit` turns by recency, optionally omitting given
+ * conversations — e.g. the migration's own chat, which would otherwise let the
+ * eval judge turns from the very conversation driving the migration. With
+ * `pinnedTurnIds`, instead re-mine EXACTLY those turns (scoped to their
+ * conversations, no recency window) so a re-run measures the same turn set as a
+ * prior run — the eval is only reproducible if the turns are held fixed while
+ * the staged corpus changes.
+ */
 export function mineTurns(
   db: DrizzleDb,
-  opts: { limit: number; perConversationCap: number; maxScan?: number },
+  opts: {
+    limit: number;
+    perConversationCap: number;
+    maxScan?: number;
+    pinnedTurnIds?: string[];
+    excludeConversationIds?: string[];
+  },
 ): MinedTurn[] {
+  if (opts.pinnedTurnIds && opts.pinnedTurnIds.length > 0) {
+    return minePinnedTurns(db, opts.pinnedTurnIds);
+  }
+
   const maxScan = opts.maxScan ?? 6000;
+  const exclude = opts.excludeConversationIds ?? [];
   const recent = db
     .select({
       conversationId: messages.conversationId,
@@ -324,20 +379,45 @@ export function mineTurns(
       and(
         sql`COALESCE(${conversations.source}, 'user') = 'user'`,
         sql`${conversations.scheduleJobId} IS NULL`,
+        ...(exclude.length > 0
+          ? [notInArray(messages.conversationId, exclude)]
+          : []),
       ),
     )
     .orderBy(desc(messages.createdAt), desc(messages.id))
     .limit(maxScan)
     .all() as RawMsgRow[];
 
-  // Re-sort ascending and group so the pairing walks each conversation in order.
-  recent.sort(
-    (a, b) =>
-      a.conversationId.localeCompare(b.conversationId) ||
-      a.createdAt - b.createdAt ||
-      a.id.localeCompare(b.id),
-  );
-  return pairTurns(recent, opts);
+  return pairTurns(sortForPairing(recent), opts);
+}
+
+/** Re-mine exactly the pinned turns from their conversations (no recency cap). */
+function minePinnedTurns(db: DrizzleDb, pinnedTurnIds: string[]): MinedTurn[] {
+  const conversationIds = [...new Set(pinnedTurnIds.map(turnConversationId))];
+  const rows = db
+    .select({
+      conversationId: messages.conversationId,
+      id: messages.id,
+      role: messages.role,
+      content: messages.content,
+      createdAt: messages.createdAt,
+    })
+    .from(messages)
+    .innerJoin(conversations, eq(messages.conversationId, conversations.id))
+    .where(
+      and(
+        inArray(messages.conversationId, conversationIds),
+        sql`COALESCE(${conversations.source}, 'user') = 'user'`,
+        sql`${conversations.scheduleJobId} IS NULL`,
+      ),
+    )
+    .all() as RawMsgRow[];
+
+  const allTurns = pairTurns(sortForPairing(rows), {
+    limit: Number.MAX_SAFE_INTEGER,
+    perConversationCap: Number.MAX_SAFE_INTEGER,
+  });
+  return selectPinnedTurns(allTurns, pinnedTurnIds);
 }
 
 // ---------------------------------------------------------------------------
@@ -436,16 +516,52 @@ export interface EvalParams {
   dense?: boolean;
   seed?: number;
   sectionCharCap?: number;
+  /**
+   * Pin the exact turns to mine (the `turn` ids from a prior run's key.json or
+   * packets.json). Re-running with the same pinned set holds the comparison
+   * fixed while the staged corpus changes — required for a reproducible re-judge.
+   */
+  turnIds?: string[];
+  /**
+   * Conversations to omit from the default (recency) mining — e.g. the
+   * migration's own chat, so the eval does not judge turns from the very
+   * conversation driving it. Ignored when `turnIds` is set.
+   */
+  excludeConversationIds?: string[];
+}
+
+/**
+ * The embedding backend identity in effect for a run, recorded in eval-meta.json.
+ * Dense retrieval is only comparable across runs with the SAME identity: a
+ * mid-migration config change (e.g. a 384-dim local model to a 3072-dim hosted
+ * one) silently re-embeds both corpora in a different vector space, so a re-run
+ * that drifted here is measuring something different — exactly the kind of hidden
+ * variable that made an eval gate look non-reproducible.
+ */
+export interface EmbeddingIdentity {
+  provider: string;
+  model: string;
+  /** Observed embedding dimension (from the first vector); null when dense is off. */
+  dims: number | null;
 }
 
 export interface EvalResult {
   turnsMined: number;
+  /** Turns requested: the pinned count, or the recency `turns` limit. */
+  turnsRequested: number;
   packetsWritten: number;
   packetsPath: string;
   keyPath: string;
+  /** eval-meta.json: seed/k/dense/embedding identity + the resolved turn ids. */
+  metaPath: string;
   snapshotPages: number;
   stagingPages: number;
   dense: boolean;
+  seed: number;
+  k: number;
+  embedding: EmbeddingIdentity;
+  /** The resolved turn ids actually packed — pass as `turnIds` to pin a re-run. */
+  turnIds: string[];
 }
 
 export interface EvalDeps {
@@ -456,6 +572,26 @@ export interface EvalDeps {
   embed?: EmbedAll;
 }
 
+/** Human-readable identity of the configured embedding model (provider + model name). */
+function describeEmbeddingModel(config: AssistantConfig): {
+  provider: string;
+  model: string;
+} {
+  const e = config.memory?.embeddings;
+  const provider = e?.provider ?? "unknown";
+  const model =
+    provider === "gemini"
+      ? e?.geminiModel
+      : provider === "openai"
+        ? e?.openaiModel
+        : provider === "ollama"
+          ? e?.ollamaModel
+          : provider === "local"
+            ? e?.localModel
+            : provider; // "auto" / future providers: name == provider
+  return { provider, model: model ?? provider };
+}
+
 /** Chunk an embed function so a large corpus never overruns provider batch limits. */
 function chunkedEmbed(embed: EmbedAll, batchSize = 96): EmbedAll {
   return async (texts: string[]) => {
@@ -502,7 +638,17 @@ export async function runMemoryEval(
   const baseEmbed: EmbedAll =
     deps.embed ??
     (async (texts) => (await embedWithRetry(deps.config, texts)).vectors);
-  const embedAll = chunkedEmbed(baseEmbed);
+  // Capture the observed embedding dimension so eval-meta.json records the
+  // vector space the dense lane actually ran in (drift detection across re-runs).
+  let observedDims: number | null = null;
+  const capturingEmbed: EmbedAll = async (texts) => {
+    const vectors = await baseEmbed(texts);
+    if (observedDims === null && vectors.length > 0 && vectors[0]) {
+      observedDims = vectors[0].length;
+    }
+    return vectors;
+  };
+  const embedAll = chunkedEmbed(capturingEmbed);
 
   const snapshotCorpus = loadCorpus(snapshotDir);
   const stagingCorpus = loadCorpus(stagingDir);
@@ -510,9 +656,17 @@ export async function runMemoryEval(
   const snapshot = await buildRetriever(snapshotCorpus, embedAll, dense);
   const staging = await buildRetriever(stagingCorpus, embedAll, dense);
 
+  const turnsRequested =
+    params.turnIds && params.turnIds.length > 0
+      ? params.turnIds.length
+      : (params.turns ?? 30);
   const turns = mineTurns(deps.db, {
     limit: params.turns ?? 30,
     perConversationCap: params.perConversationCap ?? 4,
+    ...(params.turnIds ? { pinnedTurnIds: params.turnIds } : {}),
+    ...(params.excludeConversationIds
+      ? { excludeConversationIds: params.excludeConversationIds }
+      : {}),
   });
 
   const { packets, key } = await buildPackets(
@@ -528,19 +682,50 @@ export async function runMemoryEval(
     },
   );
 
+  const embedding: EmbeddingIdentity = {
+    ...describeEmbeddingModel(deps.config),
+    dims: dense ? observedDims : null,
+  };
+  const turnIds = packets.map((p) => p.turn);
+
   mkdirSync(outDir, { recursive: true });
   const packetsPath = join(outDir, "packets.json");
   const keyPath = join(outDir, "key.json");
+  const metaPath = join(outDir, "eval-meta.json");
   writeFileSync(packetsPath, JSON.stringify(packets, null, 2));
   writeFileSync(keyPath, JSON.stringify(key, null, 2));
+  writeFileSync(
+    metaPath,
+    JSON.stringify(
+      {
+        seed,
+        k,
+        dense,
+        embedding,
+        snapshotPages: snapshotCorpus.slugs.length,
+        stagingPages: stagingCorpus.slugs.length,
+        turnsRequested,
+        turnsMined: turns.length,
+        turnIds,
+      },
+      null,
+      2,
+    ),
+  );
 
   return {
     turnsMined: turns.length,
+    turnsRequested,
     packetsWritten: packets.length,
     packetsPath,
     keyPath,
+    metaPath,
     snapshotPages: snapshotCorpus.slugs.length,
     stagingPages: stagingCorpus.slugs.length,
     dense,
+    seed,
+    k,
+    embedding,
+    turnIds,
   };
 }

package/src/memory/v3-eval/eval-tally.ts

@@ -0,0 +1,234 @@
+/**
+ * Deterministic unblind + tally for the memory-v3 corpus eval gate.
+ *
+ * The blind-judge workflow returns one verdict per packet (A vs B) without
+ * knowing which side is which; `eval` writes a separate per-turn `key.json`
+ * mapping A/B → snapshot/staging. This module joins the two and produces the
+ * gate verdict, replacing an ad-hoc hand tally.
+ *
+ * Two failure modes this exists to prevent (both seen in the field):
+ *
+ * 1. **A/B is shuffled PER TURN** (a single run has A=snapshot on some turns
+ *    and A=staging on others). A global "A won N times" count is meaningless —
+ *    the winner must be mapped through the key turn-by-turn. A hand tally that
+ *    assumed "A == snapshot" for the whole run flipped the result.
+ * 2. **Win-count over near-tied per-turn scores amplifies judge noise.** When
+ *    most decided turns hinge on a single point of a 0–10 score, a 12–11 split
+ *    is a coin flip, not a loss. The gate therefore applies a two-sided sign
+ *    test: the wiki only FAILS when the snapshot's win lead is statistically
+ *    significant. "Win or tie" passes; a within-noise difference is a tie.
+ *
+ * Supports a judge PANEL: pass multiple verdicts per turn (e.g. K judges, or
+ * the same set re-judged under different seeds) and each turn is decided by the
+ * panel's majority before the set-level tally. A larger panel tightens the
+ * per-turn signal that the sign test then aggregates.
+ */
+
+import type { EvalKeyEntry } from "./eval-packets.js";
+
+/** One judge's verdict for one packet (the blind-judge workflow's leaf output). */
+export interface JudgeVerdict {
+  turn: string;
+  /** The judge's explicit call. When absent/invalid it is derived from scores. */
+  winner?: "A" | "B" | "tie" | string;
+  scoreA: number;
+  scoreB: number;
+}
+
+export interface TallyOptions {
+  /** Significance threshold for the two-sided sign test (default 0.05). */
+  alpha?: number;
+  /** Minimum decided turns to call the result confident (default 10). */
+  minDecided?: number;
+  /** Minimum mean panel votes/turn to call the result confident (default 3). */
+  minPanel?: number;
+}
+
+export interface TallyResult {
+  /** Distinct turns with at least one verdict matched to the key. */
+  turns: number;
+  verdictsCounted: number;
+  /** Verdicts whose `turn` had no key entry (ignored). */
+  unmatchedVerdicts: number;
+  /** Votes per turn across the panel. */
+  panel: { min: number; max: number; mean: number };
+  snapshotWins: number;
+  stagingWins: number;
+  ties: number;
+  /** snapshotWins + stagingWins (turns that were not a tie). */
+  decided: number;
+  /** Mean per-turn score for each corpus (each turn weighted equally). */
+  meanSnapshot: number;
+  meanStaging: number;
+  /** Two-sided binomial sign-test p-value for the snapshot/staging win split. */
+  signTestP: number;
+  /** wiki-wins / tie / wiki-loses — significance-gated, not raw count. */
+  verdict: "wiki-wins" | "tie" | "wiki-loses";
+  /** The ship gate: the wiki must win OR tie, so `fail` iff `wiki-loses`. */
+  gate: "pass" | "fail";
+  /** Enough decided turns + panel votes to trust the verdict. */
+  confident: boolean;
+  notes: string[];
+}
+
+/**
+ * Two-sided binomial sign-test p-value for a split of `a` vs `b` successes under
+ * H0 p=0.5. Computed in log space (via log-factorials) so it is stable for the
+ * few-hundred-turn range without overflowing `2^n`.
+ */
+export function binomialTwoSidedSignP(a: number, b: number): number {
+  const n = a + b;
+  if (n === 0) return 1;
+  const logFact: number[] = new Array(n + 1);
+  logFact[0] = 0;
+  for (let i = 1; i <= n; i++) logFact[i] = logFact[i - 1]! + Math.log(i);
+  const logChoose = (k: number): number =>
+    logFact[n]! - logFact[k]! - logFact[n - k]!;
+  const hi = Math.max(a, b);
+  let tail = 0;
+  for (let k = hi; k <= n; k++) {
+    tail += Math.exp(logChoose(k) - n * Math.LN2);
+  }
+  return Math.min(1, 2 * tail);
+}
+
+function mean(xs: number[]): number {
+  return xs.length === 0 ? 0 : xs.reduce((s, x) => s + x, 0) / xs.length;
+}
+
+/** Resolve a verdict's winning corpus, deriving from scores if `winner` is unusable. */
+function winnerSide(
+  v: JudgeVerdict,
+  key: EvalKeyEntry,
+): "snapshot" | "staging" | "tie" {
+  let w = v.winner;
+  if (w !== "A" && w !== "B" && w !== "tie") {
+    w = v.scoreA > v.scoreB ? "A" : v.scoreB > v.scoreA ? "B" : "tie";
+  }
+  if (w === "tie") return "tie";
+  return w === "A" ? key.a : key.b;
+}
+
+/**
+ * Join judge verdicts to the unblinding key and produce the gate verdict.
+ * Pure over its inputs.
+ */
+export function tallyVerdicts(
+  verdicts: JudgeVerdict[],
+  key: EvalKeyEntry[],
+  opts: TallyOptions = {},
+): TallyResult {
+  const alpha = opts.alpha ?? 0.05;
+  const minDecided = opts.minDecided ?? 10;
+  const minPanel = opts.minPanel ?? 3;
+
+  const keyByTurn = new Map(key.map((k) => [k.turn, k]));
+  const byTurn = new Map<
+    string,
+    {
+      sides: ("snapshot" | "staging" | "tie")[];
+      snap: number[];
+      stage: number[];
+    }
+  >();
+  let unmatched = 0;
+  let counted = 0;
+  for (const v of verdicts) {
+    const k = keyByTurn.get(v.turn);
+    if (!k) {
+      unmatched++;
+      continue;
+    }
+    counted++;
+    let g = byTurn.get(v.turn);
+    if (!g) {
+      g = { sides: [], snap: [], stage: [] };
+      byTurn.set(v.turn, g);
+    }
+    g.sides.push(winnerSide(v, k));
+    g.snap.push(k.a === "snapshot" ? v.scoreA : v.scoreB);
+    g.stage.push(k.a === "snapshot" ? v.scoreB : v.scoreA);
+  }
+
+  let snapshotWins = 0;
+  let stagingWins = 0;
+  let ties = 0;
+  let sumSnap = 0;
+  let sumStage = 0;
+  const panelSizes: number[] = [];
+  for (const g of byTurn.values()) {
+    panelSizes.push(g.sides.length);
+    const nSnap = g.sides.filter((s) => s === "snapshot").length;
+    const nStage = g.sides.filter((s) => s === "staging").length;
+    if (nStage > nSnap) stagingWins++;
+    else if (nSnap > nStage) snapshotWins++;
+    else ties++; // tie vote, or equal snapshot/staging votes
+    sumSnap += mean(g.snap);
+    sumStage += mean(g.stage);
+  }
+
+  const turns = byTurn.size;
+  const decided = snapshotWins + stagingWins;
+  const signTestP = binomialTwoSidedSignP(snapshotWins, stagingWins);
+
+  let verdict: TallyResult["verdict"];
+  if (stagingWins > snapshotWins && signTestP < alpha) verdict = "wiki-wins";
+  else if (snapshotWins > stagingWins && signTestP < alpha)
+    verdict = "wiki-loses";
+  else verdict = "tie";
+  const gate: "pass" | "fail" = verdict === "wiki-loses" ? "fail" : "pass";
+
+  const panel = {
+    min: panelSizes.length ? Math.min(...panelSizes) : 0,
+    max: panelSizes.length ? Math.max(...panelSizes) : 0,
+    mean: mean(panelSizes),
+  };
+  const confident = decided >= minDecided && panel.mean >= minPanel;
+
+  const notes: string[] = [];
+  if (turns === 0) {
+    notes.push(
+      "No verdicts matched the key — check the verdicts/key files are from the same run.",
+    );
+  }
+  if (turns > 0 && panel.mean < minPanel) {
+    notes.push(
+      `Low panel size (mean ${panel.mean.toFixed(1)} votes/turn) — single-vote judging is noisy; ` +
+        `re-judge with a panel (K judges per turn) or repeat under multiple --seed values.`,
+    );
+  }
+  if (turns > 0 && decided < minDecided) {
+    notes.push(
+      `Only ${decided} decided turns — too few to be confident; mine more turns.`,
+    );
+  }
+  if (unmatched > 0) {
+    notes.push(
+      `${unmatched} verdict(s) had no matching key entry and were ignored.`,
+    );
+  }
+  if (verdict === "tie") {
+    notes.push(
+      "Snapshot vs wiki is within noise — a tie satisfies the win-or-tie gate, " +
+        "but the wiki did not clearly beat the current corpus.",
+    );
+  }
+
+  return {
+    turns,
+    verdictsCounted: counted,
+    unmatchedVerdicts: unmatched,
+    panel,
+    snapshotWins,
+    stagingWins,
+    ties,
+    decided,
+    meanSnapshot: turns ? sumSnap / turns : 0,
+    meanStaging: turns ? sumStage / turns : 0,
+    signTestP,
+    verdict,
+    gate,
+    confident,
+    notes,
+  };
+}

package/src/messaging/provider.ts

@@ -29,6 +29,16 @@ export interface MessagingProvider {
   /** Credential service name for token-manager (e.g. 'slack'). */
   credentialService: string;
 
+  /**
+   * Scopes a connection must carry for this provider's tools to work. Set when
+   * the credential service bundles several products behind one OAuth app and
+   * this provider only works with a subset (e.g. Gmail under the shared
+   * 'google' service needs Gmail scopes, not Calendar-only). Forwarded as
+   * `requiredScopes` to {@link resolveOAuthConnection} so a narrowly-scoped
+   * connection fails with an actionable "reconnect" error instead of a 403.
+   */
+  requiredScopes?: string[];
+
   // ── Universal operations (every platform must implement) ──────────
 
   testConnection(connection?: OAuthConnection): Promise<ConnectionInfo>;

package/src/messaging/providers/gmail/adapter.ts

@@ -100,6 +100,7 @@ export const gmailMessagingProvider: MessagingProvider = {
   id: "gmail",
   displayName: "Gmail",
   credentialService: "google",
+  requiredScopes: gmail.GMAIL_REQUIRED_SCOPES,
   capabilities: new Set([
     "threads",
     "labels",

package/src/messaging/providers/gmail/client.ts

@@ -16,6 +16,20 @@ import type {
 
 const GMAIL_BATCH_URL = "https://www.googleapis.com/batch/gmail/v1";
 
+/**
+ * Minimum Google OAuth scope a connection must carry to be usable for Gmail.
+ *
+ * The managed `google` OAuth app bundles Gmail + Calendar + Drive, but a
+ * connection can be granted a narrow subset (e.g. the onboarding check-in flow
+ * requests Calendar-only). Every Gmail read/search/send call needs at least
+ * `gmail.readonly`, so a connection lacking it cannot serve Gmail at all —
+ * resolving against this scope turns a downstream 403 into an actionable
+ * "reconnect Google and grant Gmail" error at resolution time.
+ */
+export const GMAIL_REQUIRED_SCOPES = [
+  "https://www.googleapis.com/auth/gmail.readonly",
+];
+
 /** Max sub-requests per batch HTTP call (Gmail API limit) */
 const BATCH_SUB_LIMIT = 100;
 /** Max concurrent batch calls */

package/src/messaging/providers/index.ts

@@ -197,7 +197,7 @@ async function deliverSlack(
   if (text) {
     const result = await sendSlackReply(chatId, text, {
       threadTs,
-      blocks: blocks as unknown[] | undefined,
+      blocks,
       approval: payload.approval,
       useBlocks: payload.useBlocks,
       ephemeral: payload.ephemeral,