@vellumai/assistant 0.11.3 → 0.11.4-staging.1

package/src/plugins/defaults/memory/indexer.ts

@@ -123,7 +123,9 @@ export async function indexMessageNow(
   // memory_segments has no cross-file FK to messages, so this call must not
   // leave pieces for a message with no row. Skip early when the source
   // row is already gone, the common case of a backfill job running after the
-  // message was deleted. A post-write re-check below closes the narrower window
+  // message was deleted. Any-state existence check, deliberately: a
+  // streaming row exists, and orphan prevention is about deletion, not
+  // completeness. A post-write re-check below closes the narrower window
   // where the delete lands mid-transaction.
   const sourceMessage = getDb()
     .select({ id: messages.id })

package/src/plugins/defaults/memory/memory-retrospective-accounting.ts

@@ -52,18 +52,30 @@ export function isSkillCardMessage(row: { metadata: string | null }): boolean {
 }
 
 /**
- * `getMessagesAfter` minus skill-card rows. The retrospective job's
- * new-message slice: a card-only tail yields an empty slice (the job's
- * `no_new_messages` early return), and a mixed tail's cutoff lands on the
- * last real message rather than the card.
+ * `getMessagesAfter` truncated at the first unfinalized row, minus skill-card
+ * rows. The retrospective job's new-message slice: a card-only tail yields an
+ * empty slice (the job's `no_new_messages` early return), and a mixed tail's
+ * cutoff lands on the last real message rather than the card.
+ *
+ * The slice STOPS BEFORE the first unfinalized row rather than filtering it
+ * out. The job's cutoff (and therefore `lastProcessedMessageId` on success)
+ * is chosen from this slice, while the fork the job reviews skips unfinalized
+ * rows, so the cursor must never move past one: filtering through a stale
+ * mid-slice `finalized = 0` row would advance the cursor beyond it, and once
+ * the row later finalizes its `(createdAt, id)` sits behind the cursor,
+ * unreviewed forever. Truncating parks the cursor before the unfinalized row;
+ * the rows behind it are reviewed by the retrospective that runs after the
+ * row resolves (finalizes, or is repaired away by crash recovery).
  */
 export function getRetrospectiveMessagesAfter(
   conversationId: string,
   afterMessageId: string | null,
 ): MessageRow[] {
-  return getMessagesAfter(conversationId, afterMessageId).filter(
-    (row) => !isSkillCardMessage(row),
-  );
+  const rows = getMessagesAfter(conversationId, afterMessageId);
+  const firstUnfinalized = rows.findIndex((row) => row.finalized !== 1);
+  const bounded =
+    firstUnfinalized === -1 ? rows : rows.slice(0, firstUnfinalized);
+  return bounded.filter((row) => !isSkillCardMessage(row));
 }
 
 /**

package/src/plugins/defaults/memory/src/__tests__/memory-v3-gate-stats.test.ts

@@ -0,0 +1,281 @@
+/**
+ * Tests for `handleMemoryV3GateStats` in `memory-v3-routes.ts`.
+ *
+ * The handler aggregates gate telemetry from the telemetry DB and buckets it
+ * by concept page count. These tests cover:
+ *   - empty DB → zero counts, all buckets present, scoredPassRate null
+ *   - mixed rows land in the correct buckets by real_concept_page_count
+ *   - scored vs unscored runs are counted separately
+ *   - rows with no real_concept_page_count go to unknownPageCount
+ *   - lookbackDays is clamped to [1, 90]
+ *   - malformed JSON rows are skipped
+ *   - null/absent DB returns a zero response
+ */
+
+import type { Database } from "bun:sqlite";
+import { describe, expect, test } from "bun:test";
+
+import { handleMemoryV3GateStats } from "../memory-v3-routes.js";
+
+// ---------------------------------------------------------------------------
+// Minimal fake SQLite DB that returns pre-set rows
+// ---------------------------------------------------------------------------
+
+type FakeRow = { payload: string };
+
+function makePayload(detail: Record<string, unknown>): string {
+  return JSON.stringify({
+    type: "watchdog",
+    daemon_event_id: "test-id",
+    recorded_at: Date.now(),
+    check_name: "memory_v3_injection_gate",
+    value: 1,
+    detail,
+  });
+}
+
+function fakeDb(rows: FakeRow[]): Database {
+  return {
+    query: (_sql: string) => ({
+      all: (..._args: unknown[]) => rows,
+    }),
+  } as unknown as Database;
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("handleMemoryV3GateStats", () => {
+  test("returns zero counts with all buckets when DB is null", () => {
+    const result = handleMemoryV3GateStats(30, null);
+    expect(result.totalRuns).toBe(0);
+    expect(result.buckets).toHaveLength(4);
+    for (const bucket of result.buckets) {
+      expect(bucket.total).toBe(0);
+      expect(bucket.scored).toBe(0);
+      expect(bucket.passed).toBe(0);
+      expect(bucket.scoredPassRate).toBeNull();
+      expect(bucket.reasons).toEqual({});
+    }
+    expect(result.unknownPageCount).toEqual({
+      total: 0,
+      passed: 0,
+      reasons: {},
+    });
+  });
+
+  test("returns zero counts with all buckets when DB returns no rows", () => {
+    const result = handleMemoryV3GateStats(30, fakeDb([]));
+    expect(result.totalRuns).toBe(0);
+    expect(result.buckets[0]!.pageCountRange).toBe("0–9");
+    expect(result.buckets[3]!.pageCountRange).toBe("200+");
+    for (const b of result.buckets) {
+      expect(b.scoredPassRate).toBeNull();
+    }
+  });
+
+  test("buckets a dense_pass row into 10–49 bucket", () => {
+    const rows = [
+      {
+        payload: makePayload({
+          pass: true,
+          reason: "dense_pass",
+          scored: true,
+          real_concept_page_count: 25,
+        }),
+      },
+    ];
+    const result = handleMemoryV3GateStats(30, fakeDb(rows));
+    expect(result.totalRuns).toBe(1);
+    const bucket = result.buckets.find((b) => b.pageCountRange === "10–49")!;
+    expect(bucket.total).toBe(1);
+    expect(bucket.scored).toBe(1);
+    expect(bucket.passed).toBe(1);
+    expect(bucket.scoredPassRate).toBe(1);
+    expect(bucket.reasons).toEqual({ dense_pass: 1 });
+  });
+
+  test("buckets a fail row into 200+ bucket", () => {
+    const rows = [
+      {
+        payload: makePayload({
+          pass: false,
+          reason: "fail_no_signal",
+          scored: true,
+          real_concept_page_count: 350,
+        }),
+      },
+    ];
+    const result = handleMemoryV3GateStats(30, fakeDb(rows));
+    const bucket = result.buckets.find((b) => b.pageCountRange === "200+")!;
+    expect(bucket.total).toBe(1);
+    expect(bucket.passed).toBe(0);
+    expect(bucket.scoredPassRate).toBe(0);
+    expect(bucket.reasons).toEqual({ fail_no_signal: 1 });
+  });
+
+  test("unscored pass-open rows count in total but not scored", () => {
+    const rows = [
+      {
+        payload: makePayload({
+          pass: true,
+          reason: "dense_unavailable",
+          scored: false,
+          real_concept_page_count: 5,
+        }),
+      },
+    ];
+    const result = handleMemoryV3GateStats(30, fakeDb(rows));
+    const bucket = result.buckets.find((b) => b.pageCountRange === "0–9")!;
+    expect(bucket.total).toBe(1);
+    expect(bucket.scored).toBe(0);
+    expect(bucket.passed).toBe(1);
+    // No scored runs → scoredPassRate is null (unscored pass does not count)
+    expect(bucket.scoredPassRate).toBeNull();
+  });
+
+  test("unscored pass mixed with scored fail does not inflate scoredPassRate", () => {
+    // Regression: before fix, pass-open + scored-fail in same bucket gave 100%
+    const rows = [
+      {
+        payload: makePayload({
+          pass: true,
+          reason: "dense_unavailable",
+          scored: false,
+          real_concept_page_count: 30,
+        }),
+      },
+      {
+        payload: makePayload({
+          pass: false,
+          reason: "fail_no_signal",
+          scored: true,
+          real_concept_page_count: 40,
+        }),
+      },
+    ];
+    const result = handleMemoryV3GateStats(30, fakeDb(rows));
+    const bucket = result.buckets.find((b) => b.pageCountRange === "10–49")!;
+    expect(bucket.total).toBe(2);
+    expect(bucket.scored).toBe(1);
+    expect(bucket.passed).toBe(1); // total passes (scored + unscored)
+    // Only the scored fail counts toward the rate — no scored passes
+    expect(bucket.scoredPassRate).toBe(0);
+  });
+
+  test("rows with no real_concept_page_count go to unknownPageCount", () => {
+    const rows = [
+      {
+        payload: makePayload({
+          pass: false,
+          reason: "fail_no_signal",
+          scored: true,
+        }),
+      },
+    ];
+    const result = handleMemoryV3GateStats(30, fakeDb(rows));
+    expect(result.unknownPageCount.total).toBe(1);
+    expect(result.unknownPageCount.passed).toBe(0);
+    expect(result.unknownPageCount.reasons).toEqual({ fail_no_signal: 1 });
+    for (const b of result.buckets) {
+      expect(b.total).toBe(0);
+    }
+  });
+
+  test("mixed rows across multiple buckets are aggregated correctly", () => {
+    const rows: FakeRow[] = [
+      // 0–9 bucket: 1 unscored pass
+      {
+        payload: makePayload({
+          pass: true,
+          reason: "dense_disabled",
+          scored: false,
+          real_concept_page_count: 3,
+        }),
+      },
+      // 10–49 bucket: 1 pass, 1 fail (both scored)
+      {
+        payload: makePayload({
+          pass: true,
+          reason: "dense_pass",
+          scored: true,
+          real_concept_page_count: 20,
+        }),
+      },
+      {
+        payload: makePayload({
+          pass: false,
+          reason: "fail_no_signal",
+          scored: true,
+          real_concept_page_count: 45,
+        }),
+      },
+      // 50–199 bucket: 2 passes
+      {
+        payload: makePayload({
+          pass: true,
+          reason: "sparse_only_strong",
+          scored: true,
+          real_concept_page_count: 100,
+        }),
+      },
+      {
+        payload: makePayload({
+          pass: true,
+          reason: "dense_pass",
+          scored: true,
+          real_concept_page_count: 150,
+        }),
+      },
+    ];
+
+    const result = handleMemoryV3GateStats(30, fakeDb(rows));
+    expect(result.totalRuns).toBe(5);
+
+    const b09 = result.buckets.find((b) => b.pageCountRange === "0–9")!;
+    expect(b09.total).toBe(1);
+    expect(b09.scored).toBe(0);
+    expect(b09.scoredPassRate).toBeNull();
+
+    const b1049 = result.buckets.find((b) => b.pageCountRange === "10–49")!;
+    expect(b1049.total).toBe(2);
+    expect(b1049.scored).toBe(2);
+    expect(b1049.passed).toBe(1);
+    expect(b1049.scoredPassRate).toBe(0.5);
+    expect(b1049.reasons).toEqual({ dense_pass: 1, fail_no_signal: 1 });
+
+    const b50199 = result.buckets.find((b) => b.pageCountRange === "50–199")!;
+    expect(b50199.scored).toBe(2);
+    expect(b50199.passed).toBe(2);
+    expect(b50199.scoredPassRate).toBe(1);
+  });
+
+  test("lookbackDays is clamped to 1–90", () => {
+    const r1 = handleMemoryV3GateStats(0, fakeDb([]));
+    expect(r1.lookbackDays).toBe(1);
+    const r2 = handleMemoryV3GateStats(999, fakeDb([]));
+    expect(r2.lookbackDays).toBe(90);
+    const r3 = handleMemoryV3GateStats(30, fakeDb([]));
+    expect(r3.lookbackDays).toBe(30);
+  });
+
+  test("malformed JSON payload rows are skipped", () => {
+    const rows: FakeRow[] = [
+      { payload: "not-valid-json" },
+      {
+        payload: makePayload({
+          pass: true,
+          reason: "dense_pass",
+          scored: true,
+          real_concept_page_count: 25,
+        }),
+      },
+    ];
+    const result = handleMemoryV3GateStats(30, fakeDb(rows));
+    // Only the valid row is counted
+    expect(result.totalRuns).toBe(2); // totalRuns counts raw rows before parse
+    const bucket = result.buckets.find((b) => b.pageCountRange === "10–49")!;
+    expect(bucket.total).toBe(1);
+  });
+});

package/src/plugins/defaults/memory/src/memory-v3-routes.ts

@@ -19,6 +19,7 @@ import { z } from "zod";
 
 import { getConfig } from "../../../../config/loader.js";
 import type { AssistantConfig } from "../../../../config/types.js";
+import { getTelemetrySqlite } from "../../../../persistence/db-connection.js";
 import {
   ACTOR_PRINCIPALS,
   type RoutePolicy,
@@ -26,6 +27,7 @@ import {
 import type { RouteDefinition } from "../../../../runtime/routes/types.js";
 import { getLogger } from "../logging.js";
 import { backfillAllSections } from "../v3/maintain-job.js";
+import { MEMORY_V3_INJECTION_GATE_CHECK_NAME } from "../v3/orchestrate.js";
 import { invalidateLanes } from "../v3/shadow-plugin.js";
 
 const log = getLogger("memory-v3-routes");
@@ -101,6 +103,211 @@ const WRITE_POLICY: RoutePolicy = {
   allowedPrincipalTypes: ACTOR_PRINCIPALS,
 };
 
+// ---------------------------------------------------------------------------
+// gate-stats
+// ---------------------------------------------------------------------------
+
+const DEFAULT_LOOKBACK_DAYS = 30;
+const MAX_LOOKBACK_DAYS = 90;
+const MS_PER_DAY = 24 * 60 * 60 * 1000;
+
+/** Corpus size buckets for gate-stats aggregation (in concept pages). */
+const CORPUS_BUCKETS = [
+  { label: "0–9", min: 0, max: 9 },
+  { label: "10–49", min: 10, max: 49 },
+  { label: "50–199", min: 50, max: 199 },
+  { label: "200+", min: 200, max: Infinity },
+] as const;
+
+type BucketLabel = (typeof CORPUS_BUCKETS)[number]["label"];
+
+interface BucketAcc {
+  total: number;
+  scored: number;
+  passed: number;
+  /** Passes that came from scored runs — the numerator for scoredPassRate. */
+  scoredPassed: number;
+  reasons: Record<string, number>;
+}
+
+function corpusBucketLabel(pageCount: number): BucketLabel {
+  for (const b of CORPUS_BUCKETS) {
+    if (pageCount <= b.max) {
+      return b.label;
+    }
+  }
+  return "200+";
+}
+
+interface GatePayloadDetail {
+  pass?: boolean;
+  reason?: string;
+  scored?: boolean;
+  real_concept_page_count?: number;
+}
+
+const GateStatsBucketSchema = z.object({
+  pageCountRange: z
+    .string()
+    .describe("Concept page count range (e.g. '10–49')"),
+  total: z
+    .number()
+    .describe("All gate runs in this bucket (scored + pass-open)"),
+  scored: z
+    .number()
+    .describe("Runs that weighed scores (dense lane available)"),
+  passed: z.number().describe("Runs that passed the gate"),
+  scoredPassRate: z
+    .number()
+    .nullable()
+    .describe(
+      "scoredPasses / scored (contested decisions only); null when no scored runs in bucket",
+    ),
+  reasons: z
+    .record(z.string(), z.number())
+    .describe("Gate reason code → run count (dense_pass, fail_no_signal, …)"),
+});
+
+export const GateStatsResponseSchema = z.object({
+  lookbackDays: z.number().describe("Lookback window applied"),
+  totalRuns: z.number().describe("Total gate runs found in the window"),
+  buckets: z
+    .array(GateStatsBucketSchema)
+    .describe("Stats by concept page count range, lean→large"),
+  unknownPageCount: z
+    .object({
+      total: z.number(),
+      passed: z.number(),
+      reasons: z.record(z.string(), z.number()),
+    })
+    .describe("Runs with no real_concept_page_count in telemetry detail"),
+});
+
+export type GateStatsResponse = z.infer<typeof GateStatsResponseSchema>;
+
+/**
+ * Aggregate memory v3 injection gate runs from the telemetry outbox for the
+ * given lookback window. Groups runs by concept page count bucket and computes
+ * pass rates and reason distributions per bucket.
+ *
+ * Pass-open shortcuts (dense disabled/unavailable, gate threw) are included in
+ * `total` but not in `scored`, so `scoredPassRate` reflects only contested gate
+ * decisions — the signal that actually reveals whether the gate thresholds are
+ * calibrated correctly.
+ */
+export function handleMemoryV3GateStats(
+  lookbackDays: number = DEFAULT_LOOKBACK_DAYS,
+  db: ReturnType<typeof getTelemetrySqlite> = getTelemetrySqlite(),
+): GateStatsResponse {
+  const days = Math.min(
+    Math.max(1, Math.floor(lookbackDays)),
+    MAX_LOOKBACK_DAYS,
+  );
+  const since = Date.now() - days * MS_PER_DAY;
+
+  const emptyBuckets = (): GateStatsResponse["buckets"] =>
+    CORPUS_BUCKETS.map((b) => ({
+      pageCountRange: b.label,
+      total: 0,
+      scored: 0,
+      passed: 0,
+      scoredPassRate: null,
+      reasons: {},
+    }));
+  if (!db) {
+    return {
+      lookbackDays: days,
+      totalRuns: 0,
+      buckets: emptyBuckets(),
+      unknownPageCount: { total: 0, passed: 0, reasons: {} },
+    };
+  }
+
+  const rows = db
+    .query(
+      /*sql*/ `
+      SELECT payload FROM telemetry_events
+      WHERE name = 'watchdog'
+        AND created_at >= ?
+        AND json_extract(payload, '$.check_name') = ?
+    `,
+    )
+    .all(since, MEMORY_V3_INJECTION_GATE_CHECK_NAME) as Array<{
+    payload: string;
+  }>;
+
+  const bucketMap = new Map<BucketLabel, BucketAcc>(
+    CORPUS_BUCKETS.map((b) => [
+      b.label,
+      { total: 0, scored: 0, passed: 0, scoredPassed: 0, reasons: {} },
+    ]),
+  );
+  const unknown: BucketAcc = {
+    total: 0,
+    scored: 0,
+    passed: 0,
+    scoredPassed: 0,
+    reasons: {},
+  };
+
+  for (const row of rows) {
+    let detail: GatePayloadDetail;
+    try {
+      const parsed = JSON.parse(row.payload) as { detail?: GatePayloadDetail };
+      detail = parsed.detail ?? {};
+    } catch {
+      continue;
+    }
+
+    const pageCount =
+      typeof detail.real_concept_page_count === "number"
+        ? detail.real_concept_page_count
+        : null;
+
+    const acc =
+      pageCount !== null
+        ? (bucketMap.get(corpusBucketLabel(pageCount)) ?? unknown)
+        : unknown;
+
+    acc.total += 1;
+    if (detail.scored === true) {
+      acc.scored += 1;
+      if (detail.pass === true) {
+        acc.scoredPassed += 1;
+      }
+    }
+    if (detail.pass === true) {
+      acc.passed += 1;
+    }
+    const reason =
+      typeof detail.reason === "string" ? detail.reason : "unknown";
+    acc.reasons[reason] = (acc.reasons[reason] ?? 0) + 1;
+  }
+
+  const buckets = CORPUS_BUCKETS.map((b) => {
+    const acc = bucketMap.get(b.label)!;
+    return {
+      pageCountRange: b.label,
+      total: acc.total,
+      scored: acc.scored,
+      passed: acc.passed,
+      scoredPassRate: acc.scored > 0 ? acc.scoredPassed / acc.scored : null,
+      reasons: acc.reasons,
+    };
+  });
+
+  return {
+    lookbackDays: days,
+    totalRuns: rows.length,
+    buckets,
+    unknownPageCount: {
+      total: unknown.total,
+      passed: unknown.passed,
+      reasons: unknown.reasons,
+    },
+  };
+}
+
 export const ROUTES: RouteDefinition[] = [
   {
     operationId: "memory_v3_rebuild_index",

package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts

@@ -197,6 +197,7 @@ const { memoryV2ConsolidateJob, CONSOLIDATION_FAILURE_CHECKPOINT_KEY } =
 const { CUTOFF_PLACEHOLDER, CONSOLIDATION_PROMPT } =
   await import("../prompts/consolidation.js");
 const { invalidatePageIndex } = await import("../page-index.js");
+const { formatRememberEntry } = await import("../../buffer-format.js");
 
 // The handler only reads `config.memory.enabled`, `config.memory.v2.enabled`,
 // `config.memory.v2.consolidation_prompt_path`, and
@@ -437,6 +438,38 @@ describe("memoryV2ConsolidateJob — chunked cutoff (consolidation_max_entries_p
     expect(outcome.cutoff).not.toBe("[meeting-notes]");
   });
 
+  test("a fact whose body holds a canonical entry line counts once and never supplies the cutoff", async () => {
+    // Written through the real writer, so this covers the round trip rather
+    // than a hand-built buffer: the body line carries the exact shape an entry
+    // opening has. Counting it would both inflate the per-run budget and let
+    // the cutoff be drawn from inside a fact, deferring entries against a
+    // timestamp the user never filed anything at.
+    writeFileSync(
+      bufferPath(),
+      formatRememberEntry(
+        "Alice's rollout plan:\n- [Apr 28, 9:00 AM] cutover",
+        new Date(2026, 3, 27, 9, 0),
+      ) +
+        formatRememberEntry(
+          "Bob takes his coffee black.",
+          new Date(2026, 3, 27, 9, 1),
+        ),
+    );
+
+    const outcome = await memoryV2ConsolidateJob(
+      makeJob(),
+      configWithMaxEntries(2),
+    );
+
+    expect(outcome.kind).toBe("invoked");
+    if (outcome.kind !== "invoked") {
+      throw new Error("unreachable");
+    }
+    // Two facts, cap 2 → no chunking. Three would chunk and defer one.
+    expect(outcome.deferredEntries).toBe(0);
+    expect(outcome.cutoff).not.toBe("Apr 28, 9:00 AM");
+  });
+
   test("multi-line entries under the cap → full-buffer cutoff even when raw line count exceeds it", async () => {
     writeFileSync(
       bufferPath(),