@vellumai/assistant 0.11.3 → 0.11.4-staging.2

package/src/plugins/defaults/memory/src/memory-v3-routes.ts

@@ -19,6 +19,7 @@ import { z } from "zod";
 
 import { getConfig } from "../../../../config/loader.js";
 import type { AssistantConfig } from "../../../../config/types.js";
+import { getTelemetrySqlite } from "../../../../persistence/db-connection.js";
 import {
   ACTOR_PRINCIPALS,
   type RoutePolicy,
@@ -26,6 +27,7 @@ import {
 import type { RouteDefinition } from "../../../../runtime/routes/types.js";
 import { getLogger } from "../logging.js";
 import { backfillAllSections } from "../v3/maintain-job.js";
+import { MEMORY_V3_INJECTION_GATE_CHECK_NAME } from "../v3/orchestrate.js";
 import { invalidateLanes } from "../v3/shadow-plugin.js";
 
 const log = getLogger("memory-v3-routes");
@@ -101,6 +103,211 @@ const WRITE_POLICY: RoutePolicy = {
   allowedPrincipalTypes: ACTOR_PRINCIPALS,
 };
 
+// ---------------------------------------------------------------------------
+// gate-stats
+// ---------------------------------------------------------------------------
+
+const DEFAULT_LOOKBACK_DAYS = 30;
+const MAX_LOOKBACK_DAYS = 90;
+const MS_PER_DAY = 24 * 60 * 60 * 1000;
+
+/** Corpus size buckets for gate-stats aggregation (in concept pages). */
+const CORPUS_BUCKETS = [
+  { label: "0–9", min: 0, max: 9 },
+  { label: "10–49", min: 10, max: 49 },
+  { label: "50–199", min: 50, max: 199 },
+  { label: "200+", min: 200, max: Infinity },
+] as const;
+
+type BucketLabel = (typeof CORPUS_BUCKETS)[number]["label"];
+
+interface BucketAcc {
+  total: number;
+  scored: number;
+  passed: number;
+  /** Passes that came from scored runs — the numerator for scoredPassRate. */
+  scoredPassed: number;
+  reasons: Record<string, number>;
+}
+
+function corpusBucketLabel(pageCount: number): BucketLabel {
+  for (const b of CORPUS_BUCKETS) {
+    if (pageCount <= b.max) {
+      return b.label;
+    }
+  }
+  return "200+";
+}
+
+interface GatePayloadDetail {
+  pass?: boolean;
+  reason?: string;
+  scored?: boolean;
+  real_concept_page_count?: number;
+}
+
+const GateStatsBucketSchema = z.object({
+  pageCountRange: z
+    .string()
+    .describe("Concept page count range (e.g. '10–49')"),
+  total: z
+    .number()
+    .describe("All gate runs in this bucket (scored + pass-open)"),
+  scored: z
+    .number()
+    .describe("Runs that weighed scores (dense lane available)"),
+  passed: z.number().describe("Runs that passed the gate"),
+  scoredPassRate: z
+    .number()
+    .nullable()
+    .describe(
+      "scoredPasses / scored (contested decisions only); null when no scored runs in bucket",
+    ),
+  reasons: z
+    .record(z.string(), z.number())
+    .describe("Gate reason code → run count (dense_pass, fail_no_signal, …)"),
+});
+
+export const GateStatsResponseSchema = z.object({
+  lookbackDays: z.number().describe("Lookback window applied"),
+  totalRuns: z.number().describe("Total gate runs found in the window"),
+  buckets: z
+    .array(GateStatsBucketSchema)
+    .describe("Stats by concept page count range, lean→large"),
+  unknownPageCount: z
+    .object({
+      total: z.number(),
+      passed: z.number(),
+      reasons: z.record(z.string(), z.number()),
+    })
+    .describe("Runs with no real_concept_page_count in telemetry detail"),
+});
+
+export type GateStatsResponse = z.infer<typeof GateStatsResponseSchema>;
+
+/**
+ * Aggregate memory v3 injection gate runs from the telemetry outbox for the
+ * given lookback window. Groups runs by concept page count bucket and computes
+ * pass rates and reason distributions per bucket.
+ *
+ * Pass-open shortcuts (dense disabled/unavailable, gate threw) are included in
+ * `total` but not in `scored`, so `scoredPassRate` reflects only contested gate
+ * decisions — the signal that actually reveals whether the gate thresholds are
+ * calibrated correctly.
+ */
+export function handleMemoryV3GateStats(
+  lookbackDays: number = DEFAULT_LOOKBACK_DAYS,
+  db: ReturnType<typeof getTelemetrySqlite> = getTelemetrySqlite(),
+): GateStatsResponse {
+  const days = Math.min(
+    Math.max(1, Math.floor(lookbackDays)),
+    MAX_LOOKBACK_DAYS,
+  );
+  const since = Date.now() - days * MS_PER_DAY;
+
+  const emptyBuckets = (): GateStatsResponse["buckets"] =>
+    CORPUS_BUCKETS.map((b) => ({
+      pageCountRange: b.label,
+      total: 0,
+      scored: 0,
+      passed: 0,
+      scoredPassRate: null,
+      reasons: {},
+    }));
+  if (!db) {
+    return {
+      lookbackDays: days,
+      totalRuns: 0,
+      buckets: emptyBuckets(),
+      unknownPageCount: { total: 0, passed: 0, reasons: {} },
+    };
+  }
+
+  const rows = db
+    .query(
+      /*sql*/ `
+      SELECT payload FROM telemetry_events
+      WHERE name = 'watchdog'
+        AND created_at >= ?
+        AND json_extract(payload, '$.check_name') = ?
+    `,
+    )
+    .all(since, MEMORY_V3_INJECTION_GATE_CHECK_NAME) as Array<{
+    payload: string;
+  }>;
+
+  const bucketMap = new Map<BucketLabel, BucketAcc>(
+    CORPUS_BUCKETS.map((b) => [
+      b.label,
+      { total: 0, scored: 0, passed: 0, scoredPassed: 0, reasons: {} },
+    ]),
+  );
+  const unknown: BucketAcc = {
+    total: 0,
+    scored: 0,
+    passed: 0,
+    scoredPassed: 0,
+    reasons: {},
+  };
+
+  for (const row of rows) {
+    let detail: GatePayloadDetail;
+    try {
+      const parsed = JSON.parse(row.payload) as { detail?: GatePayloadDetail };
+      detail = parsed.detail ?? {};
+    } catch {
+      continue;
+    }
+
+    const pageCount =
+      typeof detail.real_concept_page_count === "number"
+        ? detail.real_concept_page_count
+        : null;
+
+    const acc =
+      pageCount !== null
+        ? (bucketMap.get(corpusBucketLabel(pageCount)) ?? unknown)
+        : unknown;
+
+    acc.total += 1;
+    if (detail.scored === true) {
+      acc.scored += 1;
+      if (detail.pass === true) {
+        acc.scoredPassed += 1;
+      }
+    }
+    if (detail.pass === true) {
+      acc.passed += 1;
+    }
+    const reason =
+      typeof detail.reason === "string" ? detail.reason : "unknown";
+    acc.reasons[reason] = (acc.reasons[reason] ?? 0) + 1;
+  }
+
+  const buckets = CORPUS_BUCKETS.map((b) => {
+    const acc = bucketMap.get(b.label)!;
+    return {
+      pageCountRange: b.label,
+      total: acc.total,
+      scored: acc.scored,
+      passed: acc.passed,
+      scoredPassRate: acc.scored > 0 ? acc.scoredPassed / acc.scored : null,
+      reasons: acc.reasons,
+    };
+  });
+
+  return {
+    lookbackDays: days,
+    totalRuns: rows.length,
+    buckets,
+    unknownPageCount: {
+      total: unknown.total,
+      passed: unknown.passed,
+      reasons: unknown.reasons,
+    },
+  };
+}
+
 export const ROUTES: RouteDefinition[] = [
   {
     operationId: "memory_v3_rebuild_index",

package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts

@@ -197,6 +197,7 @@ const { memoryV2ConsolidateJob, CONSOLIDATION_FAILURE_CHECKPOINT_KEY } =
 const { CUTOFF_PLACEHOLDER, CONSOLIDATION_PROMPT } =
   await import("../prompts/consolidation.js");
 const { invalidatePageIndex } = await import("../page-index.js");
+const { formatRememberEntry } = await import("../../buffer-format.js");
 
 // The handler only reads `config.memory.enabled`, `config.memory.v2.enabled`,
 // `config.memory.v2.consolidation_prompt_path`, and
@@ -437,6 +438,38 @@ describe("memoryV2ConsolidateJob — chunked cutoff (consolidation_max_entries_p
     expect(outcome.cutoff).not.toBe("[meeting-notes]");
   });
 
+  test("a fact whose body holds a canonical entry line counts once and never supplies the cutoff", async () => {
+    // Written through the real writer, so this covers the round trip rather
+    // than a hand-built buffer: the body line carries the exact shape an entry
+    // opening has. Counting it would both inflate the per-run budget and let
+    // the cutoff be drawn from inside a fact, deferring entries against a
+    // timestamp the user never filed anything at.
+    writeFileSync(
+      bufferPath(),
+      formatRememberEntry(
+        "Alice's rollout plan:\n- [Apr 28, 9:00 AM] cutover",
+        new Date(2026, 3, 27, 9, 0),
+      ) +
+        formatRememberEntry(
+          "Bob takes his coffee black.",
+          new Date(2026, 3, 27, 9, 1),
+        ),
+    );
+
+    const outcome = await memoryV2ConsolidateJob(
+      makeJob(),
+      configWithMaxEntries(2),
+    );
+
+    expect(outcome.kind).toBe("invoked");
+    if (outcome.kind !== "invoked") {
+      throw new Error("unreachable");
+    }
+    // Two facts, cap 2 → no chunking. Three would chunk and defer one.
+    expect(outcome.deferredEntries).toBe(0);
+    expect(outcome.cutoff).not.toBe("Apr 28, 9:00 AM");
+  });
+
   test("multi-line entries under the cap → full-buffer cutoff even when raw line count exceeds it", async () => {
     writeFileSync(
       bufferPath(),

package/src/plugins/defaults/memory/substrate/__tests__/static-context.test.ts

@@ -5,6 +5,7 @@
  *   - Reads the four files in canonical order and joins them under headings.
  *   - Skips empty / missing files.
  *   - Returns null when every file is empty or missing.
+ *   - Caps the Buffer section at `consolidation_max_buffer_lines`.
  */
 
 import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
@@ -33,12 +34,19 @@ mock.module("../../../../../util/logger.js", () => ({
 
 let configMemoryV2Enabled = true;
 let configMemoryEnabled = true;
+// Mirrors the `memory.v2.consolidation_max_buffer_lines` schema default; the
+// Buffer-cap tests move it.
+let configMaxBufferLines: number | null = 100;
 
-// static-context reads its gates through the plugin's own config accessor.
+// static-context reads its gates and the substrate tuning through the
+// plugin's own config accessor.
 mock.module("../../config.js", () => ({
   getMemoryConfig: () => ({
     enabled: configMemoryEnabled,
-    v2: { enabled: configMemoryV2Enabled },
+    v2: {
+      enabled: configMemoryV2Enabled,
+      consolidation_max_buffer_lines: configMaxBufferLines,
+    },
   }),
 }));
 
@@ -70,6 +78,7 @@ describe("readMemoryV2StaticContent", () => {
     mkdirSync(TEST_DIR, { recursive: true });
     configMemoryV2Enabled = true;
     configMemoryEnabled = true;
+    configMaxBufferLines = 100;
   });
 
   afterEach(() => {
@@ -165,6 +174,194 @@ describe("readMemoryV2StaticContent", () => {
   });
 });
 
+describe("readMemoryV2StaticContent Buffer cap", () => {
+  /** `n` timestamped buffer entries, oldest first, in `remember()`'s shape. */
+  function bufferEntries(n: number): string {
+    return Array.from(
+      { length: n },
+      (_, i) => `- [Jan 1, 9:00 AM] entry-${i}`,
+    ).join("\n");
+  }
+
+  beforeEach(() => {
+    mkdirSync(TEST_DIR, { recursive: true });
+    configMemoryV2Enabled = true;
+    configMemoryEnabled = true;
+    configMaxBufferLines = 10;
+  });
+
+  afterEach(() => {
+    cleanupMemoryDir();
+  });
+
+  test("passes a buffer at or under the cap through byte-identically", () => {
+    const buffer = bufferEntries(10);
+    writeMemoryFile("buffer.md", buffer);
+
+    expect(readMemoryV2StaticContent()).toBe(`## Buffer\n\n${buffer}`);
+  });
+
+  test("keeps the newest entries and drops the oldest when over the cap", () => {
+    writeMemoryFile("buffer.md", bufferEntries(30));
+
+    const text = readMemoryV2StaticContent()!;
+    expect(text).toContain(
+      "(Older entries trimmed. Read memory/buffer.md for the full backlog.)",
+    );
+    // The cap counts non-empty lines, matching the scheduler's
+    // `countBufferLines`, so exactly the newest 10 entries survive.
+    const kept = text
+      .split("\n")
+      .filter((line) => line.startsWith("- ["))
+      .map((line) => line.slice(line.lastIndexOf(" ") + 1));
+    expect(kept).toEqual(
+      Array.from({ length: 10 }, (_, i) => `entry-${i + 20}`),
+    );
+  });
+
+  test("never opens mid-entry when the cap lands inside a multiline fact", () => {
+    // The 5-line fact straddles the cap: counting back 10 non-empty lines
+    // lands on one of its continuation lines. Injecting from there would show
+    // orphan bullets with no timestamp and no opening clause.
+    const multiline = [
+      "- [Jan 1, 9:00 AM] the straddling fact",
+      "  - [ ] a checklist item inside the fact",
+      "  continuation prose",
+      "  - another bullet",
+      "  closing line",
+    ].join("\n");
+    writeMemoryFile(
+      "buffer.md",
+      `${bufferEntries(20)}\n${multiline}\n${bufferEntries(8)}`,
+    );
+
+    const text = readMemoryV2StaticContent()!;
+    const body = text.slice(text.indexOf("buffer.md for the full backlog.)\n"));
+    const firstLine = body.split("\n")[1]!;
+    expect(firstLine).toMatch(/^- \[Jan 1, 9:00 AM\]/);
+    // The partial entry is dropped whole: none of its continuation lines
+    // survive on their own.
+    expect(text).not.toContain("continuation prose");
+    expect(text).not.toContain("the straddling fact");
+  });
+
+  test("keeps the newest entry attributable when its body alone exceeds the cap", () => {
+    // The newest entry has no successor to fall back to: counting back 10
+    // non-empty lines lands inside its body and there is no later timestamped
+    // entry to open on. Dropping it would leave the section with no facts at
+    // all, so the entry's opening line is kept and its head elided.
+    writeMemoryFile(
+      "buffer.md",
+      [
+        ...Array.from(
+          { length: 20 },
+          (_, i) => `- [Jan 1, 9:00 AM] entry-${i}`,
+        ),
+        "- [Jan 2, 9:00 AM] the oversized fact",
+        ...Array.from({ length: 14 }, (_, i) => `  body line ${i}`),
+      ].join("\n"),
+    );
+
+    const text = readMemoryV2StaticContent()!;
+    const body = text
+      .slice(text.indexOf("full backlog.)\n") + "full backlog.)\n".length)
+      .split("\n")
+      .filter((line) => line.trim().length > 0);
+
+    // Opens on the entry's own timestamped line, never on an orphan body line.
+    expect(body[0]).toBe("- [Jan 2, 9:00 AM] the oversized fact");
+    expect(body[1]).toBe(
+      "(This entry's body was trimmed. Read memory/buffer.md for the rest of it.)",
+    );
+    // The retained tail is the newest end of the body, not its head.
+    expect(body.at(-1)).toBe("  body line 13");
+    expect(body).toContain("  body line 4");
+    expect(body).not.toContain("  body line 3");
+    // Bounded at the cap plus exactly the opening line and the marker, so the
+    // oversized entry cannot reintroduce an unbounded injection.
+    expect(body).toHaveLength(12);
+    // The older entries are still dropped.
+    expect(text).not.toContain("entry-19");
+  });
+
+  test("does not claim a trim when the newest entry's body exactly fills the cap", () => {
+    // The cut lands on the line right after the opening, so nothing of the
+    // entry was elided and the marker would be a lie.
+    writeMemoryFile(
+      "buffer.md",
+      [
+        ...Array.from(
+          { length: 20 },
+          (_, i) => `- [Jan 1, 9:00 AM] entry-${i}`,
+        ),
+        "- [Jan 2, 9:00 AM] the exact-fit fact",
+        ...Array.from({ length: 10 }, (_, i) => `  body line ${i}`),
+      ].join("\n"),
+    );
+
+    const text = readMemoryV2StaticContent()!;
+    const body = text
+      .slice(text.indexOf("full backlog.)\n") + "full backlog.)\n".length)
+      .split("\n")
+      .filter((line) => line.trim().length > 0);
+
+    expect(body[0]).toBe("- [Jan 2, 9:00 AM] the exact-fit fact");
+    expect(text).not.toContain("This entry's body was trimmed");
+    // The entry survives whole: opening line plus its full 10-line body.
+    expect(body).toHaveLength(11);
+    expect(body.at(-1)).toBe("  body line 9");
+    expect(body).toContain("  body line 0");
+  });
+
+  test("keeps a multiline entry intact when it sits fully inside the cap", () => {
+    const multiline = [
+      "- [Jan 1, 9:00 AM] a fact with a body",
+      "  second line of the same fact",
+    ].join("\n");
+    writeMemoryFile("buffer.md", `${bufferEntries(20)}\n${multiline}`);
+
+    const text = readMemoryV2StaticContent()!;
+    expect(text).toContain("a fact with a body");
+    expect(text).toContain("second line of the same fact");
+  });
+
+  test("falls back to the line cut for a buffer with no timestamped entries", () => {
+    // A hand-written buffer that never went through `remember()` has no entry
+    // structure to preserve, so the line-based cut stands rather than dropping
+    // everything.
+    const handWritten = Array.from(
+      { length: 30 },
+      (_, i) => `just a line ${i}`,
+    ).join("\n");
+    writeMemoryFile("buffer.md", handWritten);
+
+    const text = readMemoryV2StaticContent()!;
+    expect(text).toContain("just a line 29");
+    expect(text).not.toContain("just a line 0");
+  });
+
+  test("leaves the buffer unbounded when the size trigger is disabled", () => {
+    configMaxBufferLines = null;
+    const buffer = bufferEntries(30);
+    writeMemoryFile("buffer.md", buffer);
+
+    expect(readMemoryV2StaticContent()).toBe(`## Buffer\n\n${buffer}`);
+  });
+
+  test("caps only the Buffer section, never the curated views", () => {
+    const long = Array.from({ length: 30 }, (_, i) => `line-${i}`).join("\n");
+    writeMemoryFile("essentials.md", long);
+    writeMemoryFile("threads.md", long);
+    writeMemoryFile("recent.md", long);
+    writeMemoryFile("buffer.md", bufferEntries(30));
+
+    const lines = readMemoryV2StaticContent()!.split("\n");
+    expect(lines.filter((line) => line === "line-0")).toHaveLength(3);
+    expect(lines.filter((line) => line === "line-29")).toHaveLength(3);
+    expect(lines.some((line) => line.endsWith("entry-0"))).toBe(false);
+  });
+});
+
 describe("shouldExposePersonalMemory", () => {
   test("allows guardian-trusted local conversations", () => {
     expect(

package/src/plugins/defaults/memory/substrate/consolidation-job.ts

@@ -91,7 +91,10 @@ import {
   type MemoryJobType,
 } from "../../../../persistence/jobs-store.js";
 import { runBackgroundJob } from "../../../../runtime/background-job-runner.js";
-import { formatBufferTimestamp } from "../graph/tool-handlers.js";
+import {
+  formatBufferTimestamp,
+  matchBufferEntryStart,
+} from "../buffer-format.js";
 import { getLogger } from "../logging.js";
 import { getWorkspaceDir } from "../paths.js";
 import {
@@ -597,32 +600,36 @@ function readBufferContent(bufferPath: string): string {
 
 /**
  * Extract the bracketed timestamp from a `buffer.md` entry line
- * (`- [Mon D, h:mm AM/PM] …`, see `formatBufferEntry`). Returned verbatim so
- * it can serve directly as a consolidation cutoff — both sides of the agent's
- * "timestamp ≥ cutoff" comparison then share the exact `formatBufferTimestamp`
- * shape.
+ * (`- [Mon D, h:mm AM/PM] …`, see {@link formatRememberEntry}). Returned
+ * verbatim so it can serve directly as a consolidation cutoff: both sides of
+ * the agent's "timestamp >= cutoff" comparison then share the exact
+ * {@link formatBufferTimestamp} shape.
  *
- * The bracket contents must match that shape exactly: a remembered fact's
- * continuation lines can themselves start with `- [` (markdown checklists
- * `- [ ] …`, wikilink bullets `- [[…]]`), and counting those as entries
- * would inflate the per-run budget — or worse, hand the agent a garbage
- * cutoff like a blank string. Returns `null` for anything that isn't a real
- * timestamped entry start.
+ * Recognition is delegated to the shared matcher, so a remembered fact's
+ * continuation lines never register as entries. That matters here beyond
+ * tidiness: counting a fact's `- [ ] …` checklist lines or an indented
+ * entry-shaped body line as entries would inflate the per-run budget, or hand
+ * the agent a cutoff drawn from the middle of a fact.
  */
 function extractBufferEntryTimestamp(line: string): string | null {
-  const match = /^\s*-\s*\[([A-Z][a-z]{2} \d{1,2}, \d{1,2}:\d{2} [AP]M)\]/.exec(
-    line,
-  );
-  return match ? match[1] : null;
+  return matchBufferEntryStart(line)?.timestamp ?? null;
 }
 
 /**
  * Count non-empty lines in `memory/buffer.md`. Used by the scheduler to
  * implement the size-based consolidation trigger. Missing file → 0.
  *
- * Each entry is one line (`- [Mon D, h:mm AM/PM] …\n`), so non-empty-line
- * count == entry count for a well-formed buffer; blank lines and trailing
- * newlines don't inflate the count.
+ * Lines, deliberately, not entries. A multiline `remember()` fact is one
+ * entry spread over several lines, so the two counts diverge and each answers
+ * a different question. This trigger and the injected-Buffer cap that reuses
+ * it (`capBufferSection` in `static-context.ts`) both care about how much
+ * context the buffer costs, which scales with lines. The per-run budget
+ * `consolidation_max_entries_per_run` cares about how many facts the agent
+ * must file, so it counts entry-start lines instead.
+ *
+ * Do not "fix" this to count entries: a single 200-line fact is a context
+ * problem the trigger should fire on, even though it is one entry. Blank
+ * lines and trailing newlines don't inflate the count.
  */
 export function countBufferLines(bufferPath: string): number {
   return countNonEmptyLines(readBufferContent(bufferPath));

package/src/plugins/defaults/memory/substrate/skill-content.ts

@@ -22,6 +22,13 @@ export interface SkillCapabilityInput {
  */
 export const ALWAYS_CANDIDATE_CARD_CHARS = 900;
 
+/**
+ * Character budget for an ordinary skill's capability statement. A statement
+ * that overruns is hard-truncated, dropping its trailing activation hints and
+ * its entire avoid-when list from what the selector sees.
+ */
+export const DEFAULT_CARD_CHARS = 500;
+
 /**
  * Render the prose-style capability statement embedded into the unified
  * `memory_v2_concept_pages` Qdrant collection (under the `skills/<id>` slug
@@ -32,7 +39,7 @@ export const ALWAYS_CANDIDATE_CARD_CHARS = 900;
  */
 export function buildSkillContent(
   input: SkillCapabilityInput,
-  maxChars = 500,
+  maxChars: number = DEFAULT_CARD_CHARS,
 ): string {
   const list = maxChars > 500;
   let content = `The "${input.displayName}" skill (${input.id}) is available. ${input.description}.`;