@vellumai/assistant 0.11.3 → 0.11.4-dev.202608190019.b94dbf2

package/src/plugins/defaults/memory/graph-topology/pending-buffer.ts

@@ -18,7 +18,15 @@
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 
-import { formatRememberEntry } from "../graph/tool-handlers.js";
+import {
+  formatRememberEntry,
+  matchBufferEntryStart,
+} from "../buffer-format.js";
+import {
+  extractWikilinkTargets,
+  WIKILINK_REGEX,
+  wikilinkTarget,
+} from "../substrate/page-links.js";
 import type { MemoryGraphEdge, MemoryGraphNode } from "./types.js";
 
 /** Node id prefix marking a pending buffer entry (id = prefix + text hash). */
@@ -36,22 +44,10 @@ const MAX_PENDING_NODES = 200;
 
 const PENDING_LABEL_MAX_CHARS = 60;
 
-/**
- * Matches a real entry start: `- [Mon D, h:mm AM/PM] fact`, the exact shape
- * `formatRememberEntry` writes. The timestamp must be present and
- * timestamp-shaped — a bullet with other bracketed text (e.g. a `- [ ]`
- * checklist item inside a multiline fact) is a continuation, not an entry.
- */
-const BUFFER_ENTRY_REGEX =
-  /^-\s+\[[A-Z][a-z]{2}\s+\d{1,2},\s+\d{1,2}:\d{2}\s+[AP]M\]\s*(.*)$/;
-
 /** Lenient fallback for hand-written buffers: a plain bullet with no
  * timestamped entry seen yet still counts as an entry of its own. */
 const PLAIN_BULLET_REGEX = /^-\s+(.+)$/;
 
-/** Matches `[[slug]]` / `[[slug|label]]` wikilinks inside an entry. */
-const WIKILINK_REGEX = /\[\[([^[\]|]+)(?:\|[^\]]*)?\]\]/g;
-
 export interface PendingBufferEntry {
   /** Stable node id (`buffer:` + content hash, deduped within one parse). */
   id: string;
@@ -102,10 +98,12 @@ export function parseBufferEntries(content: string): PendingBufferEntry[] {
 
   for (const rawLine of content.split("\n")) {
     const line = rawLine.trimEnd();
-    const entryStart = BUFFER_ENTRY_REGEX.exec(line.trim());
+    // Matched against the untrimmed line: an indented entry-shaped bullet is
+    // body text inside the fact above it, not a fact of its own.
+    const entryStart = matchBufferEntryStart(line);
     if (entryStart) {
       flush();
-      current = [entryStart[1]!.trim()];
+      current = [entryStart.text];
       continue;
     }
     if (current !== null) {
@@ -120,13 +118,7 @@ export function parseBufferEntries(content: string): PendingBufferEntry[] {
   flush();
 
   for (const text of texts) {
-    const slugs: string[] = [];
-    for (const link of text.matchAll(WIKILINK_REGEX)) {
-      const slug = link[1]!.trim();
-      if (slug && !slugs.includes(slug)) {
-        slugs.push(slug);
-      }
-    }
+    const slugs = [...new Set(extractWikilinkTargets(text))];
 
     const hash = hashText(text);
     const count = (seen.get(hash) ?? 0) + 1;
@@ -148,8 +140,9 @@ export function parseBufferEntries(content: string): PendingBufferEntry[] {
 function pendingLabel(text: string): string {
   const firstLine = text.split("\n", 1)[0] ?? text;
   const plain = firstLine
-    .replace(WIKILINK_REGEX, (_match, slug: string) => {
-      return slug.trim().split("/").pop() ?? slug;
+    .replace(WIKILINK_REGEX, (_match, inner: string) => {
+      const slug = wikilinkTarget(inner);
+      return slug.split("/").pop() || slug;
     })
     .trim();
   if (plain.length <= PENDING_LABEL_MAX_CHARS) {

package/src/plugins/defaults/memory/hooks/post-compact.ts

@@ -70,10 +70,7 @@ const postCompact: HookFunction<PostCompactContext> = async (ctx) => {
   // concurrent request's actor and the contact/member registry the actor
   // context derives from can be mutated by contact tools. The snapshots are the
   // values the turn's initial assembly resolved.
-  const trust =
-    conversation?.currentTurnTrustContext ??
-    conversation?.trustContext ??
-    FALLBACK_TURN_TRUST;
+  const trust = conversation?.getTurnOrRestingTrust() ?? FALLBACK_TURN_TRUST;
   const actorContext = conversation?.currentTurnInboundActorContext ?? null;
   // Render the `model_profile:` label from the turn-start notice key, using
   // the call site self-resolved from the live conversation — the same

package/src/plugins/defaults/memory/hooks/user-prompt-submit.ts

@@ -31,7 +31,10 @@ import type {
   HookFunction,
   UserPromptSubmitContext,
 } from "@vellumai/plugin-api";
-import { updateMessageMetadata } from "@vellumai/plugin-api";
+import {
+  updateMessageMetadata,
+  VOICE_ESCALATION_CONTINUATION_MESSAGE_KIND,
+} from "@vellumai/plugin-api";
 
 import type { MemoryRecalledEvent } from "../../../../api/events/memory-recalled.js";
 import { getConfig } from "../../../../config/loader.js";
@@ -48,6 +51,7 @@ import { timeLatencySubSpan } from "../../../../daemon/turn-latency-sub-spans.js
 import { broadcastMessage } from "../../../../runtime/assistant-event-hub.js";
 import type { GraphMemoryResult } from "../graph/conversation-graph-memory.js";
 import { recordMemoryRecallLog } from "../memory-recall-log-store.js";
+import { stripTailInjectionsForReinjection } from "../tail-reinjection-strip.js";
 import { MEMORY_V3_INJECTED_BLOCK_METADATA_KEY } from "../v3/ever-injected-store.js";
 
 /**
@@ -69,6 +73,39 @@ export function shouldRunLegacyMemoryRetrieval(params: {
   return params.isTrustedActor && !params.memoryV3Live;
 }
 
+/**
+ * The voice escalation continuation is a model-control message, not a
+ * retrieval query. Route memory on the latest preceding user message and strip
+ * the runtime context that was frozen onto it during its original turn.
+ */
+function legacyRetrievalRoutingMessages(
+  messages: UserPromptSubmitContext["latestMessages"],
+  routeOnPreviousUser: boolean,
+): UserPromptSubmitContext["latestMessages"] {
+  if (!routeOnPreviousUser) {
+    return messages;
+  }
+
+  for (let i = messages.length - 2; i >= 0; i -= 1) {
+    if (messages[i]?.role !== "user") {
+      continue;
+    }
+    const candidate = stripTailInjectionsForReinjection(
+      messages.slice(0, i + 1),
+    );
+    const tail = candidate[candidate.length - 1];
+    if (
+      tail?.role === "user" &&
+      tail.content.some(
+        (block) => block.type === "text" && block.text.trim().length > 0,
+      )
+    ) {
+      return candidate;
+    }
+  }
+  return [];
+}
+
 /**
  * Persist and broadcast the retrieval's side effects: the injected block on
  * the user message's metadata (so it survives reloads), a recall-log row, and
@@ -263,10 +300,11 @@ async function persistInjectionBlocks(
  * runs for every actor, writing the fully injected result back onto
  * `latestMessages` and persisting the assembled blocks.
  *
- * Memory retrieval blocks the turn — there is no soft timeout here. Memory is
- * critical context, and silently dropping it produces a worse outcome than a
- * slower turn. Cancellation still works via `ctx.signal`, which is threaded
- * into `prepareMemory`.
+ * Memory retrieval ordinarily blocks the turn. The voice front door is the
+ * latency-sensitive exception: it keeps carried memory in the front prompt
+ * and defers current-turn retrieval to the escalated leg when one is needed.
+ * Cancellation still works via the live conversation signal for ordinary
+ * retrieval.
  */
 const userPromptSubmitMemoryRetrieval: HookFunction<
   UserPromptSubmitContext
@@ -292,9 +330,14 @@ const userPromptSubmitMemoryRetrieval: HookFunction<
   // fallback — a v3 empty/failed selection yields no NEW injected memory that
   // turn (prior turns' frozen v3 cards still ride history).
   const memoryV3Live = isMemoryV3Live(config);
+  const isVoiceFrontDoor = conversation?.currentCallSite === "voiceFrontDoor";
+  if (isVoiceFrontDoor && conversation) {
+    conversation.graphMemory.recordPkbQueryVectors(undefined, undefined);
+  }
   let v2BlockPersisted = false;
   if (
     shouldRunLegacyMemoryRetrieval({ isTrustedActor, memoryV3Live }) &&
+    !isVoiceFrontDoor &&
     conversation &&
     abortSignal
   ) {
@@ -311,6 +354,11 @@ const userPromptSubmitMemoryRetrieval: HookFunction<
           config,
           abortSignal,
           broadcastMessage,
+          legacyRetrievalRoutingMessages(
+            ctx.latestMessages,
+            ctx.isHiddenPrompt === true &&
+              ctx.messageKind === VOICE_ESCALATION_CONTINUATION_MESSAGE_KIND,
+          ),
         ),
     );
 

package/src/plugins/defaults/memory/indexer.ts

@@ -123,7 +123,9 @@ export async function indexMessageNow(
   // memory_segments has no cross-file FK to messages, so this call must not
   // leave pieces for a message with no row. Skip early when the source
   // row is already gone, the common case of a backfill job running after the
-  // message was deleted. A post-write re-check below closes the narrower window
+  // message was deleted. Any-state existence check, deliberately: a
+  // streaming row exists, and orphan prevention is about deletion, not
+  // completeness. A post-write re-check below closes the narrower window
   // where the delete lands mid-transaction.
   const sourceMessage = getDb()
     .select({ id: messages.id })

package/src/plugins/defaults/memory/memory-retrospective-accounting.ts

@@ -52,18 +52,30 @@ export function isSkillCardMessage(row: { metadata: string | null }): boolean {
 }
 
 /**
- * `getMessagesAfter` minus skill-card rows. The retrospective job's
- * new-message slice: a card-only tail yields an empty slice (the job's
- * `no_new_messages` early return), and a mixed tail's cutoff lands on the
- * last real message rather than the card.
+ * `getMessagesAfter` truncated at the first unfinalized row, minus skill-card
+ * rows. The retrospective job's new-message slice: a card-only tail yields an
+ * empty slice (the job's `no_new_messages` early return), and a mixed tail's
+ * cutoff lands on the last real message rather than the card.
+ *
+ * The slice STOPS BEFORE the first unfinalized row rather than filtering it
+ * out. The job's cutoff (and therefore `lastProcessedMessageId` on success)
+ * is chosen from this slice, while the fork the job reviews skips unfinalized
+ * rows, so the cursor must never move past one: filtering through a stale
+ * mid-slice `finalized = 0` row would advance the cursor beyond it, and once
+ * the row later finalizes its `(createdAt, id)` sits behind the cursor,
+ * unreviewed forever. Truncating parks the cursor before the unfinalized row;
+ * the rows behind it are reviewed by the retrospective that runs after the
+ * row resolves (finalizes, or is repaired away by crash recovery).
  */
 export function getRetrospectiveMessagesAfter(
   conversationId: string,
   afterMessageId: string | null,
 ): MessageRow[] {
-  return getMessagesAfter(conversationId, afterMessageId).filter(
-    (row) => !isSkillCardMessage(row),
-  );
+  const rows = getMessagesAfter(conversationId, afterMessageId);
+  const firstUnfinalized = rows.findIndex((row) => row.finalized !== 1);
+  const bounded =
+    firstUnfinalized === -1 ? rows : rows.slice(0, firstUnfinalized);
+  return bounded.filter((row) => !isSkillCardMessage(row));
 }
 
 /**

package/src/plugins/defaults/memory/memory-retrospective-job.ts

@@ -702,10 +702,10 @@ interface SourceParityPins {
  * parity; no system-prompt section branches on the flag, so this pin does not
  * affect prompt output) and the tool-context pin (the live consumer, gating
  * tool availability), using the live-turn derivation: interactive
- * interfaces run `updateClient(_, false)` (`hasNoClient = false`), while
- * channel-routed and chrome-extension turns stay clientless (`true`) — the
- * exact `isInteractiveInterface` predicate `conversation-routes.ts` /
- * `process-message.ts` apply. Pinned explicitly even when it matches the
+ * interfaces run their turns with `isInteractive: true` (`hasNoClient` reads
+ * `false` for the turn), while channel-routed and chrome-extension turns stay
+ * clientless (`true`) — the exact `isInteractiveInterface` predicate
+ * `conversation-routes.ts` / `process-message.ts` apply. Pinned explicitly even when it matches the
  * fork's hydrated value (`true`) so the parity contract doesn't depend on
  * hydration defaults.
  *

package/src/plugins/defaults/memory/src/__tests__/memory-v3-gate-stats.test.ts

@@ -0,0 +1,281 @@
+/**
+ * Tests for `handleMemoryV3GateStats` in `memory-v3-routes.ts`.
+ *
+ * The handler aggregates gate telemetry from the telemetry DB and buckets it
+ * by concept page count. These tests cover:
+ *   - empty DB → zero counts, all buckets present, scoredPassRate null
+ *   - mixed rows land in the correct buckets by real_concept_page_count
+ *   - scored vs unscored runs are counted separately
+ *   - rows with no real_concept_page_count go to unknownPageCount
+ *   - lookbackDays is clamped to [1, 90]
+ *   - malformed JSON rows are skipped
+ *   - null/absent DB returns a zero response
+ */
+
+import type { Database } from "bun:sqlite";
+import { describe, expect, test } from "bun:test";
+
+import { handleMemoryV3GateStats } from "../memory-v3-routes.js";
+
+// ---------------------------------------------------------------------------
+// Minimal fake SQLite DB that returns pre-set rows
+// ---------------------------------------------------------------------------
+
+type FakeRow = { payload: string };
+
+function makePayload(detail: Record<string, unknown>): string {
+  return JSON.stringify({
+    type: "watchdog",
+    daemon_event_id: "test-id",
+    recorded_at: Date.now(),
+    check_name: "memory_v3_injection_gate",
+    value: 1,
+    detail,
+  });
+}
+
+function fakeDb(rows: FakeRow[]): Database {
+  return {
+    query: (_sql: string) => ({
+      all: (..._args: unknown[]) => rows,
+    }),
+  } as unknown as Database;
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("handleMemoryV3GateStats", () => {
+  test("returns zero counts with all buckets when DB is null", () => {
+    const result = handleMemoryV3GateStats(30, null);
+    expect(result.totalRuns).toBe(0);
+    expect(result.buckets).toHaveLength(4);
+    for (const bucket of result.buckets) {
+      expect(bucket.total).toBe(0);
+      expect(bucket.scored).toBe(0);
+      expect(bucket.passed).toBe(0);
+      expect(bucket.scoredPassRate).toBeNull();
+      expect(bucket.reasons).toEqual({});
+    }
+    expect(result.unknownPageCount).toEqual({
+      total: 0,
+      passed: 0,
+      reasons: {},
+    });
+  });
+
+  test("returns zero counts with all buckets when DB returns no rows", () => {
+    const result = handleMemoryV3GateStats(30, fakeDb([]));
+    expect(result.totalRuns).toBe(0);
+    expect(result.buckets[0]!.pageCountRange).toBe("0–9");
+    expect(result.buckets[3]!.pageCountRange).toBe("200+");
+    for (const b of result.buckets) {
+      expect(b.scoredPassRate).toBeNull();
+    }
+  });
+
+  test("buckets a dense_pass row into 10–49 bucket", () => {
+    const rows = [
+      {
+        payload: makePayload({
+          pass: true,
+          reason: "dense_pass",
+          scored: true,
+          real_concept_page_count: 25,
+        }),
+      },
+    ];
+    const result = handleMemoryV3GateStats(30, fakeDb(rows));
+    expect(result.totalRuns).toBe(1);
+    const bucket = result.buckets.find((b) => b.pageCountRange === "10–49")!;
+    expect(bucket.total).toBe(1);
+    expect(bucket.scored).toBe(1);
+    expect(bucket.passed).toBe(1);
+    expect(bucket.scoredPassRate).toBe(1);
+    expect(bucket.reasons).toEqual({ dense_pass: 1 });
+  });
+
+  test("buckets a fail row into 200+ bucket", () => {
+    const rows = [
+      {
+        payload: makePayload({
+          pass: false,
+          reason: "fail_no_signal",
+          scored: true,
+          real_concept_page_count: 350,
+        }),
+      },
+    ];
+    const result = handleMemoryV3GateStats(30, fakeDb(rows));
+    const bucket = result.buckets.find((b) => b.pageCountRange === "200+")!;
+    expect(bucket.total).toBe(1);
+    expect(bucket.passed).toBe(0);
+    expect(bucket.scoredPassRate).toBe(0);
+    expect(bucket.reasons).toEqual({ fail_no_signal: 1 });
+  });
+
+  test("unscored pass-open rows count in total but not scored", () => {
+    const rows = [
+      {
+        payload: makePayload({
+          pass: true,
+          reason: "dense_unavailable",
+          scored: false,
+          real_concept_page_count: 5,
+        }),
+      },
+    ];
+    const result = handleMemoryV3GateStats(30, fakeDb(rows));
+    const bucket = result.buckets.find((b) => b.pageCountRange === "0–9")!;
+    expect(bucket.total).toBe(1);
+    expect(bucket.scored).toBe(0);
+    expect(bucket.passed).toBe(1);
+    // No scored runs → scoredPassRate is null (unscored pass does not count)
+    expect(bucket.scoredPassRate).toBeNull();
+  });
+
+  test("unscored pass mixed with scored fail does not inflate scoredPassRate", () => {
+    // Regression: before fix, pass-open + scored-fail in same bucket gave 100%
+    const rows = [
+      {
+        payload: makePayload({
+          pass: true,
+          reason: "dense_unavailable",
+          scored: false,
+          real_concept_page_count: 30,
+        }),
+      },
+      {
+        payload: makePayload({
+          pass: false,
+          reason: "fail_no_signal",
+          scored: true,
+          real_concept_page_count: 40,
+        }),
+      },
+    ];
+    const result = handleMemoryV3GateStats(30, fakeDb(rows));
+    const bucket = result.buckets.find((b) => b.pageCountRange === "10–49")!;
+    expect(bucket.total).toBe(2);
+    expect(bucket.scored).toBe(1);
+    expect(bucket.passed).toBe(1); // total passes (scored + unscored)
+    // Only the scored fail counts toward the rate — no scored passes
+    expect(bucket.scoredPassRate).toBe(0);
+  });
+
+  test("rows with no real_concept_page_count go to unknownPageCount", () => {
+    const rows = [
+      {
+        payload: makePayload({
+          pass: false,
+          reason: "fail_no_signal",
+          scored: true,
+        }),
+      },
+    ];
+    const result = handleMemoryV3GateStats(30, fakeDb(rows));
+    expect(result.unknownPageCount.total).toBe(1);
+    expect(result.unknownPageCount.passed).toBe(0);
+    expect(result.unknownPageCount.reasons).toEqual({ fail_no_signal: 1 });
+    for (const b of result.buckets) {
+      expect(b.total).toBe(0);
+    }
+  });
+
+  test("mixed rows across multiple buckets are aggregated correctly", () => {
+    const rows: FakeRow[] = [
+      // 0–9 bucket: 1 unscored pass
+      {
+        payload: makePayload({
+          pass: true,
+          reason: "dense_disabled",
+          scored: false,
+          real_concept_page_count: 3,
+        }),
+      },
+      // 10–49 bucket: 1 pass, 1 fail (both scored)
+      {
+        payload: makePayload({
+          pass: true,
+          reason: "dense_pass",
+          scored: true,
+          real_concept_page_count: 20,
+        }),
+      },
+      {
+        payload: makePayload({
+          pass: false,
+          reason: "fail_no_signal",
+          scored: true,
+          real_concept_page_count: 45,
+        }),
+      },
+      // 50–199 bucket: 2 passes
+      {
+        payload: makePayload({
+          pass: true,
+          reason: "sparse_only_strong",
+          scored: true,
+          real_concept_page_count: 100,
+        }),
+      },
+      {
+        payload: makePayload({
+          pass: true,
+          reason: "dense_pass",
+          scored: true,
+          real_concept_page_count: 150,
+        }),
+      },
+    ];
+
+    const result = handleMemoryV3GateStats(30, fakeDb(rows));
+    expect(result.totalRuns).toBe(5);
+
+    const b09 = result.buckets.find((b) => b.pageCountRange === "0–9")!;
+    expect(b09.total).toBe(1);
+    expect(b09.scored).toBe(0);
+    expect(b09.scoredPassRate).toBeNull();
+
+    const b1049 = result.buckets.find((b) => b.pageCountRange === "10–49")!;
+    expect(b1049.total).toBe(2);
+    expect(b1049.scored).toBe(2);
+    expect(b1049.passed).toBe(1);
+    expect(b1049.scoredPassRate).toBe(0.5);
+    expect(b1049.reasons).toEqual({ dense_pass: 1, fail_no_signal: 1 });
+
+    const b50199 = result.buckets.find((b) => b.pageCountRange === "50–199")!;
+    expect(b50199.scored).toBe(2);
+    expect(b50199.passed).toBe(2);
+    expect(b50199.scoredPassRate).toBe(1);
+  });
+
+  test("lookbackDays is clamped to 1–90", () => {
+    const r1 = handleMemoryV3GateStats(0, fakeDb([]));
+    expect(r1.lookbackDays).toBe(1);
+    const r2 = handleMemoryV3GateStats(999, fakeDb([]));
+    expect(r2.lookbackDays).toBe(90);
+    const r3 = handleMemoryV3GateStats(30, fakeDb([]));
+    expect(r3.lookbackDays).toBe(30);
+  });
+
+  test("malformed JSON payload rows are skipped", () => {
+    const rows: FakeRow[] = [
+      { payload: "not-valid-json" },
+      {
+        payload: makePayload({
+          pass: true,
+          reason: "dense_pass",
+          scored: true,
+          real_concept_page_count: 25,
+        }),
+      },
+    ];
+    const result = handleMemoryV3GateStats(30, fakeDb(rows));
+    // Only the valid row is counted
+    expect(result.totalRuns).toBe(2); // totalRuns counts raw rows before parse
+    const bucket = result.buckets.find((b) => b.pageCountRange === "10–49")!;
+    expect(bucket.total).toBe(1);
+  });
+});

package/src/plugins/defaults/memory/src/memory-v2-routes.ts

@@ -22,12 +22,9 @@ import type { RouteDefinition } from "../../../../runtime/routes/types.js";
 import type { RouteHandlerArgs } from "../../../../runtime/routes/types.js";
 import { getLogger } from "../logging.js";
 import { getWorkspaceDir } from "../paths.js";
-import {
-  getEdgeIndex,
-  totalEdgeCount,
-  validateEdgeTargets,
-} from "../substrate/edge-index.js";
+import { getEdgeIndex, totalEdgeCount } from "../substrate/edge-index.js";
 import { getPageIndex } from "../substrate/page-index.js";
+import type { DanglingLink } from "../substrate/page-links.js";
 import {
   getConceptsDir,
   listPages,
@@ -111,14 +108,17 @@ async function handleBackfill({
 
 const MemoryV2ValidateParams = z.object({}).strict();
 
-type MissingEdgeEndpoint = { from: string; to: string };
 type OversizedPage = { slug: string; chars: number };
 type ParseFailure = { slug: string; error: string };
 
 export type MemoryV2ValidateResult = {
   pageCount: number;
   edgeCount: number;
-  missingEdgeEndpoints: MissingEdgeEndpoint[];
+  /**
+   * `links:`, `[[wikilink]]`, and `edges:` references whose target has no
+   * page (`PageIndex.danglingLinks`).
+   */
+  danglingLinks: DanglingLink[];
   oversizedPages: OversizedPage[];
   parseFailures: ParseFailure[];
 };
@@ -162,13 +162,15 @@ async function handleValidate({
     }
   }
 
-  const edgeIndex = await getEdgeIndex(workspaceDir);
-  const { missing } = validateEdgeTargets(edgeIndex, knownSlugs);
+  const [edgeIndex, pageIndex] = await Promise.all([
+    getEdgeIndex(workspaceDir),
+    getPageIndex(workspaceDir),
+  ]);
 
   return {
     pageCount: knownSlugs.size,
     edgeCount: totalEdgeCount(edgeIndex),
-    missingEdgeEndpoints: missing,
+    danglingLinks: pageIndex.danglingLinks,
     oversizedPages,
     parseFailures,
   };