@vellumai/assistant 0.8.9-staging.2 → 0.8.9-staging.3

package/docs/activation-funnel-telemetry.md

@@ -0,0 +1,310 @@
+# Activation Funnel Telemetry — Runbook + Analytics Handoff
+
+> **Linear:** JARVIS-1102 (event emission) — gates JARVIS-1092 (10% rollout) and JARVIS-1093 (dashboard).
+> **Funnel version:** `activation_v1_2026_06`
+> **LD flag:** `experiment-activation-flow-2026-06-03`
+> **Cohort arm tag:** `ab_variant = "variant-a"` (treatment); `control` = the no-rail arm.
+
+This doc is the single executable reference for the activation-rail funnel
+telemetry: what it measures, how it flows, the event vocabulary, the dedup
+contract, the local smoke-test runbook, the BigQuery verification query, and the
+canonical handoff blurb for the dashboard work. It is meant to be runnable by
+another engineer without reading the implementation.
+
+---
+
+## 1. Overview — what it measures and how it flows
+
+The activation funnel measures how far a new user gets on their first run of the
+**activation rail** (the onboarding experience driven by
+`BOOTSTRAP-ACTIVATION-RAIL.md`). It emits five milestone ("funnel") events per
+session into the **existing onboarding telemetry substrate** — no new event type,
+no new ingest endpoint, no new BigQuery table.
+
+The north-star "≥5 user messages within 24h per LD variant" metric is **not** a
+materialized activation event. It is computed downstream by joining the existing
+per-user-message `turn` telemetry events (`turn_index` per conversation) to the
+platform's `flag_assignment_raw` cohort-assignment table — so a dedicated event
+and a custom daemon turn-counting hook are not needed.
+
+Flow, end to end:
+
+1. **Daemon records** an activation event into the SQLite `onboarding_events`
+   table via `recordActivationEvent()` in
+   `assistant/src/memory/onboarding-events-store.ts`:
+   - emission is **deterministic, tied to a `ui_show` surface** — there is no
+     model-facing tool. The model tags the surface it is already rendering for a
+     rail move with an optional `activation_moment` parameter on `ui_show`; the
+     daemon captures that tag on the surface's server-side state and records the
+     milestone (gated on `isActivationSession`). **Timing is per-moment**
+     (`ACTIVATION_MOMENT_EMIT_AT` in `activation-funnel.ts`): most moments record
+     when the user **commits** the surface (clicks an action / submits /
+     selects) via `handleSurfaceAction()`; the one exception is
+     `first_wow_executed`, which records at **render time** in
+     `surfaceProxyResolver` — the Run result/`work_result` surface is often
+     display-only and may never be committed, so a commit-time emit would never
+     fire (and deferring it to a later click would conflate "executed" with
+     "interacted"). Show-timing tags are recorded immediately and not stored, so
+     the commit path never double-emits. The token→step-name map and the
+     show/commit timing both live in
+     `assistant/src/telemetry/activation-funnel.ts`;
+   - emission is best-effort (wrapped in try/catch) and never blocks or alters
+     the surface-action flow;
+   - `recordActivationEvent` respects the `getConfig().collectUsageData` opt-out
+     gate (returns `null` / no row when disabled).
+2. **Reporter flushes** every ~5 min: `usage-telemetry-reporter.ts`
+   (`REPORT_INTERVAL_MS = 5 * 60 * 1000`, with a one-time
+   `INITIAL_FLUSH_DELAY_MS = 30_000` after startup) POSTs unreported onboarding
+   rows to `/v1/telemetry/ingest/` as `type: "onboarding"` events, mapping the
+   funnel columns onto the wire shape.
+3. **Platform ingests** the onboarding events and writes GCS NDJSON.
+4. **BigQuery** exposes them via the external table
+   `vellum-ai-prod.telemetry.onboarding_raw`, which already carries the
+   `session_id / step_name / step_index / completed_at / funnel_version /
+ab_variant` columns. The existing dbt model dedups on `daemon_event_id`
+   (earliest-wins).
+
+No platform / dbt / terraform change is required — the activation events ride the
+`type: "onboarding"` substrate that already exists end to end.
+
+---
+
+## 2. Event vocabulary
+
+The single source of truth is `assistant/src/telemetry/activation-funnel.ts`
+(`ACTIVATION_STEPS`). `funnel_version = "activation_v1_2026_06"`
+(`ACTIVATION_FUNNEL_VERSION`). `ab_variant = "variant-a"`
+(`ACTIVATION_AB_VARIANT`, the treatment arm).
+
+Each step is recorded when the user commits the `ui_show` surface the model
+tagged with the corresponding `activation_moment` token (see §1). The
+token→step-name map is the `moment_*` column below.
+
+| step_index | step_name                         | `activation_moment` token | Recorded on commit of                                           |
+| ---------- | --------------------------------- | ------------------------- | --------------------------------------------------------------- |
+| 1          | `activation_moment_1_complete`    | `moment_1`                | Port-summary card OR no-port intake `choice` surface            |
+| 2          | `activation_moment_2_complete`    | `moment_2`                | Propose offer surface (the `ui_show` offer card/choice)         |
+| 3          | `activation_moment_3_complete`    | `moment_3`                | task-selection surface                                          |
+| 4          | `activation_first_wow_executed`   | `first_wow_executed`      | Run result surface (e.g. `work_result`)                         |
+| 5          | `activation_first_wow_interacted` | `first_wow_interacted`    | user clicks an action on the tagged result surface (see §1, §4) |
+
+All five steps are recorded deterministically on surface commit. The north star
+(≥5 user messages) is derived downstream from the existing `turn` telemetry, not
+a materialized activation event (see §1).
+
+On the wire, each onboarding event also sets `screen = step_name` (to satisfy the
+SQLite `screen TEXT NOT NULL` column and the platform's legacy-path validation),
+and `completed_at` is the ISO-8601 record time.
+
+---
+
+## 3. Dedup contract
+
+Activation rows carry a **deterministic** `daemon_event_id`:
+
+```
+daemon_event_id = `${funnel_version}:${session_id}:${step_name}`
+```
+
+Built by `buildActivationDaemonEventId()` in
+`assistant/src/telemetry/activation-funnel.ts`. The reporter overrides
+`daemon_event_id` for activation rows (where `session_id && step_name &&
+funnel_version` are all present) and keys it on the **row's stored
+`funnel_version`**, NOT the running binary's current constant. This keeps the id
+stable across a version bump so rows queued offline / flushed after an upgrade
+still collapse with already-ingested rows from the same session.
+
+A moment that fires more than once (e.g. a model double-emit) therefore lands
+with the same `daemon_event_id` and is collapsed downstream by the existing dbt
+earliest-wins dedup on `daemon_event_id`. For boolean "moment complete"
+semantics, earliest-wins is correct.
+
+**Checkpoint safety:** the SQLite watermark cursor in the reporter advances on the
+row `id` / `createdAt`, NOT on `daemon_event_id`. The deterministic id is a
+wire-only override, so overriding it never affects flush checkpointing.
+
+---
+
+## 4. Cohort scoping
+
+Events only emit for conversations explicitly marked as **activation-rail
+sessions**:
+
+- The marker lives in the `activation_sessions` table
+  (`assistant/src/memory/activation-session-store.ts`,
+  `markActivationSession` / `isActivationSession`).
+- It is set in `assistant/src/prompts/system-prompt.ts` **only when the
+  `BOOTSTRAP-ACTIVATION-RAIL.md` template is actually active** (i.e.
+  `bootstrapTemplate === ACTIVATION_RAIL_BOOTSTRAP_TEMPLATE`), which the web
+  prechat context selects only for users whose
+  `experiment-activation-flow-2026-06-03` cohort puts them on the rail. (It is
+  also set up-front in `conversation.ts` `setOnboardingContext` so the marker is
+  available before the first turn's tool resolution.)
+- Emission is **gated on the marker at commit time**: when a tagged `ui_show`
+  surface is committed, the daemon records the milestone only if
+  `isActivationSession(conversationId)` is true. A stray `activation_moment` tag
+  in a normal chat is therefore ignored and never pollutes the funnel. (`moment_4`
+  / `first_wow_interacted` deduping is handled the same way as any other moment —
+  see the dedup contract in §3 — and the per-surface tag is cleared after the
+  first record so a single surface commit emits at most once.)
+
+The activation **session id is the daemon `conversation_id`** of the rail
+conversation. That same value is `session_id` on every activation event.
+
+---
+
+## 5. Local smoke-test runbook
+
+Goal: drive one dev session through the activation cohort, emit all five events,
+flush, and verify rows in BigQuery. Executable by another engineer.
+
+### 5.1 Put the dev user in the activation cohort
+
+The rail is gated by the LD flag `experiment-activation-flow-2026-06-03`. Two
+options:
+
+- **In-app feature-flags override (fastest):** open the in-app Feature Flags
+  panel and force `experiment-activation-flow-2026-06-03` ON for the dev user.
+- **LaunchDarkly targeting:** target the dev user for the flag in the LD
+  dashboard (per the LD-flag-two-repo convention, targeting is dashboard-side).
+
+When the flag is ON, the web prechat context selects
+`BOOTSTRAP-ACTIVATION-RAIL.md` as the bootstrap template, and the daemon marks
+the conversation in `activation_sessions` on first build of the system prompt.
+
+### 5.2 Confirm usage-data collection is enabled — and do NOT run in dev mode
+
+Two gates must both be satisfied or no rows reach BigQuery:
+
+1. `recordActivationEvent` no-ops when `config.collectUsageData` is false, so the
+   dev build/config must have usage-data collection enabled, or no rows are
+   written to SQLite.
+2. **Dev mode disables the flush entirely.** `assistant/src/daemon/lifecycle.ts`
+   computes the effective setting as `collectUsageData = !isDevMode &&
+config.collectUsageData`, so when the daemon runs in dev mode (`VELLUM_DEV=1`)
+   the `UsageTelemetryReporter` is never started — rows accumulate in SQLite but
+   are never POSTed, and the "wait/restart for flush" steps below will never
+   reach BigQuery. For an end-to-end smoke test, run the daemon **outside dev
+   mode** (so the reporter starts), or explicitly invoke the reporter's
+   `flush()` via a dev hook. The SQLite rows can still be inspected directly in
+   dev mode, but the BigQuery verification (§6) requires a real flush.
+
+### 5.3 Start a fresh activation conversation and capture its id
+
+Start a brand-new conversation with the flag ON (it must be the rail bootstrap, so
+start it from the web prechat flow / activation entry, not an existing chat).
+Note its `conversation_id` — this is the `session_id` you will query on.
+
+If multiple daemons are running, kill ALL `daemon/main.ts` processes, remove
+`~/.vellum/vellum.sock`, and start fresh, so the conversation is served by a known
+daemon writing to a known SQLite DB.
+
+### 5.4 Drive the rail through all five model moments
+
+Work the conversation through the rail moves so the model tags each surface and
+you commit it (the model tags surfaces via `ui_show`'s `activation_moment`
+parameter; which surface maps to which moment is documented in
+`BOOTSTRAP-ACTIVATION-RAIL.md` and §2). Each milestone records when YOU commit
+the tagged surface:
+
+1. Commit the Port-summary card / no-port intake `choice` → `activation_moment_1_complete`.
+2. Commit the Propose offer surface → `activation_moment_2_complete`.
+3. Commit the task-selection surface → `activation_moment_3_complete`.
+4. Commit the Run result surface → `activation_first_wow_executed`.
+5. Click an action on the result surface → `activation_first_wow_interacted`.
+
+### 5.5 Force / await a telemetry flush
+
+The reporter flushes on a schedule: a first flush ~30s after startup
+(`INITIAL_FLUSH_DELAY_MS`) and then every ~5 min (`REPORT_INTERVAL_MS`). Either:
+
+- **Wait** for the next scheduled flush (≤ ~5 min), or
+- **Restart the daemon** to take the ~30s post-startup flush, or
+- call the reporter's `flush()` path directly if you have a dev hook into the
+  running daemon.
+
+A successful flush POSTs the unreported onboarding rows to
+`/v1/telemetry/ingest/` as `type: "onboarding"` events.
+
+### 5.6 Verify in BigQuery
+
+Wait for the platform → GCS NDJSON → BigQuery pipeline to land the events, then
+run the query in §6.
+
+---
+
+## 6. BigQuery verification query
+
+```sql
+SELECT step_name, step_index, COUNT(*) AS n
+FROM `vellum-ai-prod.telemetry.onboarding_raw`
+WHERE funnel_version = 'activation_v1_2026_06'
+  AND session_id = '<dev conversation id>'
+GROUP BY 1, 2
+ORDER BY 2;
+```
+
+**Expected:** all five `activation_*` step names for the one session, step_index
+1–5, each with `n = 1`:
+
+```
+activation_moment_1_complete      | 1 | 1
+activation_moment_2_complete      | 2 | 1
+activation_moment_3_complete      | 3 | 1
+activation_first_wow_executed     | 4 | 1
+activation_first_wow_interacted   | 5 | 1
+```
+
+A missing row means that milestone never emitted (re-drive that rail move); an
+`n > 1` before dbt dedup is harmless (the deterministic `daemon_event_id`
+collapses it earliest-wins).
+
+---
+
+## 7. Canonical handoff blurb (paste-ready for the final JARVIS-1102 PR description)
+
+> **Activation funnel telemetry — handoff for JARVIS-1093.** The activation rail
+> now emits five milestone funnel events into the existing onboarding telemetry
+> substrate (`type: "onboarding"` → `/v1/telemetry/ingest/` → GCS NDJSON →
+> `vellum-ai-prod.telemetry.onboarding_raw`), so no new event type, ingest
+> endpoint, or BigQuery table is involved. Cohort is gated by the LaunchDarkly
+> flag **`experiment-activation-flow-2026-06-03`**; only rail (treatment)
+> conversations emit. Every event carries **`funnel_version =
+"activation_v1_2026_06"`** and **`ab_variant`** tagged `"variant-a"` for
+> treatment (`"control"` is the no-rail arm, measured for now from generic `turn`
+> events). The event vocabulary (step_name → step_index): `activation_moment_1_complete`
+> (1), `activation_moment_2_complete` (2), `activation_moment_3_complete` (3),
+> `activation_first_wow_executed` (4), `activation_first_wow_interacted` (5).
+> `session_id` is the rail conversation's id. The **north star** ("≥5 user
+> messages within 24h per LD variant") is **not** a funnel event — compute it
+> downstream by joining the existing `turn` telemetry events (`turn_index` per
+> conversation) to the platform's `flag_assignment_raw` cohort-assignment table.
+> **Dedup contract:** each row's `daemon_event_id` is deterministic,
+> `${funnel_version}:${session_id}:${step_name}`, keyed on the row's stored
+> `funnel_version`; the existing dbt earliest-wins dedup on `daemon_event_id`
+> collapses any repeated moment to a single row. Build funnel conversion as a
+> step_index 1→5 progression per `session_id`, filtered to `funnel_version =
+'activation_v1_2026_06'`.
+
+---
+
+## 8. Notes
+
+- **JARVIS-1102 "naming check" (resolved).** Events fire **deterministically on
+  the real user commit of a `ui_show` surface, not every text turn**. The model
+  passively tags the surface for a rail move with `activation_moment`; the daemon
+  records the milestone in `handleSurfaceAction` when the user commits that
+  surface (firing conditions / surface→moment mapping documented in
+  `BOOTSTRAP-ACTIVATION-RAIL.md`). This removes the standalone
+  `emit_activation_event` tool (and its cohort preactivation) entirely and ties
+  emission to a genuine user action, resolving the naming-check open
+  interpretation.
+- **Stream B (multivariate cohort flag conversion) is NOT in this work.** It is
+  gated on the platform team's in-flight string-flag serving — until the platform
+  can serve string/multivariate flags, flipping
+  `experiment-activation-flow-2026-06-03` boolean→multivariate client-side would
+  silently disable the live allowlisted rail. Until then, `ab_variant` is the
+  constant `"variant-a"` (only treatment runs the rail), and the **control side**
+  of the comparison is measured from generic `turn` events split by the
+  platform's experiment exposure mapping. Once Stream B lands, the daemon tags the
+  real assigned arm and the cross-cohort comparison works with zero rework.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.8.9-staging.2",
+  "version": "0.8.9-staging.3",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/__tests__/activation-early-marking.test.ts

@@ -0,0 +1,120 @@
+/**
+ * Early activation-session marking.
+ *
+ * The agent loop resolves TOOLS before the SYSTEM PROMPT, so the activation
+ * marker must be written before tool resolution on the first activation-rail
+ * turn — otherwise the `activation` skill is not preactivated and the emit
+ * handler is gated off for the very turn that emits Moment 1. The marking lives
+ * in `applyBootstrapTemplate`, which is called from `setOnboardingContext` (the
+ * earliest point the conversation knows its bootstrap selection) as well as
+ * from `buildSystemPrompt` as an idempotent backstop.
+ *
+ * These tests exercise `applyBootstrapTemplate` directly against the real
+ * DB-backed `isActivationSession` and a real workspace BOOTSTRAP.md, proving the
+ * marker is set INDEPENDENT of any `buildSystemPrompt` render.
+ */
+
+import { copyFileSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+const TEST_DIR = process.env.VELLUM_WORKSPACE_DIR!;
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+const { applyBootstrapTemplate } = await import("../prompts/system-prompt.js");
+const { isActivationSession } =
+  await import("../memory/activation-session-store.js");
+const { ACTIVATION_RAIL_BOOTSTRAP_TEMPLATE } =
+  await import("../telemetry/activation-funnel.js");
+const { getDb } = await import("../memory/db-connection.js");
+const { initializeDb } = await import("../memory/db-init.js");
+const { activationSessions } = await import("../memory/schema.js");
+
+initializeDb();
+
+/** Seed the workspace BOOTSTRAP.md with the unmodified generic template so the
+ *  one-shot reseed inside `applyBootstrapTemplate` fires. */
+function seedGenericBootstrap(): void {
+  mkdirSync(TEST_DIR, { recursive: true });
+  const generic = readFileSync(
+    join(import.meta.dirname, "..", "prompts", "templates", "BOOTSTRAP.md"),
+    "utf-8",
+  );
+  writeFileSync(join(TEST_DIR, "BOOTSTRAP.md"), generic, "utf-8");
+}
+
+describe("applyBootstrapTemplate — early activation marking", () => {
+  beforeEach(() => {
+    getDb().delete(activationSessions).run();
+  });
+
+  test("marks the conversation when the activation-rail template is applied", () => {
+    seedGenericBootstrap();
+    expect(isActivationSession("rail-conv")).toBe(false);
+
+    // No buildSystemPrompt call — marking must happen here.
+    applyBootstrapTemplate(ACTIVATION_RAIL_BOOTSTRAP_TEMPLATE, "rail-conv");
+
+    expect(isActivationSession("rail-conv")).toBe(true);
+  });
+
+  test("does not mark when a non-rail bootstrap template is applied", () => {
+    seedGenericBootstrap();
+
+    applyBootstrapTemplate("BOOTSTRAP-REFERENCE.md", "non-rail-conv");
+
+    expect(isActivationSession("non-rail-conv")).toBe(false);
+  });
+
+  test("does not mark when no conversation id is supplied", () => {
+    seedGenericBootstrap();
+
+    applyBootstrapTemplate(ACTIVATION_RAIL_BOOTSTRAP_TEMPLATE);
+
+    // Nothing to assert by id; confirm the table stayed empty.
+    const rows = getDb().select().from(activationSessions).all();
+    expect(rows.length).toBe(0);
+  });
+
+  test("does not mark when BOOTSTRAP.md is customized to something else", () => {
+    mkdirSync(TEST_DIR, { recursive: true });
+    // A user-customized BOOTSTRAP.md is neither the generic template (so the
+    // reseed no-ops) nor the activation-rail template — the rail is not active.
+    writeFileSync(
+      join(TEST_DIR, "BOOTSTRAP.md"),
+      "# My custom bootstrap\n",
+      "utf-8",
+    );
+
+    applyBootstrapTemplate(ACTIVATION_RAIL_BOOTSTRAP_TEMPLATE, "custom-conv");
+
+    expect(isActivationSession("custom-conv")).toBe(false);
+  });
+
+  test("marks idempotently when BOOTSTRAP.md already holds the rail template", () => {
+    mkdirSync(TEST_DIR, { recursive: true });
+    // Simulate a later turn: BOOTSTRAP.md already holds the rail template, so
+    // the reseed no-ops but marking still fires (idempotent backstop).
+    copyFileSync(
+      join(
+        import.meta.dirname,
+        "..",
+        "prompts",
+        "templates",
+        ACTIVATION_RAIL_BOOTSTRAP_TEMPLATE,
+      ),
+      join(TEST_DIR, "BOOTSTRAP.md"),
+    );
+
+    applyBootstrapTemplate(ACTIVATION_RAIL_BOOTSTRAP_TEMPLATE, "rail-conv-2");
+    applyBootstrapTemplate(ACTIVATION_RAIL_BOOTSTRAP_TEMPLATE, "rail-conv-2");
+
+    expect(isActivationSession("rail-conv-2")).toBe(true);
+  });
+});

package/src/__tests__/agent-loop-output-hooks.test.ts

@@ -1,5 +1,5 @@
 /**
- * Tests for the `pre-model-call` and `assistant-message` plugin hooks: a plugin
+ * Tests for the `pre-model-call` and `post-model-call` plugin hooks: a plugin
  * can edit the outbound request, transform the finalized assistant message
  * (persisted + streamed), and defer the live stream so the transformed text is
  * emitted once. All hooks here use neutral transforms (redaction / uppercasing).
@@ -9,7 +9,7 @@ import { beforeEach, describe, expect, test } from "bun:test";
 import type { AgentEvent } from "../agent/loop.js";
 import { AgentLoop } from "../agent/loop.js";
 import type {
-  AssistantMessageContext,
+  PostModelCallContext,
   PreModelCallContext,
 } from "../plugin-api/types.js";
 import { resetPluginRegistryAndRegisterDefaults } from "../plugins/defaults/index.js";
@@ -55,7 +55,7 @@ function lastAssistant(history: Message[]): Message {
 
 function registerOutputHookPlugin(hooks: {
   preModelCall?: (ctx: PreModelCallContext) => void;
-  assistantMessage?: (ctx: AssistantMessageContext) => void;
+  postModelCall?: (ctx: PostModelCallContext) => void;
 }): void {
   registerPlugin({
     manifest: { name: "test-output-hooks", version: "0.0.0" },
@@ -67,10 +67,10 @@ function registerOutputHookPlugin(hooks: {
             },
           }
         : {}),
-      ...(hooks.assistantMessage
+      ...(hooks.postModelCall
         ? {
-            "assistant-message": async (ctx: AssistantMessageContext) => {
-              hooks.assistantMessage!(ctx);
+            "post-model-call": async (ctx: PostModelCallContext) => {
+              hooks.postModelCall!(ctx);
             },
           }
         : {}),
@@ -83,9 +83,9 @@ describe("agent loop output hooks", () => {
     resetPluginRegistryAndRegisterDefaults();
   });
 
-  test("assistant-message transforms the persisted message content", async () => {
+  test("post-model-call transforms the persisted message content", async () => {
     registerOutputHookPlugin({
-      assistantMessage: (ctx) => {
+      postModelCall: (ctx) => {
         ctx.content = ctx.content.map((b) =>
           b.type === "text"
             ? { type: "text", text: b.text.replace("secret", "[redacted]") }
@@ -113,7 +113,7 @@ describe("agent loop output hooks", () => {
       preModelCall: (ctx) => {
         ctx.deferAssistantOutput = true;
       },
-      assistantMessage: (ctx) => {
+      postModelCall: (ctx) => {
         ctx.content = [{ type: "text", text: "[filtered]" }];
       },
     });
@@ -137,7 +137,7 @@ describe("agent loop output hooks", () => {
 
   test("without defer, the real text still streams while storage is transformed", async () => {
     registerOutputHookPlugin({
-      assistantMessage: (ctx) => {
+      postModelCall: (ctx) => {
         ctx.content = [{ type: "text", text: "[stored]" }];
       },
     });
@@ -168,7 +168,7 @@ describe("agent loop output hooks", () => {
       stopReason: "tool_use",
     };
     registerOutputHookPlugin({
-      assistantMessage: (ctx) => {
+      postModelCall: (ctx) => {
         ctx.content = ctx.content.map((b) =>
           b.type === "text" ? { type: "text", text: b.text.toUpperCase() } : b,
         );
@@ -237,7 +237,7 @@ describe("agent loop output hooks", () => {
     // mid-mutation would survive into history. The loop must clone the
     // content before invoking the hook.
     registerOutputHookPlugin({
-      assistantMessage: (ctx) => {
+      postModelCall: (ctx) => {
         ctx.content.push({ type: "text", text: "[INJECTED]" });
         throw new Error("boom");
       },
@@ -269,7 +269,7 @@ describe("agent loop output hooks", () => {
       preModelCall: (ctx) => {
         ctx.deferAssistantOutput = true;
       },
-      assistantMessage: (ctx) => {
+      postModelCall: (ctx) => {
         seen.calls += 1;
         ctx.content = ctx.content.map((b) =>
           b.type === "text" ? { type: "text", text: b.text.toUpperCase() } : b,

package/src/__tests__/approval-cascade.test.ts

@@ -173,7 +173,7 @@ mock.module("../memory/retriever.js", () => ({
   injectMemoryRecallAsUserBlock: (msgs: Message[]) => msgs,
 }));
 
-mock.module("../context/window-manager.js", () => ({
+mock.module("../plugins/defaults/compaction/window-manager.js", () => ({
   ContextWindowManager: class {
     constructor() {}
     updateConfig() {}

package/src/__tests__/compaction-direct.test.ts

@@ -3,19 +3,29 @@
  *
  * The agent loop calls {@link defaultCompact} directly with a
  * {@link CompactionContext} rather than routing through a middleware pipeline.
- * These tests assert that the default implementation delegates to the
- * supplied {@link ContextWindowManager} and forwards the request's
- * conversational options verbatim. The orchestrator integration path
- * (conversation-agent-loop) is exercised by
+ * These tests assert that the default implementation resolves the
+ * conversation's {@link ContextWindowManager} from the compaction store and
+ * forwards the request's conversational options verbatim. The orchestrator
+ * integration path (conversation-agent-loop) is exercised by
  * `conversation-agent-loop-overflow.test.ts`.
  */
 
-import { describe, expect, test } from "bun:test";
+import { describe, expect, mock, test } from "bun:test";
 
-import {
-  type CompactionContext,
-  defaultCompact,
-} from "../plugins/defaults/compaction/compact.js";
+// `defaultCompact` resolves the manager from the per-conversation compaction
+// store. Register each test's canned stub in a map the mocked store reads from.
+const fakeContextWindowManagers = new Map<string, unknown>();
+mock.module("../plugins/defaults/compaction/manager-store.js", () => ({
+  createContextWindowManager: () => undefined,
+  getContextWindowManager: (conversationId: string) =>
+    fakeContextWindowManagers.get(conversationId),
+  disposeContextWindowManager: (conversationId: string) => {
+    fakeContextWindowManagers.delete(conversationId);
+  },
+}));
+
+const { defaultCompact } =
+  await import("../plugins/defaults/compaction/compact.js");
 
 type ContextWindowResultShape = {
   compacted: boolean;
@@ -55,13 +65,17 @@ function makeResult(
   };
 }
 
-function makeManager(result: ContextWindowResultShape) {
+/** Register a stub manager under a conversation id and return its recorder. */
+function registerManager(
+  conversationId: string,
+  result: ContextWindowResultShape,
+) {
   const observed: {
     messages: unknown;
     signal: unknown;
     options: unknown;
   }[] = [];
-  const manager = {
+  fakeContextWindowManagers.set(conversationId, {
     maybeCompact: async (
       messages: unknown,
       signal: unknown,
@@ -70,8 +84,8 @@ function makeManager(result: ContextWindowResultShape) {
       observed.push({ messages, signal, options });
       return result;
     },
-  } as unknown as CompactionContext["manager"];
-  return { manager, observed };
+  });
+  return observed;
 }
 
 describe("defaultCompact", () => {
@@ -81,13 +95,13 @@ describe("defaultCompact", () => {
       summaryText: "manager-summary",
       compactedMessages: 7,
     });
-    const { manager, observed } = makeManager(expected);
+    const observed = registerManager("conv-delegates", expected);
     const messages = [{ role: "user", content: "hi" }] as never;
     const signal = new AbortController().signal;
 
     // WHEN defaultCompact runs with those messages and a signal
     const result = (await defaultCompact({
-      manager,
+      conversationId: "conv-delegates",
       messages,
       signal,
     })) as unknown as ContextWindowResultShape;
@@ -105,11 +119,11 @@ describe("defaultCompact", () => {
 
   test("forwards the request's compaction options to the manager", async () => {
     // GIVEN a manager and a fully-populated compaction context
-    const { manager, observed } = makeManager(makeResult());
+    const observed = registerManager("conv-options", makeResult());
 
     // WHEN defaultCompact runs with force/profile/trust options
     await defaultCompact({
-      manager,
+      conversationId: "conv-options",
       messages: [] as never,
       force: true,
       overrideProfile: "fast-profile",
@@ -118,7 +132,7 @@ describe("defaultCompact", () => {
       actorTrustClass: "guardian",
     });
 
-    // THEN the manager received exactly those options, and the manager,
+    // THEN the manager received exactly those options, and the conversation id,
     // messages, and signal are not leaked into the options bag
     expect(observed).toHaveLength(1);
     expect(observed[0]!.options).toEqual({

package/src/__tests__/compaction-events.test.ts

@@ -12,8 +12,8 @@ import { describe, expect, mock, test } from "bun:test";
 
 import { CompactionCircuit } from "../agent/compaction-circuit.js";
 import type { AgentEvent } from "../agent/loop.js";
-import type { ContextWindowResult } from "../context/window-manager.js";
 import type { ServerMessage } from "../daemon/message-protocol.js";
+import type { ContextWindowResult } from "../plugins/defaults/compaction/window-manager.js";
 import type { Message, ProviderResponse } from "../providers/types.js";
 
 // ---------------------------------------------------------------------------
@@ -187,7 +187,7 @@ let mockCompactResult: ContextWindowResult = {
   summaryText: "",
 };
 
-mock.module("../context/window-manager.js", () => ({
+mock.module("../plugins/defaults/compaction/window-manager.js", () => ({
   ContextWindowManager: class {
     nonPersistedPrefixCount = 0;
     constructor() {}