@vellumai/assistant 0.8.10-dev.202606110941.6cb149d → 0.8.10-dev.202606111140.6a5e88c

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.8.10-dev.202606110941.6cb149d",
+  "version": "0.8.10-dev.202606111140.6a5e88c",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/__tests__/agent-loop-exit-reason.test.ts

@@ -13,7 +13,7 @@
  * Sites not exercised here (`aborted_via_error`) require deeper provider
  * fakery and are best covered by integration tests.
  */
-import { afterEach, describe, expect, test } from "bun:test";
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
 
 import type { PostCompactContext } from "@vellumai/plugin-api";
 
@@ -31,7 +31,10 @@ import {
   disposeContextWindowManager,
   getContextWindowManager,
 } from "../plugins/defaults/compaction/manager-store.js";
-import { registerPlugin } from "../plugins/registry.js";
+import {
+  registerPlugin,
+  resetPluginRegistryForTests,
+} from "../plugins/registry.js";
 import type {
   Message,
   Provider,
@@ -49,7 +52,7 @@ import type {
 let postCompactImpl:
   | ((input: PostCompactContext) => Promise<Message[]>)
   | null = null;
-registerPlugin({
+const testPostCompactPlugin = {
   manifest: { name: "test-post-compact", version: "0.0.0" },
   hooks: {
     [HOOKS.POST_COMPACT]: async (input: PostCompactContext): Promise<void> => {
@@ -58,7 +61,7 @@ registerPlugin({
         : input.history;
     },
   },
-});
+};
 
 // ---------------------------------------------------------------------------
 // Helpers (mirrored from agent-loop.test.ts so this file is self-contained)
@@ -169,6 +172,14 @@ function countExitEvents(events: AgentEvent[]): number {
 // ---------------------------------------------------------------------------
 
 describe("AgentLoop exit-reason instrumentation", () => {
+  // Reset the plugin registry to a known state so ambient registrations from
+  // other test files (e.g. the default plugins) cannot leak into the
+  // post-compact hook chain these tests drive.
+  beforeEach(() => {
+    resetPluginRegistryForTests();
+    registerPlugin(testPostCompactPlugin);
+  });
+
   afterEach(() => {
     disposeContextWindowManager("test-conversation");
   });
@@ -227,6 +238,41 @@ describe("AgentLoop exit-reason instrumentation", () => {
     expect(events[events.length - 1].type).toBe("agent_loop_exit");
   });
 
+  test("emits agent_loop_exit even when a stop hook throws", async () => {
+    // A third-party teardown hook that rejects must not suppress the terminal
+    // exit: the loop isolates the stop chain, logs the failure, and still emits
+    // `agent_loop_exit` exactly once with the real reason.
+    registerPlugin({
+      manifest: { name: "throwing-stop", version: "0.0.0" },
+      hooks: {
+        [HOOKS.STOP]: async (): Promise<void> => {
+          throw new Error("teardown boom");
+        },
+      },
+    });
+
+    const { provider } = createMockProvider([textResponse("Hi there!")]);
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system prompt",
+      conversationId: "test-conversation",
+    });
+
+    const events: AgentEvent[] = [];
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: (e) => {
+        events.push(e);
+      },
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    expect(countExitEvents(events)).toBe(1);
+    expect(lastExitEvent(events)?.reason).toBe("no_tool_calls");
+    expect(events[events.length - 1].type).toBe("agent_loop_exit");
+  });
+
   test("emits continuation surface event and exits on max_tokens", async () => {
     const { provider } = createMockProvider([
       maxTokensResponse("Partial answer"),

package/src/__tests__/agent-loop.test.ts

@@ -6,6 +6,7 @@ import type {
   CheckpointInfo,
 } from "../agent/loop.js";
 import { AgentLoop } from "../agent/loop.js";
+import type { StopContext } from "../plugin-api/types.js";
 import { REFUSAL_FALLBACK_TEXT } from "../plugins/defaults/empty-response/hooks/post-model-call.js";
 import { resetPluginRegistryAndRegisterDefaults } from "../plugins/defaults/index.js";
 import { registerPlugin } from "../plugins/registry.js";
@@ -544,11 +545,12 @@ describe("AgentLoop", () => {
     });
   });
 
-  test("a stop-hook throw on a successful stop surfaces through the error path without re-entering the stop chain", async () => {
-    // The error-stop recovery is confined to genuine provider rejections. A
-    // throw from the success-path stop chain must not re-enter that chain (the
-    // same hook would throw again and escape the loop, bypassing the generic
-    // error handling) — it must fall through to the standard error path.
+  test("isolates a throwing stop hook on a successful stop and still emits the terminal exit", async () => {
+    // The `stop` chain is the loop's terminal teardown: a throwing teardown
+    // hook (e.g. a third-party plugin) is logged and contained, never escalated
+    // into a turn error. The turn's real outcome stands — a successful no-tool
+    // stop — and the terminal `agent_loop_exit` still fires so the run stays
+    // observable.
 
     // GIVEN a registered stop hook that always throws, and a provider that
     // returns a successful no-tool response (a successful stop)
@@ -581,11 +583,68 @@ describe("AgentLoop", () => {
         rejected = true;
       });
 
-    // THEN the loop handled the throw via the error path (it did not reject),
-    // surfaced exactly one error, and did not re-issue the call
+    // THEN the loop did not reject, did not re-issue the call, surfaced no
+    // error event, and still emitted the terminal exit with the real reason
     expect(rejected).toBe(false);
     expect(calls).toHaveLength(1);
-    expect(events.filter((e) => e.type === "error")).toHaveLength(1);
+    expect(events.filter((e) => e.type === "error")).toHaveLength(0);
+    const exitEvents = events.filter((e) => e.type === "agent_loop_exit");
+    expect(exitEvents).toHaveLength(1);
+    expect(exitEvents[0]).toMatchObject({ reason: "no_tool_calls" });
+  });
+
+  test("fires the stop teardown chain on a checkpoint handoff without emitting agent_loop_exit", async () => {
+    // A handoff pauses this run so the orchestrator can drain a queued message,
+    // then re-enters with a fresh run. It still ends *this* turn, so the
+    // terminal `stop` chain must fire — that is what runs per-turn teardown
+    // (e.g. clearing the recovery bounds the post-model-call hooks set) before
+    // the queued message is processed. But `agent_loop_exit` must NOT be
+    // emitted, because the handoff is a control transfer, not a terminal exit.
+
+    // GIVEN a stop hook recording every exit reason it observes, and a provider
+    // whose first reply requests a tool so the loop reaches a checkpoint
+    const stopReasons: string[] = [];
+    registerPlugin({
+      manifest: { name: "recording-stop", version: "0.0.1" },
+      hooks: {
+        stop: async (ctx: StopContext) => {
+          stopReasons.push(ctx.exitReason);
+        },
+      },
+    });
+    const { provider } = createMockProvider([
+      toolUseResponse("t1", "read_file", { path: "/a.txt" }),
+      textResponse("never reached"),
+    ]);
+    const toolExecutor = async () => ({ content: "ok", isError: false });
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+      tools: dummyTools,
+      toolExecutor,
+    });
+    const events: AgentEvent[] = [];
+
+    // AND a checkpoint callback that hands off at the first opportunity
+    const onCheckpoint = (_info: CheckpointInfo): CheckpointDecision =>
+      "handoff";
+
+    // WHEN the loop runs and yields control at the checkpoint
+    const { exitReason } = await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: collectEvents(events),
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+      onCheckpoint,
+    });
+
+    // THEN the terminal stop chain fired exactly once with the handoff reason
+    // (teardown ran), the run reported the handoff, and no agent_loop_exit was
+    // emitted
+    expect(stopReasons).toEqual(["checkpoint_handoff"]);
+    expect(exitReason).toBe("handoff");
+    expect(events.filter((e) => e.type === "agent_loop_exit")).toHaveLength(0);
   });
 
   // 6. Abort signal — verify the loop respects AbortSignal

package/src/__tests__/empty-response-hook.test.ts

@@ -1,21 +1,21 @@
 /**
- * Tests for the default `empty-response` plugin's `post-model-call` hook.
+ * Tests for the default `empty-response` plugin's hooks.
  *
  * Covers:
- * - The default hook's decision for the canonical cases: empty turn after a
- *   prior tool-use turn → continue (with the canonical nudge text); visible
- *   text → stop; prior turn already delivered visible text → stop; first model
- *   call with no prior turn → stop; provider refusal → stop with the turn
- *   rewritten to the user-facing fallback; refusal-but-recovered → stop;
- *   refusal after a visible reply this run → stop with no rewrite.
+ * - The `post-model-call` hook's decision for the canonical cases: empty turn
+ *   after a prior tool-use turn → continue (with the canonical nudge text);
+ *   visible text → stop; prior turn already delivered visible text → stop;
+ *   first model call with no prior turn → stop; provider refusal → stop with
+ *   the turn rewritten to the user-facing fallback; refusal-but-recovered →
+ *   stop; refusal after a visible reply this run → stop with no rewrite.
  * - The hook scopes its prior-turn signals to the current response cycle (the
  *   messages after the last genuine user prompt), so visible text from the
  *   inbound conversation does not suppress the nudge.
  * - When the hook continues, it appends the nudge as a `user` message to
  *   `messages`.
- * - The hook owns its retry bound: it nudges at most once per run for a
- *   conversation, and a terminal outcome clears the bound so the next run
- *   nudges afresh.
+ * - The retry bound is split across the two hooks: `post-model-call` marks the
+ *   bound (nudging at most once per run) and the `stop` hook clears it on the
+ *   definitive terminal so the next run nudges afresh.
  * - The hook ignores outcomes it does not own: a provider rejection (it carries
  *   an `error`) and a tool-bearing turn (the loop continues on its own).
  * - End-to-end through `runHook` + the registry: registering the default
@@ -34,13 +34,19 @@ import { HOOKS } from "../plugin-api/constants.js";
 import type {
   PluginLogger,
   PostModelCallContext,
+  StopContext,
 } from "../plugin-api/types.js";
 import {
   NUDGE_TEXT,
   REFUSAL_FALLBACK_TEXT,
 } from "../plugins/defaults/empty-response/hooks/post-model-call.js";
 import postModelCall from "../plugins/defaults/empty-response/hooks/post-model-call.js";
-import { resetEmptyResponseNudgeStoreForTests } from "../plugins/defaults/empty-response/nudge-state-store.js";
+import stop from "../plugins/defaults/empty-response/hooks/stop.js";
+import {
+  isEmptyResponseNudged,
+  markEmptyResponseNudged,
+  resetEmptyResponseNudgeStoreForTests,
+} from "../plugins/defaults/empty-response/nudge-state-store.js";
 import { defaultEmptyResponsePlugin } from "../plugins/defaults/index.js";
 import { runHook } from "../plugins/pipeline.js";
 import {
@@ -348,69 +354,76 @@ describe("empty-response post-model-call hook — one-shot bound", () => {
     expect(second.messages).toEqual([priorToolUseTurn]);
   });
 
-  test("a provider rejection clears the bound so the next run nudges afresh", async () => {
-    // GIVEN the hook nudged once, then the re-queried call threw a provider
-    // rejection (which the hook does not own, but which ends this turn).
+  test("the hook never clears its own bound — clearing is the stop hook's job", async () => {
+    // GIVEN the hook nudged once this run for the conversation.
     const nudged = makeCtx({
-      conversationId: "conv-err-reset",
+      conversationId: "conv-no-self-clear",
       messages: [priorToolUseTurn],
       content: [],
     });
     await postModelCall(nudged);
-    const rejected = makeCtx({
-      conversationId: "conv-err-reset",
-      messages: [priorToolUseTurn],
-      content: [],
-      error: new Error("provider rejected the request"),
-    });
-    await postModelCall(rejected);
 
-    // WHEN a fresh empty-after-tools turn arrives for the same conversation.
-    const next = makeCtx({
-      conversationId: "conv-err-reset",
-      messages: [priorToolUseTurn],
-      content: [],
-    });
-    await postModelCall(next);
+    // WHEN the re-queried call comes back as a provider rejection, then as a
+    // recovered visible-text reply — outcomes that end the turn.
+    await postModelCall(
+      makeCtx({
+        conversationId: "conv-no-self-clear",
+        messages: [priorToolUseTurn],
+        content: [],
+        error: new Error("provider rejected the request"),
+      }),
+    );
+    await postModelCall(
+      makeCtx({
+        conversationId: "conv-no-self-clear",
+        messages: [priorToolUseTurn],
+        content: [{ type: "text", text: "here is the summary" }],
+      }),
+    );
 
-    // THEN the bound was cleared on the rejection, so the hook nudges again.
-    expect(next.decision).toBe("continue");
-    expect(next.messages.at(-1)).toEqual({
-      role: "user",
-      content: [{ type: "text", text: NUDGE_TEXT }],
-    });
+    // THEN the bound stays marked — `post-model-call` only ever marks; the
+    // sibling `stop` hook is what clears it at the turn boundary.
+    expect(isEmptyResponseNudged("conv-no-self-clear")).toBe(true);
   });
+});
 
-  test("a recovered turn clears the bound so the next run nudges afresh", async () => {
-    // GIVEN the hook nudged once, then the run recovered with visible text
-    // (which clears the bound).
-    const nudged = makeCtx({
-      conversationId: "conv-reset",
-      messages: [priorToolUseTurn],
-      content: [],
-    });
-    await postModelCall(nudged);
-    const recovered = makeCtx({
-      conversationId: "conv-reset",
-      messages: [priorToolUseTurn],
-      content: [{ type: "text", text: "here is the summary" }],
-    });
-    await postModelCall(recovered);
+// ─── stop hook (terminal cleanup) ────────────────────────────────────────────
 
-    // WHEN a fresh empty-after-tools turn arrives for the same conversation.
-    const next = makeCtx({
-      conversationId: "conv-reset",
-      messages: [priorToolUseTurn],
-      content: [],
-    });
-    await postModelCall(next);
+function makeStopCtx(overrides: Partial<StopContext> = {}): StopContext {
+  return {
+    conversationId: "conv-stop",
+    messages: [],
+    exitReason: "no_tool_calls",
+    logger: noopLogger,
+    ...overrides,
+  };
+}
 
-    // THEN the bound was cleared, so the hook nudges again.
-    expect(next.decision).toBe("continue");
-    expect(next.messages.at(-1)).toEqual({
-      role: "user",
-      content: [{ type: "text", text: NUDGE_TEXT }],
-    });
+describe("empty-response stop hook — terminal cleanup", () => {
+  test("a terminal stop clears the nudge bound", async () => {
+    // GIVEN a conversation that nudged this run.
+    const conversationId = "conv-stop-clear";
+    markEmptyResponseNudged(conversationId);
+
+    // WHEN the terminal stop hook runs.
+    await stop(makeStopCtx({ conversationId }));
+
+    // THEN the bound is cleared so the next run nudges afresh.
+    expect(isEmptyResponseNudged(conversationId)).toBe(false);
+  });
+
+  test("clears the bound regardless of how the turn ended", async () => {
+    // GIVEN a conversation that nudged this run, ending on an abort rather than
+    // a finalized reply.
+    const conversationId = "conv-stop-abort";
+    markEmptyResponseNudged(conversationId);
+
+    // WHEN the terminal stop hook runs for that exit.
+    await stop(makeStopCtx({ conversationId, exitReason: "aborted_pre_call" }));
+
+    // THEN the bound is still cleared — `stop` is the definitive terminal, so
+    // the next run always nudges afresh.
+    expect(isEmptyResponseNudged(conversationId)).toBe(false);
   });
 });
 

package/src/__tests__/history-repair-hook.test.ts

@@ -14,20 +14,17 @@
  *
  * `post-model-call` (error-recovery):
  * - On a provider rejection carrying a repairable ordering error, the hook
- *   deep-repairs the messages and asks the loop to continue.
- * - Bounded to one pass per turn via the per-conversation repair state: a
- *   second consecutive ordering rejection is left to surface; the bound clears
- *   at the turn boundary.
- * - Leaves the bound intact across a mid-turn tool-bearing turn so a later
- *   ordering rejection is still the exhausted second attempt.
- * - Ignores non-ordering errors and finalized (non-error) replies.
+ *   deep-repairs the messages, marks the per-conversation bound, and asks the
+ *   loop to continue.
+ * - Bounded to one pass per turn via that bound: a second consecutive ordering
+ *   rejection is left to surface (the bound is cleared by the `stop` hook, not
+ *   here).
+ * - Ignores non-ordering errors, tool-bearing turns, and finalized (non-error)
+ *   replies — none of which it ever touches the bound for.
  *
  * `stop` (terminal cleanup):
- * - Clears the repair bound on a terminal stop, covering the retry the loop's
- *   per-run backstop overrides — the one terminal `post-model-call` cannot
- *   resolve — so the next turn repairs afresh.
- * - Leaves the bound intact on a `"continue"` stop, where a hook is re-querying
- *   the model this turn.
+ * - Clears the repair bound unconditionally on the definitive terminal stop, so
+ *   the next turn always repairs afresh no matter how the turn ended.
  */
 
 import { beforeEach, describe, expect, test } from "bun:test";
@@ -266,39 +263,12 @@ describe("history-repair post-model-call hook — direct", () => {
     expect(ctx.decision).toBe("stop");
     expect(ctx.messages).toEqual(messages);
 
-    // AND the exhausted bound is cleared so the next turn repairs afresh.
-    expect(isOrderingRepairAttempted(conversationId)).toBe(false);
-  });
-
-  test("a finalized reply clears the bound so a later turn repairs again", async () => {
-    // GIVEN a conversation that already attempted a repair this turn.
-    const conversationId = "conv-reset";
-    markOrderingRepairAttempted(conversationId);
-
-    // AND a finalized no-tool reply ends that turn, which the hook treats as
-    // terminal.
-    await postModelCall(
-      makePostModelCallCtx({
-        conversationId,
-        content: [{ type: "text", text: "done" }],
-        stopReason: "end_turn",
-      }),
-    );
-    expect(isOrderingRepairAttempted(conversationId)).toBe(false);
-
-    // WHEN a later turn hits a repairable ordering rejection.
-    const ctx = makePostModelCallCtx({
-      conversationId,
-      messages: orderingViolatingHistory(),
-      error: new Error(ORDERING_ERROR_MESSAGE),
-    });
-    await postModelCall(ctx);
-
-    // THEN the hook repairs independently of the prior turn.
-    expect(ctx.decision).toBe("continue");
+    // AND the bound stays marked — the hook never clears it; the `stop` hook
+    // clears it at the turn boundary.
+    expect(isOrderingRepairAttempted(conversationId)).toBe(true);
   });
 
-  test("a non-ordering outcome already continuing this turn keeps the bound", async () => {
+  test("a non-ordering error while continuing is a no-op", async () => {
     // GIVEN this turn already repaired an ordering rejection (bound marked),
     // then an earlier hook recovered a different error and set the decision to
     // continue.
@@ -313,16 +283,13 @@ describe("history-repair post-model-call hook — direct", () => {
     // WHEN the history-repair hook runs after that earlier hook.
     await postModelCall(ctx);
 
-    // THEN it leaves the in-flight continue alone and keeps its bound, so a
-    // later ordering rejection in the same turn is recognized as the exhausted
-    // second attempt rather than a fresh first one. The error retry shares the
-    // loop's per-run backstop, but clearing here could let alternating
-    // recoverable rejections reissue the provider call within that cap.
+    // THEN it leaves the in-flight continue and the bound alone — it only acts
+    // on a repairable ordering rejection.
     expect(ctx.decision).toBe("continue");
     expect(isOrderingRepairAttempted(conversationId)).toBe(true);
   });
 
-  test("a mid-turn tool-bearing turn keeps the bound", async () => {
+  test("a mid-turn tool-bearing turn is a no-op", async () => {
     // GIVEN this turn already repaired an ordering rejection (bound marked),
     // then the model returned a tool-bearing turn the loop continues on its
     // own.
@@ -336,9 +303,8 @@ describe("history-repair post-model-call hook — direct", () => {
     // WHEN the hook runs over the tool-bearing turn.
     await postModelCall(ctx);
 
-    // THEN it leaves the decision alone and keeps its bound, so a later
-    // ordering rejection in the same turn is still the exhausted second
-    // attempt.
+    // THEN it leaves the decision and the bound alone — there is no provider
+    // rejection to act on.
     expect(ctx.decision).toBe("stop");
     expect(isOrderingRepairAttempted(conversationId)).toBe(true);
   });
@@ -384,9 +350,7 @@ function makeStopCtx(overrides: Partial<StopContext> = {}): StopContext {
   return {
     conversationId: "conv-stop",
     messages: [],
-    responseContent: [],
-    stopReason: null,
-    decision: "stop",
+    exitReason: "no_tool_calls",
     logger: noopLogger,
     ...overrides,
   };
@@ -397,31 +361,29 @@ describe("history-repair stop hook — direct", () => {
     resetRepairStateStoreForTests();
   });
 
-  test("a terminal stop clears a bound the backstop stranded", async () => {
-    // GIVEN a turn marked a repair-retry, but the loop's per-run backstop
-    // refused the continue and surfaced the rejection through the terminal
-    // stop chain without re-running post-model-call — so the bound is stranded.
+  test("a terminal stop clears the repair bound", async () => {
+    // GIVEN a turn marked a repair-retry.
     const conversationId = "conv-backstop";
     markOrderingRepairAttempted(conversationId);
 
     // WHEN the terminal stop hook runs.
     await stop(makeStopCtx({ conversationId }));
 
-    // THEN the stranded bound is cleared so the next turn repairs afresh.
+    // THEN the bound is cleared so the next turn repairs afresh.
     expect(isOrderingRepairAttempted(conversationId)).toBe(false);
   });
 
-  test("a continuing stop keeps the bound for the in-flight retry", async () => {
-    // GIVEN a turn marked a repair-retry, and a stop hook is re-querying the
-    // model this turn (decision is continue).
-    const conversationId = "conv-continue";
+  test("clears the bound regardless of how the turn ended", async () => {
+    // GIVEN a turn marked a repair-retry that ends on an abort rather than a
+    // finalized reply.
+    const conversationId = "conv-abort";
     markOrderingRepairAttempted(conversationId);
 
-    // WHEN the stop hook runs on that continuing stop.
-    await stop(makeStopCtx({ conversationId, decision: "continue" }));
+    // WHEN the terminal stop hook runs for that exit.
+    await stop(makeStopCtx({ conversationId, exitReason: "aborted_pre_call" }));
 
-    // THEN the bound survives so a later ordering rejection this turn is still
-    // recognized as the exhausted second attempt.
-    expect(isOrderingRepairAttempted(conversationId)).toBe(true);
+    // THEN the bound is still cleared — `stop` is the definitive terminal, so
+    // the next turn always repairs afresh.
+    expect(isOrderingRepairAttempted(conversationId)).toBe(false);
   });
 });