@vellumai/assistant 0.8.10-dev.202606101436.d73da44 → 0.8.10-dev.202606101714.d4b22de

package/openapi.yaml

@@ -19406,15 +19406,7 @@ paths:
                       - type: "null"
                     description: Resolved version (installed copy first, then repo `package.json`).
                   source:
-                    oneOf:
-                      - type: object
-                        properties:
-                          kind:
-                            type: string
-                            const: first-party
-                        required:
-                          - kind
-                        additionalProperties: false
+                    anyOf:
                       - type: object
                         properties:
                           kind:
@@ -19434,7 +19426,9 @@ paths:
                           - repo
                           - ref
                         additionalProperties: false
-                    description: "Origin of the match: a first-party in-repo plugin or a whitelisted external one."
+                        description: "Origin of the match: a whitelisted external plugin repository."
+                      - type: "null"
+                    description: Pinned origin from the marketplace entry, or null when an installed copy has no catalog entry.
                   readme:
                     anyOf:
                       - type: string
@@ -19475,13 +19469,12 @@ paths:
       operationId: plugins_install_post
       summary: Install a plugin
       description:
-        "Install a plugin by name from the canonical source — a whitelisted `plugins/marketplace.json` entry, else
-        the first-party `plugins/<name>/` convention. Always resolves against the curated default git ref (no
-        caller-supplied ref): installing from an unreviewed revision would bypass the marketplace/first-party curation
-        boundary and let attacker-controlled code be loaded. Materializes the plugin under
-        `<workspaceDir>/plugins/<name>/`; the assistant must be restarted to load it. Mirrors the CLI's `assistant
-        plugins install <name>`. An already-installed name without `force` returns 409; a name that resolves to nothing
-        returns 404. Sibling to `POST /v1/skills/install`."
+        "Install a plugin by name from the canonical source — a whitelisted `plugins/marketplace.json` entry.
+        Always resolves against the curated default git ref (no caller-supplied ref): installing from an unreviewed
+        revision would bypass the marketplace curation boundary and let attacker-controlled code be loaded. Materializes
+        the plugin under `<workspaceDir>/plugins/<name>/`; the assistant must be restarted to load it. Mirrors the CLI's
+        `assistant plugins install <name>`. An already-installed name without `force` returns 409; a name that resolves
+        to nothing returns 404. Sibling to `POST /v1/skills/install`."
       tags:
         - plugins
       responses:
@@ -19529,7 +19522,7 @@ paths:
               properties:
                 name:
                   type: string
-                  description: Install name to resolve against the catalog (first-party directory or marketplace entry).
+                  description: Install name to resolve against the marketplace catalog.
                 force:
                   description: Overwrite an existing install in place. Defaults to false.
                   type: boolean
@@ -19541,9 +19534,9 @@ paths:
       operationId: plugins_search_get
       summary: Search the plugin catalog
       description:
-        List installable plugins from the canonical `vellum-ai/vellum-assistant` catalog at `plugins/`. The query
-        is an ECMAScript regex matched case-insensitively against the directory name (e.g. `memory`, `^simple`). Empty
-        query returns every entry. Mirrors the CLI's `assistant plugins search`.
+        List installable plugins from the curated `plugins/marketplace.json` catalog. The query is an ECMAScript
+        regex matched case-insensitively against the plugin name (e.g. `memory`, `^simple`). Empty query returns every
+        entry. Mirrors the CLI's `assistant plugins search`.
       tags:
         - plugins
       responses:
@@ -19570,42 +19563,31 @@ paths:
                           description: Install name. Matches `assistant plugins install <name>`.
                         path:
                           type: string
-                          description:
-                            "Human-readable origin: a repo-relative path (`plugins/<name>`) for first-party plugins or a
-                            `github:owner/repo@ref` locator for external ones."
+                          description: "Human-readable origin: a `github:owner/repo@ref` locator for the external plugin."
                         description:
                           description: Short description, when known (external entries only today).
                           type: string
                         source:
-                          oneOf:
-                            - type: object
-                              properties:
-                                kind:
-                                  type: string
-                                  const: first-party
-                              required:
-                                - kind
-                              additionalProperties: false
-                            - type: object
-                              properties:
-                                kind:
-                                  type: string
-                                  const: github
-                                repo:
-                                  type: string
-                                  description: "`owner/repo` of the external plugin repository."
-                                path:
-                                  description: Directory within the repo, when the plugin is not at the root.
-                                  type: string
-                                ref:
-                                  type: string
-                                  description: Pinned git ref the plugin is fetched from.
-                              required:
-                                - kind
-                                - repo
-                                - ref
-                              additionalProperties: false
-                          description: "Origin of the match: a first-party in-repo plugin or a whitelisted external one."
+                          type: object
+                          properties:
+                            kind:
+                              type: string
+                              const: github
+                            repo:
+                              type: string
+                              description: "`owner/repo` of the external plugin repository."
+                            path:
+                              description: Directory within the repo, when the plugin is not at the root.
+                              type: string
+                            ref:
+                              type: string
+                              description: Pinned git ref the plugin is fetched from.
+                          required:
+                            - kind
+                            - repo
+                            - ref
+                          additionalProperties: false
+                          description: "Origin of the match: a whitelisted external plugin repository."
                       required:
                         - name
                         - path

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.8.10-dev.202606101436.d73da44",
+  "version": "0.8.10-dev.202606101714.d4b22de",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/__tests__/agent-loop.test.ts

@@ -8,6 +8,7 @@ import type {
 import { AgentLoop } from "../agent/loop.js";
 import { REFUSAL_FALLBACK_TEXT } from "../plugins/defaults/empty-response/hooks/stop.js";
 import { resetPluginRegistryAndRegisterDefaults } from "../plugins/defaults/index.js";
+import { registerPlugin } from "../plugins/registry.js";
 import type {
   ContentBlock,
   Message,
@@ -362,6 +363,219 @@ describe("AgentLoop", () => {
     expect(events.filter((e) => e.type === "error")).toHaveLength(1);
   });
 
+  // 5c. Reactive ordering-error recovery
+  //
+  // When the provider rejects a call because the history violates
+  // tool-use/tool-result pairing or role-alternation rules, the loop runs
+  // `deepRepairHistory` directly and re-issues the call. It is bounded to a
+  // single repair pass per provider call, so a second consecutive ordering
+  // rejection falls through to the error path instead of looping forever.
+  test("repairs the history and re-issues on a provider ordering rejection", async () => {
+    // GIVEN a provider that rejects the first call with an ordering error and
+    // succeeds on the retry
+    const { provider, calls } = createMockProvider([
+      new Error(
+        "tool_result blocks that are not immediately after a tool_use block",
+      ),
+      textResponse("recovered"),
+    ]);
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+    });
+    const events: AgentEvent[] = [];
+
+    // WHEN the loop runs
+    const { history } = await loop.run({
+      messages: [userMessage],
+      onEvent: collectEvents(events),
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    // THEN it re-issued the call after repairing, surfaced no error, and
+    // appended the recovered assistant response
+    expect(calls).toHaveLength(2);
+    expect(events.filter((e) => e.type === "error")).toHaveLength(0);
+    expect(history[history.length - 1]).toEqual({
+      role: "assistant",
+      content: [{ type: "text", text: "recovered" }],
+    });
+  });
+
+  test("surfaces an ordering error after a single repair fails to recover", async () => {
+    // GIVEN a provider that rejects every call with an ordering error
+    const { provider, calls } = createMockProvider([
+      new Error("tool_use_id provided without a matching tool_result"),
+    ]);
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+    });
+    const events: AgentEvent[] = [];
+
+    // WHEN the loop runs
+    await loop.run({
+      messages: [userMessage],
+      onEvent: collectEvents(events),
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    // THEN it repaired once, re-issued once, then surfaced the error rather
+    // than retrying again
+    expect(calls).toHaveLength(2);
+    expect(events.filter((e) => e.type === "error")).toHaveLength(1);
+  });
+
+  test("reports only the retry's output as new messages when deep-repair shrinks the base", async () => {
+    // The wrapper reconstructs the persisted transcript from the loop's
+    // `newMessages` tail, so after `deepRepairHistory` removes a base message
+    // the new-message boundary must track the re-normalized history — otherwise
+    // the recovered assistant response is sliced off and lost.
+
+    // GIVEN an input whose leading assistant message deep-repair strips, a
+    // provider that rejects the first call on ordering grounds, then succeeds
+    const leadingAssistantMessage: Message = {
+      role: "assistant",
+      content: [{ type: "text", text: "stale leading turn" }],
+    };
+    const { provider } = createMockProvider([
+      new Error(
+        "tool_result blocks that are not immediately after a tool_use block",
+      ),
+      textResponse("recovered"),
+    ]);
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+    });
+
+    // WHEN the loop runs
+    const { history, newMessages } = await loop.run({
+      messages: [leadingAssistantMessage, userMessage],
+      onEvent: () => {},
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    // THEN deep-repair dropped the leading assistant from the base, and
+    // `newMessages` is exactly the retry's appended response
+    expect(history).toEqual([
+      userMessage,
+      { role: "assistant", content: [{ type: "text", text: "recovered" }] },
+    ]);
+    expect(newMessages).toEqual([
+      { role: "assistant", content: [{ type: "text", text: "recovered" }] },
+    ]);
+  });
+
+  test("repairs a fresh ordering rejection on a later run after a prior run left the bound spent", async () => {
+    // The ordering-repair bound is owned by the history-repair plugin's
+    // per-conversation state. A run scopes that bound, so the loop clears it on
+    // entry — otherwise a direct `run` caller that bypasses the daemon
+    // orchestrator (e.g. an agent wake) inherits a spent bound from a prior run
+    // on the same conversation and surfaces a repairable rejection instead of
+    // repairing it.
+
+    // GIVEN a first run that rejects on ordering grounds, repairs once, then
+    // rejects again — ending with the bound spent and the error surfaced
+    const { provider: firstProvider, calls: firstCalls } = createMockProvider([
+      new Error("tool_use_id provided without a matching tool_result"),
+      new Error("tool_use_id provided without a matching tool_result"),
+    ]);
+    const conversationId = "ordering-bound-across-runs";
+    const firstLoop = new AgentLoop({
+      provider: firstProvider,
+      systemPrompt: "system",
+      conversationId,
+    });
+    const firstEvents: AgentEvent[] = [];
+    await firstLoop.run({
+      messages: [userMessage],
+      onEvent: collectEvents(firstEvents),
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+    expect(firstCalls).toHaveLength(2);
+    expect(firstEvents.filter((e) => e.type === "error")).toHaveLength(1);
+
+    // AND a second run on the same conversation whose first call rejects on
+    // ordering grounds, then succeeds on the retry
+    const { provider: secondProvider, calls: secondCalls } = createMockProvider(
+      [
+        new Error(
+          "tool_result blocks that are not immediately after a tool_use block",
+        ),
+        textResponse("recovered"),
+      ],
+    );
+    const secondLoop = new AgentLoop({
+      provider: secondProvider,
+      systemPrompt: "system",
+      conversationId,
+    });
+    const secondEvents: AgentEvent[] = [];
+
+    // WHEN the second run executes
+    const { history } = await secondLoop.run({
+      messages: [userMessage],
+      onEvent: collectEvents(secondEvents),
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    // THEN it repaired and re-issued (the bound did not leak across runs),
+    // surfaced no error, and appended the recovered response
+    expect(secondCalls).toHaveLength(2);
+    expect(secondEvents.filter((e) => e.type === "error")).toHaveLength(0);
+    expect(history[history.length - 1]).toEqual({
+      role: "assistant",
+      content: [{ type: "text", text: "recovered" }],
+    });
+  });
+
+  test("a stop-hook throw on a successful stop surfaces through the error path without re-entering the stop chain", async () => {
+    // The error-stop recovery is confined to genuine provider rejections. A
+    // throw from the success-path stop chain must not re-enter that chain (the
+    // same hook would throw again and escape the loop, bypassing the generic
+    // error handling) — it must fall through to the standard error path.
+
+    // GIVEN a registered stop hook that always throws, and a provider that
+    // returns a successful no-tool response (a successful stop)
+    registerPlugin({
+      manifest: { name: "throwing-stop", version: "0.0.1" },
+      hooks: {
+        stop: async () => {
+          throw new Error("stop hook boom");
+        },
+      },
+    });
+    const { provider, calls } = createMockProvider([textResponse("done")]);
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+    });
+    const events: AgentEvent[] = [];
+
+    // WHEN the loop runs
+    let rejected = false;
+    await loop
+      .run({
+        messages: [userMessage],
+        onEvent: collectEvents(events),
+        trust: { sourceChannel: "vellum", trustClass: "unknown" },
+      })
+      .catch(() => {
+        rejected = true;
+      });
+
+    // THEN the loop handled the throw via the error path (it did not reject),
+    // surfaced exactly one error, and did not re-issue the call
+    expect(rejected).toBe(false);
+    expect(calls).toHaveLength(1);
+    expect(events.filter((e) => e.type === "error")).toHaveLength(1);
+  });
+
   // 6. Abort signal — verify the loop respects AbortSignal
   test("stops when abort signal is triggered before provider call", async () => {
     const controller = new AbortController();

package/src/__tests__/agent-wake-disk-pressure-callsite.test.ts

@@ -76,7 +76,6 @@ function makeTarget(): Conversation {
       run: async ({ messages }: { messages: Message[] }) => ({
         history: messages,
         exitReason: null,
-        appendedNewMessages: false,
         newMessages: [],
       }),
     },

package/src/__tests__/conversation-abort-tool-results.test.ts

@@ -274,7 +274,6 @@ mock.module("../agent/loop.js", () => ({
       return {
         history,
         exitReason: null,
-        appendedNewMessages: true,
         newMessages: history.slice(messages.length),
       };
     }

package/src/__tests__/conversation-agent-loop-inference-profile.test.ts

@@ -264,6 +264,7 @@ mock.module("../plugins/defaults/history-repair/terminal.js", () => ({
     },
   }),
   deepRepairHistory: (msgs: Message[]) => ({ messages: msgs, stats: {} }),
+  isRepairableOrderingError: () => false,
 }));
 
 mock.module("../daemon/conversation-usage.js", () => ({
@@ -331,10 +332,6 @@ mock.module("../daemon/conversation-error.js", () => ({
   isContextTooLarge: () => false,
 }));
 
-mock.module("../daemon/conversation-slash.js", () => ({
-  isProviderOrderingError: () => false,
-}));
-
 mock.module("../util/truncate.js", () => ({
   truncate: (s: string) => s,
 }));

package/src/__tests__/conversation-agent-loop-overflow.test.ts

@@ -380,6 +380,7 @@ mock.module("../plugins/defaults/history-repair/terminal.js", () => ({
     },
   }),
   deepRepairHistory: (msgs: Message[]) => ({ messages: msgs, stats: {} }),
+  isRepairableOrderingError: () => false,
 }));
 
 const recordUsageMock = mock((..._args: unknown[]) => {});
@@ -479,11 +480,6 @@ mock.module("../daemon/conversation-error.js", () => ({
   }),
 }));
 
-mock.module("../daemon/conversation-slash.js", () => ({
-  isProviderOrderingError: (msg: string) =>
-    /ordering|before.*after|messages.*order/i.test(msg),
-}));
-
 mock.module("../util/truncate.js", () => ({
   truncate: (s: string) => s,
 }));

package/src/__tests__/conversation-agent-loop.test.ts

@@ -496,6 +496,7 @@ mock.module("../plugins/defaults/history-repair/terminal.js", () => ({
     },
   }),
   deepRepairHistory: (msgs: Message[]) => ({ messages: msgs, stats: {} }),
+  isRepairableOrderingError: () => false,
 }));
 
 const recordUsageMock = mock(() => {});
@@ -573,11 +574,6 @@ mock.module("../daemon/conversation-error.js", () => ({
   isContextTooLarge: (msg: string) => /context.?length.?exceeded/i.test(msg),
 }));
 
-mock.module("../daemon/conversation-slash.js", () => ({
-  isProviderOrderingError: (msg: string) =>
-    /ordering|before.*after|messages.*order/i.test(msg),
-}));
-
 mock.module("../util/truncate.js", () => ({
   truncate: (s: string) => s,
 }));
@@ -1545,40 +1541,6 @@ describe("session-agent-loop", () => {
     });
   });
 
-  describe("provider ordering error retry", () => {
-    test("retries with deep repair when ordering error is detected", async () => {
-      const events: ServerMessage[] = [];
-
-      // The provider rejects the first call with an ordering error, then
-      // succeeds once the orchestrator's deep repair re-sends the turn.
-      const { provider, calls } = createMockProvider([
-        new Error("messages ordering error"),
-        textResponse("fixed"),
-      ]);
-
-      const ctx = makeCtx({ loopProvider: provider });
-      await runAgentLoopImpl(ctx, "hello", "msg-1", (msg) => events.push(msg));
-
-      expect(calls.length).toBe(2);
-    });
-
-    test("emits deferred ordering error when retry also fails", async () => {
-      const events: ServerMessage[] = [];
-
-      // The provider rejects every call with an ordering error, so even the
-      // deep-repair retry fails and the orchestrator surfaces the error.
-      const ctx = makeCtx({
-        providerResponses: [new Error("messages ordering error")],
-      });
-      await runAgentLoopImpl(ctx, "hello", "msg-1", (msg) => events.push(msg));
-
-      const conversationError = events.find(
-        (e) => e.type === "conversation_error",
-      );
-      expect(conversationError).toBeDefined();
-    });
-  });
-
   describe("checkpoint handoff (infinite loop prevention)", () => {
     test("yields at checkpoint when canHandoffAtCheckpoint returns true", async () => {
       const events: ServerMessage[] = [];
@@ -3194,10 +3156,10 @@ describe("session-agent-loop", () => {
         estimatedTokens: 5000,
       });
 
-      // GIVEN a real loop that appends a tool turn (so the run reports
-      // `appendedNewMessages`) and then rejects with a context-too-large
-      // error on the following call — reactive overflow recovery strips that
-      // appended history when it compacts before a final call recovers.
+      // GIVEN a real loop that appends a tool turn and then rejects with a
+      // context-too-large error on the following call — reactive overflow
+      // recovery strips that appended history when it compacts before a final
+      // call recovers.
       const ctx = makeCtx({
         providerResponses: [
           toolUseResponse("t1", "file_read", {}),

package/src/__tests__/conversation-provider-retry-repair.test.ts

@@ -199,10 +199,11 @@ mock.module("../agent/loop.js", () => ({
       await onEvent({ type: "llm_call_started" });
       agentLoopRunCount++;
 
-      // The ordering error only fires on the first call; the wrapper's
-      // deep-repair retry then re-runs the loop, which succeeds.
-      const shouldError =
-        firstRunErrorMode === "ordering" && agentLoopRunCount === 1;
+      // Ordering-error recovery lives inside `AgentLoop.run`; with the loop
+      // mocked here, an unrecovered ordering error surfaces as a terminal
+      // `error` event so we can assert the wrapper's surface behavior — it
+      // routes the error to a durable conversation_error and does not re-run.
+      const shouldError = firstRunErrorMode === "ordering";
 
       if (shouldError) {
         onEvent({
@@ -220,7 +221,6 @@ mock.module("../agent/loop.js", () => ({
         return {
           history: [...messages],
           exitReason: null,
-          appendedNewMessages: false,
           newMessages: [],
         };
       }
@@ -243,7 +243,6 @@ mock.module("../agent/loop.js", () => ({
       return {
         history,
         exitReason: null,
-        appendedNewMessages: true,
         newMessages: history.slice(messages.length),
       };
     }
@@ -300,7 +299,7 @@ function makeConversation(): Conversation {
   );
 }
 
-describe("provider ordering error retry", () => {
+describe("terminal provider ordering error", () => {
   beforeEach(() => {
     agentLoopRunCount = 0;
     firstRunErrorMode = "ordering";
@@ -311,29 +310,13 @@ describe("provider ordering error retry", () => {
     resetPluginRegistryAndRegisterDefaults();
   });
 
-  test("simulated strict provider error triggers exactly one retry", async () => {
+  test("surfaces a durable conversation error without re-running the loop", async () => {
+    // GIVEN the agent loop reports an unrecovered ordering error
     firstRunErrorMode = "ordering";
-
-    const conversation = makeConversation();
-    await conversation.loadFromDb();
-
-    const events: Array<Record<string, unknown>> = [];
-    await conversation.processMessage({
-      content: "Hello",
-      attachments: [],
-      onEvent: (msg) => events.push(msg as unknown as Record<string, unknown>),
-    });
-
-    // Should have been called exactly 2 times: original + one retry
-    expect(agentLoopRunCount).toBe(2);
-  });
-
-  test("[experimental] retry succeeds with repaired history and no spurious error event", async () => {
-    firstRunErrorMode = "ordering";
-
     const conversation = makeConversation();
     await conversation.loadFromDb();
 
+    // WHEN the conversation processes a message
     const events: Array<Record<string, unknown>> = [];
     await conversation.processMessage({
       content: "Hello",
@@ -341,26 +324,26 @@ describe("provider ordering error retry", () => {
       onEvent: (msg) => events.push(msg as unknown as Record<string, unknown>),
     });
 
-    // Should have a message_complete event (from successful retry)
-    const messageComplete = events.find((e) => e.type === "message_complete");
-    expect(messageComplete).toBeDefined();
-
-    // Ordering error should be suppressed when retry succeeds — no error events
-    const errorEvents = events.filter((e) => e.type === "error");
-    expect(errorEvents.length).toBe(0);
+    // THEN the wrapper invokes the loop exactly once — ordering recovery is
+    // the loop's responsibility, not a wrapper-level re-run
+    expect(agentLoopRunCount).toBe(1);
 
-    // Should also have the assistant response in memory
-    const messages = conversation.getMessages();
-    const lastMsg = messages[messages.length - 1];
-    expect(lastMsg.role).toBe("assistant");
+    // AND the error is surfaced as a durable conversation_error consistent
+    // with every other terminal provider error
+    const conversationError = events.find(
+      (e) => e.type === "conversation_error",
+    );
+    expect(conversationError).toBeDefined();
+    expect(conversationError?.code).toBe("PROVIDER_ORDERING");
   });
 
-  test("non-ordering errors do not trigger retry", async () => {
+  test("runs the loop once and emits no error on a normal turn", async () => {
+    // GIVEN the agent loop completes the turn without error
     firstRunErrorMode = "none";
-
     const conversation = makeConversation();
     await conversation.loadFromDb();
 
+    // WHEN the conversation processes a message
     const events: Array<Record<string, unknown>> = [];
     await conversation.processMessage({
       content: "Hello",
@@ -368,7 +351,10 @@ describe("provider ordering error retry", () => {
       onEvent: (msg) => events.push(msg as unknown as Record<string, unknown>),
     });
 
-    // Should have been called exactly 1 time (no retry for non-ordering errors)
+    // THEN the loop ran once and the assistant response was persisted
     expect(agentLoopRunCount).toBe(1);
+    expect(events.some((e) => e.type === "conversation_error")).toBe(false);
+    const messages = conversation.getMessages();
+    expect(messages[messages.length - 1].role).toBe("assistant");
   });
 });