@vellumai/assistant 0.12.2-staging.2 → 0.12.2-staging.4

package/src/__tests__/host-cu-proxy.test.ts

@@ -553,7 +553,7 @@ describe("HostCuProxy", () => {
       });
       const result2 = await p2;
       // First unchanged: simple warning
-      expect(result2.content).toContain("NO VISIBLE EFFECT");
+      expect(result2.content).toContain("tree did not change");
       expect(result2.content).not.toContain("2 consecutive");
 
       // Third request — still same AX tree, no diff (unchanged step 2)
@@ -571,7 +571,7 @@ describe("HostCuProxy", () => {
       const result3 = await p3;
       // Should now have the consecutive warning
       expect(result3.content).toContain(
-        "2 consecutive actions had NO VISIBLE EFFECT",
+        "did not change across 2 consecutive actions",
       );
     });
 
@@ -592,7 +592,93 @@ describe("HostCuProxy", () => {
         // No axDiff on first observation — this is normal, not unchanged
       });
       const result1 = await p1;
-      expect(result1.content).not.toContain("NO VISIBLE EFFECT");
+      expect(result1.content).not.toContain("tree did not change");
+    });
+
+    async function stepWithUnchangedTree(
+      toolName: string,
+      input: Record<string, unknown>,
+      step: number,
+    ) {
+      const pending = proxy.request(toolName, input, "session-1", step);
+      proxy.recordAction(toolName, input);
+      const sent = sentMessages[step - 1] as Record<string, unknown>;
+      proxy.processObservation(sent.requestId as string, {
+        axTree: "Button [1]",
+      });
+      return pending;
+    }
+
+    test("an AppleScript or an observation neither warns nor moves the streak", async () => {
+      for (const [toolName, input] of [
+        [
+          "computer_use_run_applescript",
+          { script: 'tell application "Finder" to activate' },
+        ],
+        ["computer_use_observe", {}],
+      ] as const) {
+        setup();
+        await stepWithUnchangedTree("computer_use_click", { element_id: 1 }, 1);
+
+        const result = await stepWithUnchangedTree(toolName, input, 2);
+
+        expect(result.content).not.toContain("tree did not change");
+        expect(proxy.consecutiveUnchangedSteps).toBe(0);
+      }
+    });
+
+    test("judges each concurrent response by the action it carried", async () => {
+      setup();
+      await stepWithUnchangedTree("computer_use_click", { element_id: 1 }, 1);
+
+      // A click and an observation dispatched from one model response, with
+      // the observation recorded last and the click answered last.
+      const clickPromise = proxy.request(
+        "computer_use_click",
+        { element_id: 1 },
+        "session-1",
+        2,
+      );
+      proxy.recordAction("computer_use_click", { element_id: 1 });
+      const observePromise = proxy.request(
+        "computer_use_observe",
+        {},
+        "session-1",
+        3,
+      );
+      proxy.recordAction("computer_use_observe", {});
+      const click = sentMessages[1] as Record<string, unknown>;
+      const observe = sentMessages[2] as Record<string, unknown>;
+
+      proxy.processObservation(observe.requestId as string, {
+        axTree: "Button [1]",
+      });
+      proxy.processObservation(click.requestId as string, {
+        axTree: "Button [1]",
+      });
+
+      expect((await observePromise).content).not.toContain(
+        "tree did not change",
+      );
+      expect((await clickPromise).content).toContain("tree did not change");
+      expect(proxy.consecutiveUnchangedSteps).toBe(1);
+    });
+
+    test("an observation between two unchanged clicks keeps the streak", async () => {
+      setup();
+      await stepWithUnchangedTree("computer_use_click", { element_id: 1 }, 1);
+      await stepWithUnchangedTree("computer_use_click", { element_id: 1 }, 2);
+      await stepWithUnchangedTree("computer_use_observe", {}, 3);
+
+      const result = await stepWithUnchangedTree(
+        "computer_use_click",
+        { element_id: 1 },
+        4,
+      );
+
+      expect(result.content).toContain(
+        "did not change across 2 consecutive actions",
+      );
     });
 
     test("skips unchanged warning after computer_use_wait", async () => {
@@ -626,7 +712,7 @@ describe("HostCuProxy", () => {
         // No axDiff — screen unchanged, but that's expected after wait
       });
       const result2 = await p2;
-      expect(result2.content).not.toContain("NO VISIBLE EFFECT");
+      expect(result2.content).not.toContain("tree did not change");
     });
 
     test("skips unchanged warning and counter after a selection-only key (cmd+a)", async () => {
@@ -661,7 +747,7 @@ describe("HostCuProxy", () => {
         // No axDiff — selection change is invisible in the AX tree
       });
       const result2 = await p2;
-      expect(result2.content).not.toContain("NO VISIBLE EFFECT");
+      expect(result2.content).not.toContain("tree did not change");
       expect(proxy.consecutiveUnchangedSteps).toBe(0);
     });
 
@@ -707,7 +793,7 @@ describe("HostCuProxy", () => {
           // No axDiff — invisible-by-design change
         });
         const result2 = await p2;
-        expect(result2.content).not.toContain("NO VISIBLE EFFECT");
+        expect(result2.content).not.toContain("tree did not change");
         expect(proxy.consecutiveUnchangedSteps).toBe(0);
       }
     });
@@ -807,7 +893,7 @@ describe("HostCuProxy", () => {
         axTree: "Button [1]",
       });
       const result2 = await p2;
-      expect(result2.content).toContain("NO VISIBLE EFFECT");
+      expect(result2.content).toContain("tree did not change");
       expect(proxy.consecutiveUnchangedSteps).toBe(1);
     });
 
@@ -840,7 +926,7 @@ describe("HostCuProxy", () => {
         axTree: "Button [1]",
       });
       const result2 = await p2;
-      expect(result2.content).toContain("NO VISIBLE EFFECT");
+      expect(result2.content).toContain("tree did not change");
       expect(proxy.consecutiveUnchangedSteps).toBe(1);
     });
 
@@ -860,7 +946,7 @@ describe("HostCuProxy", () => {
       expect(result.content).toContain(
         "WARNING: You've repeated the same action (computer_use_key) 3 times",
       );
-      expect(result.content).not.toContain("NO VISIBLE EFFECT");
+      expect(result.content).not.toContain("tree did not change");
     });
 
     test("loop detection fires for the same exempt combo spelled differently", () => {
@@ -1706,9 +1792,21 @@ describe("HostCuProxy", () => {
     async function finish(
       input: Record<string, unknown>,
       observation: Record<string, string>,
+      // The step the unchanged streak is judged on. An observation leaves the
+      // streak alone, so a test of how snapshots reset it stands in an action.
+      toolName = "computer_use_observe",
     ) {
-      proxy.recordAction("computer_use_observe", input);
-      const pending = observe(input);
+      proxy.recordAction(toolName, input);
+      const pending = proxy.request(
+        toolName,
+        input,
+        "session-1",
+        1,
+        undefined,
+        undefined,
+        undefined,
+        "user-1",
+      );
       const sent = sentMessages.at(-1) as { requestId: string };
       proxy.processObservation(sent.requestId, observation);
       return await pending;
@@ -1777,8 +1875,8 @@ describe("HostCuProxy", () => {
     test("targeted snapshots reset history without no-effect warnings or cross-window diffs", async () => {
       setup();
       connect();
-      await finish({}, { axTree: "Private desktop" });
-      await finish({}, { axTree: "Private desktop" });
+      await finish({}, { axTree: "Private desktop" }, "computer_use_click");
+      await finish({}, { axTree: "Private desktop" }, "computer_use_click");
       expect(proxy.consecutiveUnchangedSteps).toBe(1);
       for (const id of [12, 12, 13]) {
         const result = await finish(
@@ -1789,16 +1887,16 @@ describe("HostCuProxy", () => {
             secondaryWindows: "Private secondary window",
           },
         );
-        expect(result.content).not.toContain("NO VISIBLE EFFECT");
+        expect(result.content).not.toContain("tree did not change");
         expect(result.content).not.toContain("Private");
         expect(proxy.previousAXTree).toBeUndefined();
         expect(proxy.consecutiveUnchangedSteps).toBe(0);
       }
       const desktop = await finish({}, { axTree: "Returned desktop" });
-      expect(desktop.content).not.toContain("NO VISIBLE EFFECT");
+      expect(desktop.content).not.toContain("tree did not change");
       expect(proxy.previousAXTree).toBe("Returned desktop");
       expect(proxy.consecutiveUnchangedSteps).toBe(0);
-      await finish({}, { axTree: "Returned desktop" });
+      await finish({}, { axTree: "Returned desktop" }, "computer_use_click");
       expect(proxy.consecutiveUnchangedSteps).toBe(1);
     });
     test("failed targeted captures also break the desktop comparison baseline", async () => {
@@ -1815,7 +1913,7 @@ describe("HostCuProxy", () => {
       ).toBe(true);
       expect(proxy.previousAXTree).toBeUndefined();
       const result = await finish({}, { axTree: "Desktop" });
-      expect(result.content).not.toContain("NO VISIBLE EFFECT");
+      expect(result.content).not.toContain("tree did not change");
     });
     test("legacy camel-case companion targets also reset observation history", async () => {
       setup();
@@ -1825,7 +1923,7 @@ describe("HostCuProxy", () => {
         { captureDisplayId: 1 },
         { axTree: "Display" },
       );
-      expect(result.content).not.toContain("NO VISIBLE EFFECT");
+      expect(result.content).not.toContain("tree did not change");
       expect(proxy.previousAXTree).toBeUndefined();
     });
   });

package/src/__tests__/list-all-apps.test.ts

@@ -22,18 +22,29 @@ function freshWorkspace(): string {
 
 /**
  * Materialize an installed plugin under `<workspace>/plugins/<name>/` with a
- * package.json manifest. Returns the plugin directory path.
+ * plugin manifest. Returns the plugin directory path.
  */
 function installPlugin(
   name: string,
-  opts: { disabled?: boolean } = {},
+  opts: { disabled?: boolean; standard?: boolean } = {},
 ): string {
   const pluginDir = join(getWorkspacePluginsDir(), name);
   mkdirSync(pluginDir, { recursive: true });
-  writeFileSync(
-    join(pluginDir, "package.json"),
-    JSON.stringify({ name, version: "1.0.0" }),
-  );
+  if (opts.standard) {
+    writeFileSync(
+      join(pluginDir, "plugin.json"),
+      JSON.stringify({
+        $schema: "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
+        name,
+        version: "1.0.0",
+      }),
+    );
+  } else {
+    writeFileSync(
+      join(pluginDir, "package.json"),
+      JSON.stringify({ name, version: "1.0.0" }),
+    );
+  }
   if (opts.disabled) {
     writeFileSync(join(pluginDir, ".disabled"), "");
   }
@@ -104,7 +115,16 @@ describe("listAllApps", () => {
     expect(all.map((a) => a.origin.kind)).toEqual(["workspace", "plugin"]);
   });
 
-  test("ignores directories under plugins/ without a package.json manifest", () => {
+  test("discovers apps from a standard-only plugin", () => {
+    const pluginDir = installPlugin("standard-plugin", { standard: true });
+    bundleApp(pluginDir, "dashboard");
+
+    expect(listPluginApps().map((app) => app.id)).toEqual([
+      "plugins~standard-plugin~dashboard",
+    ]);
+  });
+
+  test("ignores directories under plugins/ without a root manifest", () => {
     // A stray directory that is not an installed plugin, even with an apps/ dir.
     const strayDir = join(getWorkspacePluginsDir(), "not-a-plugin");
     mkdirSync(join(strayDir, "apps", "ghost"), { recursive: true });
@@ -166,6 +186,18 @@ describe("resolveAppSource", () => {
     expect(source!.sourceDir).toBe(join(pluginDir, "apps", "acme-dashboard"));
   });
 
+  test("resolves an app from a standard-only plugin", () => {
+    const pluginDir = installPlugin("standard-plugin", { standard: true });
+    bundleApp(pluginDir, "dashboard");
+
+    const source = resolveAppSource("plugins~standard-plugin~dashboard");
+    expect(source?.sourceDir).toBe(join(pluginDir, "apps", "dashboard"));
+    expect(source?.origin).toEqual({
+      kind: "plugin",
+      pluginName: "standard-plugin",
+    });
+  });
+
   test("returns null for an unknown workspace id", () => {
     expect(resolveAppSource("11111111-1111-1111-1111-111111111111")).toBeNull();
   });
@@ -177,7 +209,7 @@ describe("resolveAppSource", () => {
     expect(resolveAppSource("plugins~acme~acme-dashboard")).toBeNull();
   });
 
-  test("returns null for a plugin dir without a package.json manifest", () => {
+  test("returns null for a plugin dir without a root manifest", () => {
     // Stray directory shaped like a plugin app but not an installed plugin.
     const strayApp = join(
       getWorkspacePluginsDir(),

package/src/__tests__/mcp-auth-routes.test.ts

@@ -3,11 +3,29 @@ import { beforeEach, describe, expect, mock, test } from "bun:test";
 // ── Module mocks (must precede imports) ───────────────────────────────────────
 
 const mockReloadMcpServers = mock(async () => {});
+let pluginServers: Array<{
+  id: string;
+  pluginName: string;
+  serverKey: string;
+  config: {
+    source: "plugin";
+    pluginName: string;
+    serverKey: string;
+    transport: {
+      type: "streamable-http";
+      url: string;
+    };
+  };
+}> = [];
 
 mock.module("../daemon/mcp-reload-service.js", () => ({
   reloadMcpServers: () => mockReloadMcpServers(),
 }));
 
+mock.module("../plugins/mcp-servers.js", () => ({
+  readPluginMcpServers: () => ({ servers: pluginServers, issues: [] }),
+}));
+
 const mockOrchestrateConnect = mock(
   async (_args: { serverId: string; transport: unknown }) => ({
     auth_url: "https://provider.example.com/authorize?state=abc",
@@ -55,6 +73,7 @@ describe("mcp-auth-routes", () => {
     mockGetMcpAuthState.mockClear();
     mockGetMcpAuthState.mockImplementation(() => null);
     mockReloadMcpServers.mockClear(); // ← add this line
+    pluginServers = [];
   });
 
   describe("POST internal/mcp/auth/start", () => {
@@ -68,6 +87,86 @@ describe("mcp-auth-routes", () => {
         auth_url: "https://provider.example.com/authorize?state=abc",
         state: "my-server",
       });
+      expect(mockOrchestrateConnect).toHaveBeenCalledWith(
+        expect.objectContaining({
+          serverId: "my-server",
+          credentialTarget: {
+            source: "workspace",
+            serverId: "my-server",
+          },
+        }),
+      );
+    });
+
+    test("resolves an installed plugin server by its public id", async () => {
+      pluginServers = [
+        {
+          id: "plugin-auth__remote",
+          pluginName: "plugin-auth",
+          serverKey: "remote",
+          config: {
+            source: "plugin",
+            pluginName: "plugin-auth",
+            serverKey: "remote",
+            transport: {
+              type: "streamable-http",
+              url: "https://mcp.example.com/plugin",
+            },
+          },
+        },
+      ];
+
+      const startRoute = findRoute("internal_mcp_auth_start");
+      await startRoute.handler({
+        body: { serverId: "plugin-auth__remote" },
+      });
+
+      expect(mockOrchestrateConnect).toHaveBeenCalledWith(
+        expect.objectContaining({
+          serverId: "plugin-auth__remote",
+          credentialTarget: {
+            source: "plugin",
+            pluginName: "plugin-auth",
+            serverKey: "remote",
+            transportType: "streamable-http",
+            url: "https://mcp.example.com/plugin",
+          },
+        }),
+      );
+    });
+
+    test("keeps workspace precedence when a plugin declares the same id", async () => {
+      pluginServers = [
+        {
+          id: "my-server",
+          pluginName: "my-server",
+          serverKey: "my-server",
+          config: {
+            source: "plugin",
+            pluginName: "my-server",
+            serverKey: "my-server",
+            transport: {
+              type: "streamable-http",
+              url: "https://loses.example.com/mcp",
+            },
+          },
+        },
+      ];
+
+      const startRoute = findRoute("internal_mcp_auth_start");
+      await startRoute.handler({ body: { serverId: "my-server" } });
+
+      expect(mockOrchestrateConnect).toHaveBeenCalledWith(
+        expect.objectContaining({
+          transport: expect.objectContaining({
+            url: "https://mcp.example.com",
+          }),
+          credentialTarget: {
+            source: "workspace",
+            serverId: "my-server",
+          },
+        }),
+      );
     });
 
     test("rejects unknown serverId with BadRequestError", async () => {

package/src/__tests__/mcp-client-auth.test.ts

@@ -130,6 +130,60 @@ describe("McpClient auth error detection", () => {
   });
 });
 
+describe("McpClient connection lifecycle", () => {
+  test("a close during connect cannot restore a stale connected state", async () => {
+    const onUnexpectedClose = jest.fn();
+    const client = new McpClient("test-server", "workspace", onUnexpectedClose);
+    let finishConnect: (() => void) | undefined;
+    (client as any).createTransport = () => ({});
+    (client as any).client.connect = () =>
+      new Promise<void>((resolve) => {
+        finishConnect = resolve;
+      });
+
+    const connecting = client.connect(httpTransport);
+    while (!finishConnect) {
+      await Promise.resolve();
+    }
+    (client as any).client.onclose();
+    finishConnect!();
+    await connecting;
+
+    expect(client.isConnected).toBe(false);
+    expect(client.lastError?.message).toContain("closed during initialization");
+    expect(onUnexpectedClose).not.toHaveBeenCalled();
+  });
+
+  test("reports an unexpected close after connecting", async () => {
+    const onUnexpectedClose = jest.fn();
+    const client = new McpClient("test-server", "workspace", onUnexpectedClose);
+    (client as any).createTransport = () => ({});
+    (client as any).client.connect = async () => {};
+
+    await client.connect(httpTransport);
+    (client as any).client.onclose();
+
+    expect(client.isConnected).toBe(false);
+    expect(onUnexpectedClose).toHaveBeenCalledTimes(1);
+  });
+
+  test("does not report an intentional disconnect as a transport failure", async () => {
+    const onUnexpectedClose = jest.fn();
+    const client = new McpClient("test-server", "workspace", onUnexpectedClose);
+    (client as any).createTransport = () => ({});
+    (client as any).client.connect = async () => {};
+    (client as any).client.close = async () => {
+      (client as any).client.onclose();
+    };
+
+    await client.connect(httpTransport);
+    await client.disconnect();
+
+    expect(client.isConnected).toBe(false);
+    expect(onUnexpectedClose).not.toHaveBeenCalled();
+  });
+});
+
 describe("McpOAuthProvider redirectUrl", () => {
   test("redirectUrl is undefined until startCallbackServer() is called", () => {
     const nonInteractive = new McpOAuthProvider(

package/src/__tests__/mcp-health-check.test.ts

@@ -1,23 +1,19 @@
 import { beforeEach, describe, expect, jest, mock, test } from "bun:test";
 
-const mockConnect = jest.fn();
-const mockDisconnect = jest.fn();
-let mockIsConnected = true;
-let mockLastError: Error | null = null;
+const getServerState = jest.fn();
 
 mock.module("../mcp/client.js", () => ({
   McpClient: class {
-    get isConnected() {
-      return mockIsConnected;
+    constructor() {
+      throw new Error("list route must not construct an MCP client");
     }
-    get lastError() {
-      return mockLastError;
-    }
-    connect = mockConnect;
-    disconnect = mockDisconnect;
   },
 }));
 
+mock.module("../mcp/manager.js", () => ({
+  getMcpServerManager: () => ({ getServerState }),
+}));
+
 import { setWorkspaceMcp } from "./helpers/set-workspace-mcp.js";
 
 // Seed the MCP server the list route reads from workspace mcp.json.
@@ -56,28 +52,22 @@ const listHandler = ROUTES.find(
   (r: { operationId: string }) => r.operationId === "internal_mcp_list",
 )!.handler;
 
-describe("checkServerHealth (via internal_mcp_list route)", () => {
+describe("internal_mcp_list runtime status", () => {
   beforeEach(() => {
-    mockConnect.mockReset();
-    mockDisconnect.mockReset();
-    mockIsConnected = true;
-    mockLastError = null;
+    getServerState.mockReset();
   });
 
-  test("returns connected when server connects successfully", async () => {
-    mockConnect.mockResolvedValue(undefined);
-    mockDisconnect.mockResolvedValue(undefined);
+  test("returns connected from the manager's recorded state", async () => {
+    getServerState.mockReturnValue("connected");
 
     const result = (await listHandler({})) as {
       servers: { status: string }[];
     };
     expect(result.servers[0].status).toBe("connected");
-    expect(mockDisconnect).toHaveBeenCalled();
   });
 
-  test("returns needs-auth when isConnected is false and no lastError", async () => {
-    mockConnect.mockResolvedValue(undefined);
-    mockIsConnected = false;
+  test("returns needs-auth from the manager's recorded state", async () => {
+    getServerState.mockReturnValue("needs-auth");
 
     const result = (await listHandler({})) as {
       servers: { status: string }[];
@@ -85,15 +75,25 @@ describe("checkServerHealth (via internal_mcp_list route)", () => {
     expect(result.servers[0].status).toBe("needs-auth");
   });
 
-  test("returns error when connect fails with lastError", async () => {
-    mockConnect.mockResolvedValue(undefined);
-    mockIsConnected = false;
-    mockLastError = new Error("Connection refused");
-    mockDisconnect.mockResolvedValue(undefined);
+  test.each([undefined, "connecting", "error"])(
+    "returns the legacy error status for runtime state %s",
+    async (runtimeState) => {
+      getServerState.mockReturnValue(runtimeState);
 
-    const result = (await listHandler({})) as {
-      servers: { status: string }[];
-    };
-    expect(result.servers[0].status).toBe("error");
+      const result = (await listHandler({})) as {
+        servers: { status: string }[];
+      };
+      expect(result.servers[0].status).toBe("error");
+    },
+  );
+
+  test("repeated reads do not connect or spawn a configured server", async () => {
+    getServerState.mockReturnValue("connected");
+
+    await listHandler({});
+    await listHandler({});
+
+    expect(getServerState).toHaveBeenCalledTimes(2);
+    expect(getServerState).toHaveBeenCalledWith("test", "workspace");
   });
 });