@checkstack/ai-backend 0.1.0

package/src/mcp/mcp-conformance.it.test.ts

@@ -0,0 +1,128 @@
+import { describe, expect, test } from "bun:test";
+
+/**
+ * MCP Streamable-HTTP wire conformance (matrix #9).
+ *
+ * Gated behind `CHECKSTACK_IT=1` so the default `bun test` never runs it. Drives
+ * a REAL running backend's MCP endpoint over HTTP to validate the
+ * initialize -> tools/list -> tools/call round-trip against a live, OAuth-token
+ * authenticated session. Configure:
+ * - `CHECKSTACK_IT_MCP_URL`   (defaults to http://localhost:3000/api/ai/mcp)
+ * - `CHECKSTACK_IT_MCP_TOKEN` (a minted opaque OAuth access token, NARROWED to a
+ *   read scope that does NOT include `ai.tools.manage` — so `ai.secrets` is out
+ *   of scope and a mutating tool like `incident.close` is not invocable)
+ *
+ * This is the only assertion that exercises the full opaque-token introspection
+ * + live-router re-entry path end-to-end, so it lives as an integration test.
+ * Phase 5 adds the negative wire assertions: an out-of-scope tool is refused
+ * (and the router is never re-entered), and a mutating tool is refused by the
+ * structural effect-gate (propose/apply only).
+ */
+const MCP_URL =
+  process.env.CHECKSTACK_IT_MCP_URL ?? "http://localhost:3000/api/ai/mcp";
+const MCP_TOKEN = process.env.CHECKSTACK_IT_MCP_TOKEN ?? "";
+/** A tool the IT token's scope does NOT grant (admin-only). */
+const OUT_OF_SCOPE_TOOL =
+  process.env.CHECKSTACK_IT_MCP_OUT_OF_SCOPE_TOOL ?? "ai.secrets";
+/** A mutating tool that must be refused via the effect-gate (never run). */
+const MUTATING_TOOL =
+  process.env.CHECKSTACK_IT_MCP_MUTATING_TOOL ?? "incident.close";
+
+async function rpc(body: unknown): Promise<{ status: number; json: unknown }> {
+  const res = await fetch(MCP_URL, {
+    method: "POST",
+    headers: {
+      "content-type": "application/json",
+      authorization: `Bearer ${MCP_TOKEN}`,
+    },
+    body: JSON.stringify(body),
+  });
+  const text = await res.text();
+  return { status: res.status, json: text ? JSON.parse(text) : null };
+}
+
+// Requires a REAL running backend (CHECKSTACK_IT_MCP_URL) reachable with a
+// minted, scope-narrowed OAuth token (CHECKSTACK_IT_MCP_TOKEN). The plain
+// `CHECKSTACK_IT` PG/Redis integration lane does not stand up that backend, so
+// gate on the token too - otherwise these tests fail with "Unable to connect"
+// instead of skipping where the prerequisite isn't provided.
+describe.skipIf(!process.env.CHECKSTACK_IT || !process.env.CHECKSTACK_IT_MCP_TOKEN)(
+  "MCP Streamable-HTTP conformance",
+  () => {
+    test("initialize advertises a protocol version and a session id", async () => {
+      const res = await fetch(MCP_URL, {
+        method: "POST",
+        headers: {
+          "content-type": "application/json",
+          authorization: `Bearer ${MCP_TOKEN}`,
+        },
+        body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "initialize" }),
+      });
+      const json = (await res.json()) as {
+        result?: { protocolVersion?: string };
+      };
+      expect(json.result?.protocolVersion).toBeTruthy();
+      expect(res.headers.get("mcp-session-id")).toBeTruthy();
+    });
+
+    test("tools/list returns the read-only tool surface", async () => {
+      const { json } = (await rpc({
+        jsonrpc: "2.0",
+        id: 2,
+        method: "tools/list",
+      })) as { json: { result?: { tools?: Array<{ name: string }> } } };
+      const names = (json.result?.tools ?? []).map((t) => t.name);
+      // The principal must at least be able to list incidents in the IT env.
+      expect(names).toContain("incident.list");
+    });
+
+    test("tools/call returns a non-error content block", async () => {
+      const { json } = (await rpc({
+        jsonrpc: "2.0",
+        id: 3,
+        method: "tools/call",
+        params: { name: "incident.list", arguments: {} },
+      })) as { json: { result?: { isError?: boolean } } };
+      expect(json.result?.isError).toBe(false);
+    });
+
+    test("tools/list never lists an out-of-scope tool", async () => {
+      const { json } = (await rpc({
+        jsonrpc: "2.0",
+        id: 4,
+        method: "tools/list",
+      })) as { json: { result?: { tools?: Array<{ name: string }> } } };
+      const names = (json.result?.tools ?? []).map((t) => t.name);
+      // The narrowed token does not grant the admin rule, so the tool is hidden.
+      expect(names).not.toContain(OUT_OF_SCOPE_TOOL);
+      // A mutating tool is never on the read-only list either.
+      expect(names).not.toContain(MUTATING_TOOL);
+    });
+
+    test("tools/call for an out-of-scope tool is REFUSED 403 (not merely hidden)", async () => {
+      const { status, json } = (await rpc({
+        jsonrpc: "2.0",
+        id: 5,
+        method: "tools/call",
+        params: { name: OUT_OF_SCOPE_TOOL, arguments: {} },
+      })) as { status: number; json: { error?: { message?: string } } };
+      // Handler-side authz holds when the model misbehaves: the resolver gate
+      // refuses BEFORE the live router is re-entered.
+      expect(status).toBe(403);
+      expect(json.error?.message).toBeTruthy();
+    });
+
+    test("tools/call for a mutating tool is refused by the structural effect-gate", async () => {
+      const { status, json } = (await rpc({
+        jsonrpc: "2.0",
+        id: 6,
+        method: "tools/call",
+        params: { name: MUTATING_TOOL, arguments: {} },
+      })) as { status: number; json: { error?: { message?: string } } };
+      // A bare tools/call may only run a read tool; mutating tools MUST go
+      // through propose/apply. The handler refuses with 403 and never executes.
+      expect(status).toBe(403);
+      expect(json.error?.message).toContain("propose/apply");
+    });
+  },
+);

package/src/mcp/server.test.ts

@@ -0,0 +1,285 @@
+import { describe, expect, test } from "bun:test";
+import { z } from "zod";
+import type { AuthService, AuthUser } from "@checkstack/backend-api";
+import { createMcpRequestHandler } from "./server";
+import type { McpExecutableTool } from "./server";
+import { createAiToolResolver } from "../resolver";
+import { createAiToolRegistry } from "../tool-registry";
+import type { McpToolInvoker } from "./tool-invoker";
+import { createMcpConnectionRegistry } from "./connection-registry";
+import type { RegisteredAiTool } from "../tool-registry";
+
+function readTool(name: string, rule: string): RegisteredAiTool {
+  return {
+    name,
+    description: `${name} (read-only)`,
+    effect: "read",
+    input: z.object({}),
+    requiredAccessRules: [rule],
+    execute: () => Promise.resolve({}),
+  };
+}
+
+function mutateTool(name: string, rule: string): RegisteredAiTool {
+  return {
+    name,
+    description: `${name} (mutating)`,
+    effect: "mutate",
+    input: z.object({}),
+    requiredAccessRules: [rule],
+    execute: () => Promise.resolve({}),
+  };
+}
+
+function buildHandler({
+  principal,
+  invoke,
+  enforceBudget,
+  recordExecuted,
+}: {
+  principal: AuthUser | undefined;
+  invoke?: McpToolInvoker["invoke"];
+  enforceBudget?: (p: { kind: string; id: string }) => Promise<void>;
+  recordExecuted?: (args: {
+    principal: { kind: string; id: string };
+    toolName: string;
+    argsHash: string;
+  }) => Promise<void>;
+}) {
+  const registry = createAiToolRegistry();
+  const incidentTool = readTool("incident.list", "incident.incident.read");
+  const adminTool = readTool("ai.secrets", "ai.tools.manage");
+  // A mutating tool the limited principal IS allowed for (same access rule as
+  // incident.list). The ONLY thing that may refuse a bare tools/call for it is
+  // the structural effect-gate, not the resolver.
+  const mutating = mutateTool("incident.close", "incident.incident.read");
+  registry.register(incidentTool);
+  registry.register(adminTool);
+  registry.register(mutating);
+  const resolver = createAiToolResolver({ registry });
+
+  const tools: McpExecutableTool[] = [
+    { tool: incidentTool, pluginId: "incident", procedureKey: "listIncidents" },
+    { tool: adminTool, pluginId: "ai", procedureKey: "secrets" },
+    { tool: mutating, pluginId: "incident", procedureKey: "closeIncident" },
+  ];
+
+  const invoker: McpToolInvoker = {
+    invoke: invoke ?? (() => Promise.resolve({ ok: true })),
+  };
+
+  const auth: AuthService = {
+    authenticate: () => Promise.resolve(principal),
+    getCredentials: () => Promise.resolve({ headers: {} }),
+    getAnonymousAccessRules: () => Promise.resolve([]),
+    checkResourceTeamAccess: () => Promise.resolve(false),
+  } as unknown as AuthService;
+
+  return createMcpRequestHandler({
+    tools,
+    resolver,
+    invoker,
+    auth,
+    connections: createMcpConnectionRegistry(),
+    enforceBudget,
+    recordExecuted,
+  });
+}
+
+function mcpPost(body: unknown, token = "opaque-token"): Request {
+  return new Request("http://localhost/api/ai/mcp", {
+    method: "POST",
+    headers: {
+      "content-type": "application/json",
+      authorization: `Bearer ${token}`,
+    },
+    body: JSON.stringify(body),
+  });
+}
+
+const limitedPrincipal: AuthUser = {
+  type: "user",
+  id: "u1",
+  accessRules: ["incident.incident.read"],
+};
+
+describe("MCP server (read-only Streamable-HTTP)", () => {
+  test("initialize returns protocol version + a session id header", async () => {
+    const handler = buildHandler({ principal: limitedPrincipal });
+    const res = await handler(
+      mcpPost({ jsonrpc: "2.0", id: 1, method: "initialize" }),
+    );
+    const json = await res.json();
+    expect(json.result.protocolVersion).toBeDefined();
+    expect(res.headers.get("mcp-session-id")).toBeTruthy();
+  });
+
+  test("tools/list only surfaces tools the principal may call", async () => {
+    const handler = buildHandler({ principal: limitedPrincipal });
+    const res = await handler(
+      mcpPost({ jsonrpc: "2.0", id: 2, method: "tools/list" }),
+    );
+    const json = await res.json();
+    const names = json.result.tools.map((t: { name: string }) => t.name);
+    expect(names).toEqual(["incident.list"]);
+    expect(names).not.toContain("ai.secrets");
+  });
+
+  test("tools/list returns 401 for an unauthenticated caller", async () => {
+    const handler = buildHandler({ principal: undefined });
+    const res = await handler(
+      mcpPost({ jsonrpc: "2.0", id: 3, method: "tools/list" }),
+    );
+    expect(res.status).toBe(401);
+  });
+
+  // Matrix #8: an MCP call for a tool OUTSIDE the token's scopes is rejected,
+  // not just hidden — handler-side authz holds when the model misbehaves.
+  test("tools/call for an out-of-scope tool is REFUSED (not merely hidden)", async () => {
+    let invoked = false;
+    const handler = buildHandler({
+      principal: limitedPrincipal,
+      invoke: () => {
+        invoked = true;
+        return Promise.resolve({});
+      },
+    });
+    // The model names a tool the principal lacks the rule for.
+    const res = await handler(
+      mcpPost({
+        jsonrpc: "2.0",
+        id: 4,
+        method: "tools/call",
+        params: { name: "ai.secrets", arguments: {} },
+      }),
+    );
+    expect(res.status).toBe(403);
+    // Crucially: the live router was NEVER re-entered for a forbidden tool.
+    expect(invoked).toBe(false);
+  });
+
+  test("tools/call for an allowed tool re-enters the router with the bearer token", async () => {
+    let seenToken: string | undefined;
+    let seenRoute: string | undefined;
+    const handler = buildHandler({
+      principal: limitedPrincipal,
+      invoke: ({ bearerToken, pluginId, procedureKey, input }) => {
+        seenToken = bearerToken;
+        seenRoute = `${pluginId}.${procedureKey}`;
+        return Promise.resolve({ echoed: input });
+      },
+    });
+    const res = await handler(
+      mcpPost(
+        {
+          jsonrpc: "2.0",
+          id: 5,
+          method: "tools/call",
+          params: { name: "incident.list", arguments: { status: "open" } },
+        },
+        "tok-123",
+      ),
+    );
+    const json = await res.json();
+    expect(json.result.isError).toBe(false);
+    // The caller's own token is forwarded so the router re-checks authz live.
+    expect(seenToken).toBe("tok-123");
+    expect(seenRoute).toBe("incident.listIncidents");
+  });
+
+  // P2 review fix: the read-only-over-MCP guarantee is STRUCTURAL. Even when
+  // the principal IS authorized for a mutating tool, a bare tools/call must be
+  // refused with 403 and the live router must never be re-entered — mutating
+  // tools go through propose/apply, never a direct invocation.
+  test("tools/call for a mutating tool is structurally REFUSED even when authorized", async () => {
+    let invoked = false;
+    const handler = buildHandler({
+      principal: limitedPrincipal,
+      invoke: () => {
+        invoked = true;
+        return Promise.resolve({});
+      },
+    });
+    const res = await handler(
+      mcpPost({
+        jsonrpc: "2.0",
+        id: 7,
+        method: "tools/call",
+        params: { name: "incident.close", arguments: {} },
+      }),
+    );
+    expect(res.status).toBe(403);
+    const json = await res.json();
+    expect(json.error.message).toContain("propose/apply");
+    expect(invoked).toBe(false);
+  });
+
+  test("tools/list excludes mutating tools (only the read-only surface)", async () => {
+    const handler = buildHandler({ principal: limitedPrincipal });
+    const res = await handler(
+      mcpPost({ jsonrpc: "2.0", id: 8, method: "tools/list" }),
+    );
+    const json = await res.json();
+    const names = json.result.tools.map((t: { name: string }) => t.name);
+    expect(names).toContain("incident.list");
+    expect(names).not.toContain("incident.close");
+  });
+
+  // §14.5: per-principal tool budget enforced on tools/call (shared-Postgres).
+  test("tools/call over the per-principal budget returns 429 and never invokes", async () => {
+    let invoked = false;
+    const handler = buildHandler({
+      principal: limitedPrincipal,
+      invoke: () => {
+        invoked = true;
+        return Promise.resolve({});
+      },
+      enforceBudget: () => Promise.reject(new Error("budget exceeded")),
+    });
+    const res = await handler(
+      mcpPost({
+        jsonrpc: "2.0",
+        id: 9,
+        method: "tools/call",
+        params: { name: "incident.list", arguments: {} },
+      }),
+    );
+    expect(res.status).toBe(429);
+    expect(invoked).toBe(false);
+  });
+
+  test("a within-budget tools/call records an audit row (matrix #13)", async () => {
+    const recorded: Array<{ toolName: string; argsHash: string }> = [];
+    const handler = buildHandler({
+      principal: limitedPrincipal,
+      invoke: () => Promise.resolve({ ok: true }),
+      enforceBudget: () => Promise.resolve(),
+      recordExecuted: ({ toolName, argsHash }) => {
+        recorded.push({ toolName, argsHash });
+        return Promise.resolve();
+      },
+    });
+    const res = await handler(
+      mcpPost({
+        jsonrpc: "2.0",
+        id: 10,
+        method: "tools/call",
+        params: { name: "incident.list", arguments: { status: "open" } },
+      }),
+    );
+    expect(res.status).toBe(200);
+    expect(recorded).toHaveLength(1);
+    expect(recorded[0]?.toolName).toBe("incident.list");
+    // The args hash is a SHA-256 hex digest, never the raw args.
+    expect(recorded[0]?.argsHash).toMatch(/^[0-9a-f]{64}$/);
+  });
+
+  test("unknown method returns a JSON-RPC method-not-found error", async () => {
+    const handler = buildHandler({ principal: limitedPrincipal });
+    const res = await handler(
+      mcpPost({ jsonrpc: "2.0", id: 6, method: "bogus/method" }),
+    );
+    const json = await res.json();
+    expect(json.error.code).toBe(-32601);
+  });
+});