mongodb-mcp-server 0.1.3 → 0.2.0

package/tests/integration/tools/mongodb/read/find.test.ts

@@ -1,3 +1,4 @@
+import { beforeEach, describe, expect, it } from "vitest";
 import {
     getResponseContent,
     databaseCollectionParameters,
@@ -33,7 +34,7 @@ describeWithMongoDB("find tool", (integration) => {
         {
             name: "sort",
             description:
-                "A document, describing the sort order, matching the syntax of the sort argument of cursor.sort()",
+                "A document, describing the sort order, matching the syntax of the sort argument of cursor.sort(). The keys of the object are the fields to sort on, while the values are the sort directions (1 for ascending, -1 for descending).",
             type: "object",
             required: false,
         },

package/tests/integration/tools/mongodb/update/renameCollection.test.ts

@@ -4,6 +4,7 @@ import {
     validateToolMetadata,
     validateThrowsForInvalidArguments,
 } from "../../../helpers.js";
+import { describe, expect, it } from "vitest";
 import { describeWithMongoDB, validateAutoConnectBehavior } from "../mongodbHelpers.js";
 
 describeWithMongoDB("renameCollection tool", (integration) => {

package/tests/integration/tools/mongodb/update/updateMany.test.ts

@@ -4,6 +4,7 @@ import {
     validateThrowsForInvalidArguments,
     getResponseContent,
 } from "../../../helpers.js";
+import { beforeEach, describe, expect, it } from "vitest";
 import { describeWithMongoDB, validateAutoConnectBehavior } from "../mongodbHelpers.js";
 
 describeWithMongoDB("updateMany tool", (integration) => {

package/tests/integration/transports/stdio.test.ts

@@ -0,0 +1,40 @@
+import { describe, expect, it, beforeAll, afterAll } from "vitest";
+import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+
+describe("StdioRunner", () => {
+    describe("client connects successfully", () => {
+        let client: Client;
+        let transport: StdioClientTransport;
+        beforeAll(async () => {
+            transport = new StdioClientTransport({
+                command: "node",
+                args: ["dist/index.js"],
+                env: {
+                    MDB_MCP_TRANSPORT: "stdio",
+                },
+            });
+            client = new Client({
+                name: "test",
+                version: "0.0.0",
+            });
+            await client.connect(transport);
+        });
+
+        afterAll(async () => {
+            await client.close();
+            await transport.close();
+        });
+
+        it("handles requests and sends responses", async () => {
+            const response = await client.listTools();
+            expect(response).toBeDefined();
+            expect(response.tools).toBeDefined();
+            expect(response.tools).toHaveLength(20);
+
+            const sortedTools = response.tools.sort((a, b) => a.name.localeCompare(b.name));
+            expect(sortedTools[0]?.name).toBe("aggregate");
+            expect(sortedTools[0]?.description).toBe("Run an aggregation against a MongoDB collection");
+        });
+    });
+});

package/tests/integration/transports/streamableHttp.test.ts

@@ -0,0 +1,56 @@
+import { StreamableHttpRunner } from "../../../src/transports/streamableHttp.js";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
+import { describe, expect, it, beforeAll, afterAll } from "vitest";
+import { config } from "../../../src/common/config.js";
+
+describe("StreamableHttpRunner", () => {
+    let runner: StreamableHttpRunner;
+    let oldTelemetry: "enabled" | "disabled";
+    let oldLoggers: ("stderr" | "disk" | "mcp")[];
+
+    beforeAll(async () => {
+        oldTelemetry = config.telemetry;
+        oldLoggers = config.loggers;
+        config.telemetry = "disabled";
+        config.loggers = ["stderr"];
+        runner = new StreamableHttpRunner(config);
+        await runner.start();
+    });
+
+    afterAll(async () => {
+        await runner.close();
+        config.telemetry = oldTelemetry;
+        config.loggers = oldLoggers;
+    });
+
+    describe("client connects successfully", () => {
+        let client: Client;
+        let transport: StreamableHTTPClientTransport;
+        beforeAll(async () => {
+            transport = new StreamableHTTPClientTransport(new URL("http://127.0.0.1:3000/mcp"));
+
+            client = new Client({
+                name: "test",
+                version: "0.0.0",
+            });
+            await client.connect(transport);
+        });
+
+        afterAll(async () => {
+            await client.close();
+            await transport.close();
+        });
+
+        it("handles requests and sends responses", async () => {
+            const response = await client.listTools();
+            expect(response).toBeDefined();
+            expect(response.tools).toBeDefined();
+            expect(response.tools.length).toBeGreaterThan(0);
+
+            const sortedTools = response.tools.sort((a, b) => a.name.localeCompare(b.name));
+            expect(sortedTools[0]?.name).toBe("aggregate");
+            expect(sortedTools[0]?.description).toBe("Run an aggregation against a MongoDB collection");
+        });
+    });
+});

package/tests/matchers/toIncludeSameMembers.test.ts

@@ -0,0 +1,59 @@
+import { describe, expect, it } from "vitest";
+
+describe("toIncludeSameMembers matcher", () => {
+    it("should pass when arrays contain the same elements in different order", () => {
+        const array1 = [1, 2, 3];
+        const array2 = [3, 1, 2];
+
+        expect(array1).toIncludeSameMembers(array2);
+    });
+
+    it("should pass when arrays contain the same elements in same order", () => {
+        const array1 = [1, 2, 3];
+        const array2 = [1, 2, 3];
+
+        expect(array1).toIncludeSameMembers(array2);
+    });
+
+    it("should fail when arrays have different lengths", () => {
+        const array1 = [1, 2, 3];
+        const array2 = [1, 2];
+
+        expect(() => expect(array1).toIncludeSameMembers(array2)).toThrow();
+    });
+
+    it("should fail when arrays contain different elements", () => {
+        const array1 = [1, 2, 3];
+        const array2 = [4, 5, 6];
+
+        expect(() => expect(array1).toIncludeSameMembers(array2)).toThrow();
+    });
+
+    it("should work with string arrays", () => {
+        const array1 = ["apple", "banana", "cherry"];
+        const array2 = ["cherry", "apple", "banana"];
+
+        expect(array1).toIncludeSameMembers(array2);
+    });
+
+    it("should work with object arrays", () => {
+        const array1 = [{ name: "Alice" }, { name: "Bob" }];
+        const array2 = [{ name: "Bob" }, { name: "Alice" }];
+
+        expect(array1).toIncludeSameMembers(array2);
+    });
+
+    it("should work with mixed type arrays", () => {
+        const array1 = [1, "hello", { key: "value" }];
+        const array2 = [{ key: "value" }, 1, "hello"];
+
+        expect(array1).toIncludeSameMembers(array2);
+    });
+
+    it("should work with empty arrays", () => {
+        const array1: unknown[] = [];
+        const array2: unknown[] = [];
+
+        expect(array1).toIncludeSameMembers(array2);
+    });
+});

package/tests/matchers/toIncludeSameMembers.ts

@@ -0,0 +1,12 @@
+import { expect } from "vitest";
+
+export function toIncludeSameMembers<T>(actual: T[], expected: T[]): { pass: boolean; message: () => string } {
+    expect(actual).toEqual(expect.arrayContaining(expected as unknown[]));
+    expect(expected).toEqual(expect.arrayContaining(actual as unknown[]));
+
+    return {
+        pass: true,
+        message: () =>
+            `Expected arrays to include the same members.\nExpected: ${JSON.stringify(expected)}\nReceived: ${JSON.stringify(actual)}`,
+    };
+}

package/tests/setup.ts

@@ -0,0 +1,7 @@
+import { expect } from "vitest";
+import { toIncludeSameMembers } from "./matchers/toIncludeSameMembers.js";
+
+// Extend vitest's expect with custom matchers
+expect.extend({
+    toIncludeSameMembers,
+});

package/tests/unit/accessListUtils.test.ts

@@ -0,0 +1,39 @@
+import { describe, it, expect, vi } from "vitest";
+import { ApiClient } from "../../src/common/atlas/apiClient.js";
+import { ensureCurrentIpInAccessList, DEFAULT_ACCESS_LIST_COMMENT } from "../../src/common/atlas/accessListUtils.js";
+import { ApiClientError } from "../../src/common/atlas/apiClientError.js";
+
+describe("accessListUtils", () => {
+    it("should add the current IP to the access list", async () => {
+        const apiClient = {
+            getIpInfo: vi.fn().mockResolvedValue({ currentIpv4Address: "127.0.0.1" } as never),
+            createProjectIpAccessList: vi.fn().mockResolvedValue(undefined as never),
+        } as unknown as ApiClient;
+        await ensureCurrentIpInAccessList(apiClient, "projectId");
+        // eslint-disable-next-line @typescript-eslint/unbound-method
+        expect(apiClient.createProjectIpAccessList).toHaveBeenCalledWith({
+            params: { path: { groupId: "projectId" } },
+            body: [{ groupId: "projectId", ipAddress: "127.0.0.1", comment: DEFAULT_ACCESS_LIST_COMMENT }],
+        });
+    });
+
+    it("should not fail if the current IP is already in the access list", async () => {
+        const apiClient = {
+            getIpInfo: vi.fn().mockResolvedValue({ currentIpv4Address: "127.0.0.1" } as never),
+            createProjectIpAccessList: vi
+                .fn()
+                .mockRejectedValue(
+                    ApiClientError.fromError(
+                        { status: 409, statusText: "Conflict" } as Response,
+                        { message: "Conflict" } as never
+                    ) as never
+                ),
+        } as unknown as ApiClient;
+        await ensureCurrentIpInAccessList(apiClient, "projectId");
+        // eslint-disable-next-line @typescript-eslint/unbound-method
+        expect(apiClient.createProjectIpAccessList).toHaveBeenCalledWith({
+            params: { path: { groupId: "projectId" } },
+            body: [{ groupId: "projectId", ipAddress: "127.0.0.1", comment: DEFAULT_ACCESS_LIST_COMMENT }],
+        });
+    });
+});

package/tests/unit/accuracyScorer.test.ts

@@ -0,0 +1,390 @@
+import { describe, expect, it } from "vitest";
+import { calculateToolCallingAccuracy } from "../accuracy/sdk/accuracyScorer.js";
+import { ExpectedToolCall, LLMToolCall } from "../accuracy/sdk/accuracyResultStorage/resultStorage.js";
+import { Matcher } from "../accuracy/sdk/matcher.js";
+
+describe("calculateToolCallingAccuracy", () => {
+    describe("edge cases", () => {
+        it("should return 1 when both expected and actual are empty", () => {
+            const result = calculateToolCallingAccuracy([], []);
+            expect(result).toBe(1);
+        });
+
+        it("should return 0.75 when expected is empty but actual has tool calls", () => {
+            const actualToolCalls: LLMToolCall[] = [{ toolCallId: "1", toolName: "find", parameters: { db: "test" } }];
+            const result = calculateToolCallingAccuracy([], actualToolCalls);
+            expect(result).toBe(0.75);
+        });
+
+        it("should return 0 when expected has tool calls but actual is empty", () => {
+            const expectedToolCalls: ExpectedToolCall[] = [{ toolName: "find", parameters: { db: "test" } }];
+            const result = calculateToolCallingAccuracy(expectedToolCalls, []);
+            expect(result).toBe(0);
+        });
+    });
+
+    describe("perfect matches", () => {
+        it("should return 1 for exact match with nested parameters", () => {
+            const expected: ExpectedToolCall[] = [
+                {
+                    toolName: "find",
+                    parameters: { db: "test", collection: "users", filter: { age: { $gte: 18 }, status: "active" } },
+                },
+            ];
+            const actual: LLMToolCall[] = [
+                {
+                    toolCallId: "1",
+                    toolName: "find",
+                    parameters: { db: "test", collection: "users", filter: { age: { $gte: 18 }, status: "active" } },
+                },
+            ];
+            const result = calculateToolCallingAccuracy(expected, actual);
+            expect(result).toBe(1);
+        });
+
+        it("should return 1 for exact match with multiple diverse tool calls", () => {
+            const expected: ExpectedToolCall[] = [
+                { toolName: "find", parameters: { db: "test", collection: "users", filter: { status: "active" } } },
+                {
+                    toolName: "aggregate",
+                    parameters: { db: "test", collection: "orders", pipeline: [{ $match: { total: { $gt: 100 } } }] },
+                },
+                { toolName: "count", parameters: { db: "test", collection: "products" } },
+            ];
+            const actual: LLMToolCall[] = [
+                {
+                    toolCallId: "1",
+                    toolName: "find",
+                    parameters: { db: "test", collection: "users", filter: { status: "active" } },
+                },
+                {
+                    toolCallId: "2",
+                    toolName: "aggregate",
+                    parameters: { db: "test", collection: "orders", pipeline: [{ $match: { total: { $gt: 100 } } }] },
+                },
+                { toolCallId: "3", toolName: "count", parameters: { db: "test", collection: "products" } },
+            ];
+            const result = calculateToolCallingAccuracy(expected, actual);
+            expect(result).toBe(1);
+        });
+    });
+
+    describe("additional parameters", () => {
+        it("should return 0 when tool call has additional nested parameters (default behavior)", () => {
+            const expected: ExpectedToolCall[] = [
+                { toolName: "find", parameters: { db: "test", collection: "users", filter: { status: "active" } } },
+            ];
+            const actual: LLMToolCall[] = [
+                {
+                    toolCallId: "1",
+                    toolName: "find",
+                    parameters: {
+                        db: "test",
+                        collection: "users",
+                        filter: { status: "active", age: { $gte: 18 } },
+                        limit: 10,
+                    },
+                },
+            ];
+            const result = calculateToolCallingAccuracy(expected, actual);
+            expect(result).toBe(0);
+        });
+
+        it("should return 1 when expected has no filter but actual has empty filter", () => {
+            const expected: ExpectedToolCall[] = [
+                {
+                    toolName: "find",
+                    parameters: {
+                        database: "mflix",
+                        collection: "movies",
+                        filter: Matcher.emptyObjectOrUndefined,
+                    },
+                },
+            ];
+            const actual: LLMToolCall[] = [
+                {
+                    toolCallId: "1",
+                    toolName: "find",
+                    parameters: {
+                        database: "mflix",
+                        collection: "movies",
+                        filter: {},
+                    },
+                },
+            ];
+            const result = calculateToolCallingAccuracy(expected, actual);
+            expect(result).toBe(1);
+        });
+
+        it("should return 1 when expected has no filter and actual has no filter", () => {
+            const expected: ExpectedToolCall[] = [
+                {
+                    toolName: "find",
+                    parameters: {
+                        database: "mflix",
+                        collection: "movies",
+                        filter: Matcher.emptyObjectOrUndefined,
+                    },
+                },
+            ];
+            const actual: LLMToolCall[] = [
+                {
+                    toolCallId: "1",
+                    toolName: "find",
+                    parameters: {
+                        database: "mflix",
+                        collection: "movies",
+                    },
+                },
+            ];
+            const result = calculateToolCallingAccuracy(expected, actual);
+            expect(result).toBe(1);
+        });
+
+        it("should return 0 when expected has no filter but actual has non-empty filter", () => {
+            const expected: ExpectedToolCall[] = [
+                {
+                    toolName: "find",
+                    parameters: {
+                        database: "mflix",
+                        collection: "movies",
+                        filter: Matcher.emptyObjectOrUndefined,
+                    },
+                },
+            ];
+            const actual: LLMToolCall[] = [
+                {
+                    toolCallId: "1",
+                    toolName: "find",
+                    parameters: {
+                        database: "mflix",
+                        collection: "movies",
+                        filter: { genre: "Horror" },
+                    },
+                },
+            ];
+            const result = calculateToolCallingAccuracy(expected, actual);
+            expect(result).toBe(0);
+        });
+
+        it("should return 0 when there are additional nested fields", () => {
+            const expected: ExpectedToolCall[] = [
+                {
+                    toolName: "find",
+                    parameters: {
+                        database: "mflix",
+                        collection: "movies",
+                        filter: { runtime: { $lt: 100 } },
+                    },
+                },
+            ];
+            const actual: LLMToolCall[] = [
+                {
+                    toolCallId: "1",
+                    toolName: "find",
+                    parameters: {
+                        database: "mflix",
+                        collection: "movies",
+                        filter: { runtime: { $lt: 100 }, genre: "Horror" },
+                    },
+                },
+            ];
+            const result = calculateToolCallingAccuracy(expected, actual);
+            expect(result).toBe(0);
+        });
+
+        it("should return 1 when ignored additional fields are provided", () => {
+            const expected: ExpectedToolCall[] = [
+                {
+                    toolName: "find",
+                    parameters: {
+                        database: "mflix",
+                        collection: "movies",
+                        filter: { runtime: { $lt: 100 } },
+                        limit: Matcher.number(),
+                        sort: Matcher.anyValue,
+                    },
+                },
+            ];
+            const actual: LLMToolCall[] = [
+                {
+                    toolCallId: "1",
+                    toolName: "find",
+                    parameters: {
+                        database: "mflix",
+                        collection: "movies",
+                        filter: { runtime: { $lt: 100 } },
+                        limit: 10,
+                        sort: { title: 1 },
+                    },
+                },
+            ];
+            const result = calculateToolCallingAccuracy(expected, actual);
+            expect(result).toBe(1);
+        });
+
+        it("should return 1 for array where additional elements are allowed", () => {
+            const expected: ExpectedToolCall[] = [
+                {
+                    toolName: "aggregate",
+                    parameters: {
+                        database: "mflix",
+                        collection: "movies",
+                        pipeline: [{ $match: { genre: "Horror" } }, Matcher.anyOf(Matcher.undefined, Matcher.anyValue)],
+                    },
+                },
+            ];
+
+            const actual: LLMToolCall[] = [
+                {
+                    toolCallId: "1",
+                    toolName: "aggregate",
+                    parameters: {
+                        database: "mflix",
+                        collection: "movies",
+                        pipeline: [{ $match: { genre: "Horror" } }, { $sort: { title: 1 } }],
+                    },
+                },
+            ];
+
+            const result = calculateToolCallingAccuracy(expected, actual);
+            expect(result).toBe(1);
+        });
+
+        it("should return 1 for array where additional elements are allowed but not provided", () => {
+            const expected: ExpectedToolCall[] = [
+                {
+                    toolName: "aggregate",
+                    parameters: {
+                        database: "mflix",
+                        collection: "movies",
+                        pipeline: [{ $match: { genre: "Horror" } }, Matcher.anyOf(Matcher.undefined, Matcher.anyValue)],
+                    },
+                },
+            ];
+
+            const actual: LLMToolCall[] = [
+                {
+                    toolCallId: "1",
+                    toolName: "aggregate",
+                    parameters: {
+                        database: "mflix",
+                        collection: "movies",
+                        pipeline: [{ $match: { genre: "Horror" } }],
+                    },
+                },
+            ];
+
+            const result = calculateToolCallingAccuracy(expected, actual);
+            expect(result).toBe(1);
+        });
+    });
+
+    describe("missing or incorrect parameters", () => {
+        it("should return 0 when tool call has missing nested parameters", () => {
+            const expected: ExpectedToolCall[] = [
+                {
+                    toolName: "find",
+                    parameters: { db: "test", collection: "users", filter: { status: "active", age: { $gte: 18 } } },
+                },
+            ];
+            const actual: LLMToolCall[] = [
+                {
+                    toolCallId: "1",
+                    toolName: "find",
+                    parameters: { db: "test", collection: "users", filter: { status: "active" } },
+                },
+            ];
+            const result = calculateToolCallingAccuracy(expected, actual);
+            expect(result).toBe(0);
+        });
+
+        it("should return 0 when aggregate tool call has incorrect pipeline", () => {
+            const expected: ExpectedToolCall[] = [
+                {
+                    toolName: "aggregate",
+                    parameters: { db: "test", collection: "orders", pipeline: [{ $match: { total: { $gt: 100 } } }] },
+                },
+            ];
+            const actual: LLMToolCall[] = [
+                {
+                    toolCallId: "1",
+                    toolName: "aggregate",
+                    parameters: { db: "test", collection: "orders", pipeline: [{ $match: { total: { $lt: 50 } } }] },
+                },
+            ];
+            const result = calculateToolCallingAccuracy(expected, actual);
+            expect(result).toBe(0);
+        });
+    });
+
+    describe("additional tool calls", () => {
+        it("should cap accuracy at 0.75 when LLM calls extra tools", () => {
+            const expected: ExpectedToolCall[] = [
+                { toolName: "find", parameters: { db: "test", collection: "users", filter: { status: "active" } } },
+            ];
+            const actual: LLMToolCall[] = [
+                {
+                    toolCallId: "1",
+                    toolName: "find",
+                    parameters: { db: "test", collection: "users", filter: { status: "active" } },
+                },
+                { toolCallId: "2", toolName: "count", parameters: { db: "test", collection: "orders" } },
+                {
+                    toolCallId: "3",
+                    toolName: "aggregate",
+                    parameters: {
+                        db: "test",
+                        collection: "products",
+                        pipeline: [{ $group: { _id: "$category", total: { $sum: 1 } } }],
+                    },
+                },
+            ];
+            const result = calculateToolCallingAccuracy(expected, actual);
+            expect(result).toBe(0.75);
+        });
+
+        it("should cap accuracy at 0.75 when LLM calls same tool multiple times with variations", () => {
+            const expected: ExpectedToolCall[] = [
+                { toolName: "find", parameters: { db: "test", collection: "users", filter: { status: "active" } } },
+            ];
+            const actual: LLMToolCall[] = [
+                {
+                    toolCallId: "1",
+                    toolName: "find",
+                    parameters: { db: "test", collection: "users", filter: { status: "active" } },
+                },
+                {
+                    toolCallId: "2",
+                    toolName: "find",
+                    parameters: { db: "test", collection: "users", filter: { status: "active", age: { $gte: 18 } } },
+                },
+                { toolCallId: "3", toolName: "find", parameters: { db: "test", collection: "users", limit: 10 } },
+            ];
+            const result = calculateToolCallingAccuracy(expected, actual);
+            expect(result).toBe(0.75);
+        });
+    });
+
+    describe("missing tool calls", () => {
+        it("should return 0 if any expected tool call was not called", () => {
+            const expected: ExpectedToolCall[] = [
+                { toolName: "find", parameters: { db: "test", collection: "users", filter: { status: "active" } } },
+                {
+                    toolName: "aggregate",
+                    parameters: { db: "test", collection: "orders", pipeline: [{ $match: { total: { $gt: 100 } } }] },
+                },
+            ];
+            const actual: LLMToolCall[] = [
+                {
+                    toolCallId: "1",
+                    toolName: "find",
+                    parameters: { db: "test", collection: "users", filter: { status: "active" } },
+                },
+                // Missing the aggregate tool call
+            ];
+            const result = calculateToolCallingAccuracy(expected, actual);
+            expect(result).toBe(0); // One expected tool call was not called
+        });
+    });
+});