mongodb-mcp-server 0.1.2 → 0.2.0

package/tests/accuracy/explain.test.ts

@@ -0,0 +1,73 @@
+import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js";
+
+/**
+ * None of these tests score a parameter match on any of the models, likely
+ * because we are using Zod.union, when we probably should've used
+ * Zod.discriminatedUnion
+ */
+describeAccuracyTests([
+    {
+        prompt: `Will fetching documents, where release_year is 2020, from 'mflix.movies' namespace perform a collection scan?`,
+        expectedToolCalls: [
+            {
+                toolName: "explain",
+                parameters: {
+                    database: "mflix",
+                    collection: "movies",
+                    method: [
+                        {
+                            name: "find",
+                            arguments: {
+                                filter: { release_year: 2020 },
+                            },
+                        },
+                    ],
+                },
+            },
+        ],
+    },
+    {
+        prompt: `Will aggregating documents, where release_year is 2020, from 'mflix.movies' namespace perform a collection scan?`,
+        expectedToolCalls: [
+            {
+                toolName: "explain",
+                parameters: {
+                    database: "mflix",
+                    collection: "movies",
+                    method: [
+                        {
+                            name: "aggregate",
+                            arguments: {
+                                pipeline: [
+                                    {
+                                        $match: { release_year: 2020 },
+                                    },
+                                ],
+                            },
+                        },
+                    ],
+                },
+            },
+        ],
+    },
+    {
+        prompt: `Will counting documents, where release_year is 2020, from 'mflix.movies' namespace perform a collection scan?`,
+        expectedToolCalls: [
+            {
+                toolName: "explain",
+                parameters: {
+                    database: "mflix",
+                    collection: "movies",
+                    method: [
+                        {
+                            name: "count",
+                            arguments: {
+                                query: { release_year: 2020 },
+                            },
+                        },
+                    ],
+                },
+            },
+        ],
+    },
+]);

package/tests/accuracy/find.test.ts

@@ -0,0 +1,114 @@
+import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js";
+import { Matcher } from "./sdk/matcher.js";
+
+describeAccuracyTests([
+    {
+        prompt: "List all the movies in 'mflix.movies' namespace.",
+        expectedToolCalls: [
+            {
+                toolName: "find",
+                parameters: {
+                    database: "mflix",
+                    collection: "movies",
+                    filter: Matcher.emptyObjectOrUndefined,
+                },
+            },
+        ],
+    },
+    {
+        prompt: "List all the documents in 'comics.books' namespace.",
+        expectedToolCalls: [
+            {
+                toolName: "find",
+                parameters: {
+                    database: "comics",
+                    collection: "books",
+                    filter: Matcher.emptyObjectOrUndefined,
+                },
+            },
+        ],
+    },
+    {
+        prompt: "Find all the movies in 'mflix.movies' namespace with runtime less than 100.",
+        expectedToolCalls: [
+            {
+                toolName: "find",
+                parameters: {
+                    database: "mflix",
+                    collection: "movies",
+                    filter: {
+                        runtime: { $lt: 100 },
+                    },
+                },
+            },
+        ],
+    },
+    {
+        prompt: "Find all movies in 'mflix.movies' collection where director is 'Christina Collins'",
+        expectedToolCalls: [
+            {
+                toolName: "find",
+                parameters: {
+                    database: "mflix",
+                    collection: "movies",
+                    filter: {
+                        director: "Christina Collins",
+                    },
+                },
+            },
+        ],
+    },
+    {
+        prompt: "Give me all the movie titles available in 'mflix.movies' namespace",
+        expectedToolCalls: [
+            {
+                toolName: "find",
+                parameters: {
+                    database: "mflix",
+                    collection: "movies",
+                    projection: {
+                        title: 1,
+                        _id: Matcher.anyOf(
+                            Matcher.undefined,
+                            Matcher.number((value) => value === 0)
+                        ),
+                    },
+                    filter: Matcher.emptyObjectOrUndefined,
+                },
+            },
+        ],
+    },
+    {
+        prompt: "Use 'mflix.movies' namespace to answer who were casted in the movie 'Certain Fish'",
+        expectedToolCalls: [
+            {
+                toolName: "find",
+                parameters: {
+                    database: "mflix",
+                    collection: "movies",
+                    filter: { title: "Certain Fish" },
+                    projection: {
+                        cast: 1,
+                        _id: Matcher.anyOf(Matcher.undefined, Matcher.number()),
+                    },
+                    limit: Matcher.number((value) => value > 0),
+                },
+            },
+        ],
+    },
+    {
+        prompt: "From the mflix.movies namespace, give me first 2 movies of Horror genre sorted ascending by their runtime",
+        expectedToolCalls: [
+            {
+                toolName: "find",
+                parameters: {
+                    database: "mflix",
+                    collection: "movies",
+                    filter: { genres: "Horror" },
+                    sort: { runtime: 1 },
+                    limit: 2,
+                },
+            },
+        ],
+    },
+]);

package/tests/accuracy/insertMany.test.ts

@@ -0,0 +1,48 @@
+import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js";
+import { Matcher } from "./sdk/matcher.js";
+
+describeAccuracyTests([
+    {
+        prompt: [
+            "In my namespace 'mflix.movies', insert 3 documents each with the following fields:",
+            "- id: an incremental number starting from 1",
+            "- name: a string of format 'name<id>'",
+        ].join("\n"),
+        expectedToolCalls: [
+            {
+                toolName: "insert-many",
+                parameters: {
+                    database: "mflix",
+                    collection: "movies",
+                    documents: [
+                        {
+                            id: 1,
+                            name: "name1",
+                        },
+                        {
+                            id: 2,
+                            name: "name2",
+                        },
+                        {
+                            id: 3,
+                            name: "name3",
+                        },
+                    ],
+                },
+            },
+        ],
+    },
+    {
+        prompt: "Add three empty documents in collection 'movies' inside database 'mflix'",
+        expectedToolCalls: [
+            {
+                toolName: "insert-many",
+                parameters: {
+                    database: "mflix",
+                    collection: "movies",
+                    documents: [{ _id: Matcher.anyValue }, { _id: Matcher.anyValue }, { _id: Matcher.anyValue }],
+                },
+            },
+        ],
+    },
+]);

package/tests/accuracy/listCollections.test.ts

@@ -0,0 +1,60 @@
+import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js";
+
+describeAccuracyTests([
+    {
+        prompt: "How many collections do I have in database mflix?",
+        expectedToolCalls: [
+            {
+                toolName: "list-collections",
+                parameters: { database: "mflix" },
+            },
+        ],
+    },
+    {
+        prompt: "List all the collections in my MongoDB database mflix.",
+        expectedToolCalls: [
+            {
+                toolName: "list-collections",
+                parameters: { database: "mflix" },
+            },
+        ],
+    },
+    {
+        prompt: "Is there a shows collection in my MongoDB database mflix?",
+        expectedToolCalls: [
+            {
+                toolName: "list-collections",
+                parameters: { database: "mflix" },
+            },
+        ],
+    },
+    {
+        prompt: "List all the collections that I have in total on my cluster?",
+        expectedToolCalls: [
+            {
+                toolName: "list-databases",
+                parameters: {},
+            },
+            {
+                toolName: "list-collections",
+                parameters: { database: "admin" },
+            },
+            {
+                toolName: "list-collections",
+                parameters: { database: "comics" },
+            },
+            {
+                toolName: "list-collections",
+                parameters: { database: "config" },
+            },
+            {
+                toolName: "list-collections",
+                parameters: { database: "local" },
+            },
+            {
+                toolName: "list-collections",
+                parameters: { database: "mflix" },
+            },
+        ],
+    },
+]);

package/tests/accuracy/listDatabases.test.ts

@@ -0,0 +1,31 @@
+import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js";
+
+describeAccuracyTests([
+    {
+        prompt: "How many databases do I have?",
+        expectedToolCalls: [
+            {
+                toolName: "list-databases",
+                parameters: {},
+            },
+        ],
+    },
+    {
+        prompt: "List all the databases that I have in my clusters",
+        expectedToolCalls: [
+            {
+                toolName: "list-databases",
+                parameters: {},
+            },
+        ],
+    },
+    {
+        prompt: "Is there a mflix database in my cluster?",
+        expectedToolCalls: [
+            {
+                toolName: "list-databases",
+                parameters: {},
+            },
+        ],
+    },
+]);

package/tests/accuracy/logs.test.ts

@@ -0,0 +1,28 @@
+import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js";
+import { Matcher } from "./sdk/matcher.js";
+
+describeAccuracyTests([
+    {
+        prompt: "Were there any startup warnings for my MongoDB server?",
+        expectedToolCalls: [
+            {
+                toolName: "mongodb-logs",
+                parameters: {
+                    type: "startupWarnings",
+                },
+            },
+        ],
+    },
+    {
+        prompt: "Retrieve first 10 logs for my MongoDB server?",
+        expectedToolCalls: [
+            {
+                toolName: "mongodb-logs",
+                parameters: {
+                    type: Matcher.anyOf(Matcher.undefined, Matcher.value("global")),
+                    limit: 10,
+                },
+            },
+        ],
+    },
+]);

package/tests/accuracy/renameCollection.test.ts

@@ -0,0 +1,31 @@
+import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js";
+
+describeAccuracyTests([
+    {
+        prompt: "Rename my 'mflix.movies' namespace to 'mflix.new_movies'",
+        expectedToolCalls: [
+            {
+                toolName: "rename-collection",
+                parameters: {
+                    database: "mflix",
+                    collection: "movies",
+                    newName: "new_movies",
+                },
+            },
+        ],
+    },
+    {
+        prompt: "Rename my 'mflix.movies' namespace to 'mflix.new_movies' while removing the old namespace.",
+        expectedToolCalls: [
+            {
+                toolName: "rename-collection",
+                parameters: {
+                    database: "mflix",
+                    collection: "movies",
+                    newName: "new_movies",
+                    dropTarget: true,
+                },
+            },
+        ],
+    },
+]);

package/tests/accuracy/sdk/accuracyResultStorage/diskStorage.ts

@@ -0,0 +1,189 @@
+import path from "path";
+import fs from "fs/promises";
+import { lock } from "proper-lockfile";
+import { ACCURACY_RESULTS_DIR, LATEST_ACCURACY_RUN_NAME } from "../constants.js";
+import {
+    AccuracyResult,
+    AccuracyResultStorage,
+    AccuracyRunStatus,
+    AccuracyRunStatuses,
+    ExpectedToolCall,
+    ModelResponse,
+} from "./resultStorage.js";
+
+export class DiskBasedResultStorage implements AccuracyResultStorage {
+    async getAccuracyResult(commitSHA: string, runId?: string): Promise<AccuracyResult | null> {
+        const filePath = runId
+            ? // If we have both commit and runId then we get the path for
+              // specific file. Common case when saving prompt responses during an
+              // accuracy run
+              this.getAccuracyResultFilePath(commitSHA, runId)
+            : // If we only have commit then we grab the latest successful run for the
+              // commit. The latest run is a link to the last run that was
+              // marked as successful.
+              this.getAccuracyResultFilePath(commitSHA, LATEST_ACCURACY_RUN_NAME);
+
+        return this.withFileLock<AccuracyResult | null>(filePath, () => this.getAccuracyResultWithoutLock(filePath));
+    }
+
+    async updateRunStatus(commitSHA: string, runId: string, status: AccuracyRunStatuses): Promise<void> {
+        const resultFilePath = this.getAccuracyResultFilePath(commitSHA, runId);
+        await this.withFileLock(resultFilePath, async () => {
+            const accuracyResult = await this.getAccuracyResultWithoutLock(resultFilePath);
+            if (!accuracyResult) {
+                throw new Error("Results not found!");
+            }
+
+            await fs.writeFile(
+                resultFilePath,
+                JSON.stringify(
+                    {
+                        ...accuracyResult,
+                        runStatus: status,
+                    },
+                    null,
+                    2
+                ),
+                { encoding: "utf8" }
+            );
+        });
+
+        // This bit is important to mark the current run as the latest run for a
+        // commit so that we can use that during baseline comparison.
+        if (status === AccuracyRunStatus.Done) {
+            const latestResultFilePath = this.getLatestResultFilePath(commitSHA);
+            await this.ensureFileWithInitialData(latestResultFilePath, JSON.stringify({}));
+            await this.withFileLock(latestResultFilePath, async () => {
+                await fs.unlink(latestResultFilePath);
+                await fs.link(resultFilePath, latestResultFilePath);
+            });
+        }
+    }
+
+    async saveModelResponseForPrompt({
+        commitSHA,
+        runId,
+        prompt,
+        expectedToolCalls,
+        modelResponse,
+    }: {
+        commitSHA: string;
+        runId: string;
+        prompt: string;
+        expectedToolCalls: ExpectedToolCall[];
+        modelResponse: ModelResponse;
+    }): Promise<void> {
+        const initialData: AccuracyResult = {
+            runId,
+            runStatus: AccuracyRunStatus.InProgress,
+            createdOn: Date.now(),
+            commitSHA,
+            promptResults: [
+                {
+                    prompt,
+                    expectedToolCalls,
+                    modelResponses: [modelResponse],
+                },
+            ],
+        };
+        const resultFilePath = this.getAccuracyResultFilePath(commitSHA, runId);
+        const { fileCreatedWithInitialData } = await this.ensureFileWithInitialData(
+            resultFilePath,
+            JSON.stringify(initialData, null, 2)
+        );
+
+        if (fileCreatedWithInitialData) {
+            return;
+        }
+
+        await this.withFileLock(resultFilePath, async () => {
+            let accuracyResult = await this.getAccuracyResultWithoutLock(resultFilePath);
+            if (!accuracyResult) {
+                throw new Error("Expected at-least initial accuracy result to be present");
+            }
+
+            const existingPromptIdx = accuracyResult.promptResults.findIndex((result) => result.prompt === prompt);
+            const promptResult = accuracyResult.promptResults[existingPromptIdx];
+            if (promptResult) {
+                accuracyResult.promptResults.splice(existingPromptIdx, 1, {
+                    prompt: promptResult.prompt,
+                    expectedToolCalls: promptResult.expectedToolCalls,
+                    modelResponses: [...promptResult.modelResponses, modelResponse],
+                });
+            } else {
+                accuracyResult = {
+                    ...accuracyResult,
+                    promptResults: [
+                        ...accuracyResult.promptResults,
+                        {
+                            prompt,
+                            expectedToolCalls,
+                            modelResponses: [modelResponse],
+                        },
+                    ],
+                };
+            }
+
+            await fs.writeFile(resultFilePath, JSON.stringify(accuracyResult, null, 2));
+        });
+    }
+
+    close(): Promise<void> {
+        return Promise.resolve();
+    }
+
+    private async getAccuracyResultWithoutLock(filePath: string): Promise<AccuracyResult | null> {
+        try {
+            const raw = await fs.readFile(filePath, "utf8");
+            return JSON.parse(raw) as AccuracyResult;
+        } catch (error) {
+            if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+                return null;
+            }
+            throw error;
+        }
+    }
+
+    private async ensureFileWithInitialData(
+        filePath: string,
+        initialData: string
+    ): Promise<{
+        fileCreatedWithInitialData: boolean;
+    }> {
+        try {
+            await fs.mkdir(path.dirname(filePath), { recursive: true });
+            await fs.writeFile(filePath, initialData, { flag: "wx" });
+            return {
+                fileCreatedWithInitialData: true,
+            };
+        } catch (error) {
+            if ((error as NodeJS.ErrnoException).code === "EEXIST") {
+                return {
+                    fileCreatedWithInitialData: false,
+                };
+            }
+            throw error;
+        }
+    }
+
+    private async withFileLock<R>(filePath: string, callback: () => Promise<R>): Promise<R> {
+        let releaseLock: (() => Promise<void>) | undefined;
+        try {
+            releaseLock = await lock(filePath, { retries: 10 });
+            return await callback();
+        } catch (error) {
+            console.warn(`Could not acquire lock for file - ${filePath}.`, error);
+            throw error;
+        } finally {
+            await releaseLock?.();
+        }
+    }
+
+    private getAccuracyResultFilePath(commitSHA: string, runId: string): string {
+        return path.join(ACCURACY_RESULTS_DIR, commitSHA, `${runId}.json`);
+    }
+
+    private getLatestResultFilePath(commitSHA: string): string {
+        return path.join(ACCURACY_RESULTS_DIR, commitSHA, `${LATEST_ACCURACY_RUN_NAME}.json`);
+    }
+}

package/tests/accuracy/sdk/accuracyResultStorage/getAccuracyResultStorage.ts

@@ -0,0 +1,11 @@
+import { DiskBasedResultStorage } from "./diskStorage.js";
+import { MongoDBBasedResultStorage } from "./mongodbStorage.js";
+import { AccuracyResultStorage } from "./resultStorage.js";
+
+export function getAccuracyResultStorage(): AccuracyResultStorage {
+    const { MDB_ACCURACY_MDB_URL, MDB_ACCURACY_MDB_DB, MDB_ACCURACY_MDB_COLLECTION } = process.env;
+    if (MDB_ACCURACY_MDB_URL && MDB_ACCURACY_MDB_DB && MDB_ACCURACY_MDB_COLLECTION) {
+        return new MongoDBBasedResultStorage(MDB_ACCURACY_MDB_URL, MDB_ACCURACY_MDB_DB, MDB_ACCURACY_MDB_COLLECTION);
+    }
+    return new DiskBasedResultStorage();
+}