mongodb-mcp-server 0.1.3 → 0.2.0

package/tests/accuracy/sdk/accuracyResultStorage/mongodbStorage.ts

@@ -0,0 +1,151 @@
+import { Collection, MongoClient } from "mongodb";
+import {
+    AccuracyResult,
+    AccuracyResultStorage,
+    AccuracyRunStatus,
+    AccuracyRunStatuses,
+    ExpectedToolCall,
+    ModelResponse,
+} from "./resultStorage.js";
+
+// We could decide to omit some fields from the model response to reduce the size of the stored results. Since
+// so far, the responses are not too big, we do not omit any fields, but if we decide to do so in the future,
+// we could add `"messages"` and `"text"` to this list.
+const OMITTED_MODEL_RESPONSE_FIELDS: (keyof ModelResponse)[] = [];
+
+export class MongoDBBasedResultStorage implements AccuracyResultStorage {
+    private client: MongoClient;
+    private resultCollection: Collection<AccuracyResult>;
+
+    constructor(connectionString: string, database: string, collection: string) {
+        this.client = new MongoClient(connectionString);
+        this.resultCollection = this.client.db(database).collection<AccuracyResult>(collection);
+    }
+
+    async getAccuracyResult(commitSHA: string, runId?: string): Promise<AccuracyResult | null> {
+        const filters: Partial<AccuracyResult> = runId
+            ? { commitSHA, runId }
+            : // Note that we use the `Done` status filter only when asked for
+              // a commit. That is because the one use case of asking for a run
+              // for commit is when you want the last successful run of that
+              // particular commit.
+              { commitSHA, runStatus: AccuracyRunStatus.Done };
+
+        return await this.resultCollection.findOne(filters, {
+            sort: {
+                createdOn: -1,
+            },
+        });
+    }
+
+    async updateRunStatus(commitSHA: string, runId: string, status: AccuracyRunStatuses): Promise<void> {
+        await this.resultCollection.updateOne(
+            { commitSHA, runId },
+            {
+                $set: {
+                    runStatus: status,
+                },
+            }
+        );
+    }
+
+    async saveModelResponseForPrompt({
+        commitSHA,
+        runId,
+        prompt,
+        expectedToolCalls,
+        modelResponse,
+    }: {
+        commitSHA: string;
+        runId: string;
+        prompt: string;
+        expectedToolCalls: ExpectedToolCall[];
+        modelResponse: ModelResponse;
+    }): Promise<void> {
+        const modelResponseToSave: ModelResponse = {
+            ...modelResponse,
+        };
+
+        for (const field of OMITTED_MODEL_RESPONSE_FIELDS) {
+            delete modelResponseToSave[field];
+        }
+
+        await this.resultCollection.updateOne(
+            { commitSHA, runId },
+            [
+                {
+                    $set: {
+                        runStatus: { $ifNull: ["$runStatus", AccuracyRunStatus.InProgress] },
+                        createdOn: { $ifNull: ["$createdOn", Date.now()] },
+                        commitSHA: { $ifNull: ["$commitSHA", commitSHA] },
+                        runId: { $ifNull: ["$runId", runId] },
+                        promptResults: {
+                            $ifNull: ["$promptResults", []],
+                        },
+                    },
+                },
+                {
+                    $set: {
+                        promptResults: {
+                            $let: {
+                                vars: {
+                                    existingPromptIndex: {
+                                        $indexOfArray: ["$promptResults.prompt", prompt],
+                                    },
+                                },
+                                in: {
+                                    $cond: [
+                                        { $eq: ["$$existingPromptIndex", -1] },
+                                        {
+                                            $concatArrays: [
+                                                "$promptResults",
+                                                [
+                                                    {
+                                                        $literal: {
+                                                            prompt,
+                                                            expectedToolCalls,
+                                                            modelResponses: [modelResponseToSave],
+                                                        },
+                                                    },
+                                                ],
+                                            ],
+                                        },
+                                        {
+                                            $map: {
+                                                input: "$promptResults",
+                                                as: "promptResult",
+                                                in: {
+                                                    $cond: [
+                                                        { $eq: ["$$promptResult.prompt", prompt] },
+                                                        {
+                                                            prompt: "$$promptResult.prompt",
+                                                            expectedToolCalls: {
+                                                                $literal: expectedToolCalls,
+                                                            },
+                                                            modelResponses: {
+                                                                $concatArrays: [
+                                                                    "$$promptResult.modelResponses",
+                                                                    [{ $literal: modelResponseToSave }],
+                                                                ],
+                                                            },
+                                                        },
+                                                        "$$promptResult",
+                                                    ],
+                                                },
+                                            },
+                                        },
+                                    ],
+                                },
+                            },
+                        },
+                    },
+                },
+            ],
+            { upsert: true }
+        );
+    }
+
+    async close(): Promise<void> {
+        await this.client.close();
+    }
+}

package/tests/accuracy/sdk/accuracyResultStorage/resultStorage.ts

@@ -0,0 +1,117 @@
+export interface LLMToolCall {
+    toolCallId: string;
+    toolName: string;
+    parameters: Record<string, unknown>;
+}
+
+export type ExpectedToolCall = Omit<LLMToolCall, "toolCallId">;
+
+export const AccuracyRunStatus = {
+    Done: "done",
+    Failed: "failed",
+    InProgress: "in-progress",
+} as const;
+
+export type AccuracyRunStatuses = (typeof AccuracyRunStatus)[keyof typeof AccuracyRunStatus];
+
+export interface AccuracyResult {
+    /**
+     * A unique id for each accuracy run. Should either be generated by the
+     * script triggering the accuracy run or provided via environment variables.
+     * */
+    runId: string;
+    /**
+     * Represents the status of accuracy run. Each test completion, during an
+     * accuracy run, is supposed to submit an accuracy result entry with
+     * InProgress status which then later, after completion of accuracy run, is
+     * updated to either Done or Failed, depending on whether there were errors
+     * during the run or not. */
+    runStatus: AccuracyRunStatuses;
+    /**
+     * Timestamp of when this result entry was generated. */
+    createdOn: number;
+    /**
+     * The commit SHA for which the accuracy run was triggered. */
+    commitSHA: string;
+    /**
+     * A list of results for different prompts tested in the accuracy run. */
+    promptResults: PromptResult[];
+}
+
+export interface PromptResult {
+    /**
+     * The actual prompt that was provided to LLM as test */
+    prompt: string;
+    /**
+     * A list of tools, along with their parameters, that are expected to be
+     * called by the LLM in test. */
+    expectedToolCalls: ExpectedToolCall[];
+    /**
+     * The responses from the LLMs tested, when provided with the prompt. */
+    modelResponses: ModelResponse[];
+}
+
+export interface ModelResponse {
+    /**
+     * The LLM provider providing the LLM APIs */
+    provider: string;
+    /**
+     * The LLM which was requested to respond to our test prompts */
+    requestedModel: string;
+    /**
+     * The ID of the model that actually responded to our prompt request. */
+    respondingModel: string;
+    /**
+     * The total time taken by LLM to respond to our prompt. */
+    llmResponseTime: number;
+    /**
+     * A number between 0 and 1, representing how accurately the expected tools
+     * were called by LLM when responding to the provided prompts. To know more
+     * about how this number is generated, check - toolCallingAccuracy.ts */
+    toolCallingAccuracy: number;
+    /**
+     * A list of tools, along with their parameters, that were actually called
+     * by the LLM in test. */
+    llmToolCalls: LLMToolCall[];
+    /**
+     * Token usage data, returned as part of LLM prompt response. */
+    tokensUsed?: TokensUsed;
+    /**
+     * The final response text generated by the LLM, in response to our prompt
+     * request. */
+    text?: string;
+    /**
+     * A list of messages, exchanged between LLM and our testing agent, in
+     * response to our prompt request. This is particularly helpful for
+     * debugging. */
+    messages?: Record<string, unknown>[];
+}
+
+interface TokensUsed {
+    promptTokens?: number;
+    completionTokens?: number;
+    totalTokens?: number;
+}
+
+export interface AccuracyResultStorage {
+    /**
+     * Retrieves the accuracy result for the provided commit SHA and optionally
+     * the run id. When the run id is omitted, the implementation fetches the
+     * result for the last successful accuracy run otherwise it fetches the
+     * result regardless of the run status. */
+    getAccuracyResult(commitSHA: string, runId?: string): Promise<AccuracyResult | null>;
+    /**
+     * Updates the status of the run */
+    updateRunStatus(commitSHA: string, runId: string, status: AccuracyRunStatuses): Promise<void>;
+    /**
+     * Attempts to atomically insert the model response for the prompt in the
+     * stored accuracy result. */
+    saveModelResponseForPrompt(data: {
+        commitSHA: string;
+        runId: string;
+        prompt: string;
+        expectedToolCalls: ExpectedToolCall[];
+        modelResponse: ModelResponse;
+    }): Promise<void>;
+    close(): Promise<void>;
+}

package/tests/accuracy/sdk/accuracyScorer.ts

@@ -0,0 +1,93 @@
+import { ExpectedToolCall, LLMToolCall } from "./accuracyResultStorage/resultStorage.js";
+import { Matcher } from "./matcher.js";
+
+/**
+ * Tool calling accuracy is a single number calculated based on two dimensions.
+ * 1. Did LLM call the right tool?
+ * 2. Did LLM call the tool with correct and required parameters?
+ *
+ * The number can be one of:
+ * - 0: When LLM:
+ *    - did not call the right tool
+ *    - did not call the tool with correct parameters
+ * - 0.75: When LLM:
+ *    - called the right tool but hallucinated and called some extra tools as
+ *      well or called the same tool but with different parameters
+ *    - called the right tool but hallucinated and called it with some
+ *      non-required parameters
+ * - 1: When LLM:
+ *    - called exactly the tools that were expected
+ *    - called the expected tools exactly with the expected parameters
+ *
+ * To calculate this number we must have:
+ * 1. a list of expected tool calls with their expected parameters
+ * 2. a list of LLM tool calls with their parameters
+ *
+ * For each expected tool call we find the best matching LLM tool call. Best
+ * matching LLM tool call will have:
+ * 1. the same name as that of the expected tool call
+ * 2. highest parameter similarity score, with at-least 0.75 to ensure an actual
+ *    match. And in case of competing scores, we take the first one that appears
+ *    in the LLM tool calls.
+ *
+ * Using the above logic we establish pairs between expected and actual tool
+ * calls.
+ *
+ * 1. If we could not pair some LLM tool calls with expected tool calls that
+ *    means the LLM hallucinated over the extra tool calls. For that reason we
+ *    will cap the maximum achievable accuracy to 0.75.
+ *
+ * 2. If we could not pair some expected tool calls with LLM tool calls that
+ *    means the LLM did not call one of the expected tool required to solve the
+ *    problem. For that reason we will mark the accuracy as 0 and exit early.
+ *
+ * 3. Now for each of the established tool call pairs, we will determine how
+ *    correctly the parameters were called using the parameter similarity score.
+ *    The parameter similarity score follow the same accuracy number pattern
+ *    described above:
+ *      - 0 : for missing parameters, incorrect parameter values
+ *      - 0.75 : for additional parameters
+ *      - 1 : for a perfect match
+ *
+ * The final accuracy score is then calculated as the least of:
+ * - Maximum achievable accuracy from #1
+ * - The least of parameter similarity score from the established pairs in #3
+ *
+ * For examples: see the test cases in - tests/unit/accuracy-scorer.test.ts
+ */
+export function calculateToolCallingAccuracy(
+    expectedToolCalls: ExpectedToolCall[],
+    actualToolCalls: LLMToolCall[]
+): number {
+    if (expectedToolCalls.length === 0) {
+        return actualToolCalls.length === 0 ? 1 : 0.75;
+    }
+
+    let currentScore = actualToolCalls.length > expectedToolCalls.length ? 0.75 : 1;
+    const checkedActualToolCallIndexes = new Set<number>();
+
+    for (const expectedCall of expectedToolCalls) {
+        const candidates = actualToolCalls
+            .map((call, index) => ({ call, index }))
+            .filter(
+                ({ call, index }) => !checkedActualToolCallIndexes.has(index) && call.toolName === expectedCall.toolName
+            )
+            .map(({ call, index }) => ({
+                call,
+                index,
+                score: Matcher.value(expectedCall.parameters).match(call.parameters),
+            }))
+            .filter(({ score }) => score >= 0.75)
+            .sort((a, b) => b.score - a.score || a.index - b.index);
+
+        const bestMatch = candidates[0];
+        if (!bestMatch || bestMatch.score === 0) {
+            return 0; // No matching tool call found, return 0
+        }
+
+        checkedActualToolCallIndexes.add(bestMatch.index);
+        currentScore = Math.min(currentScore, bestMatch.score);
+    }
+
+    return currentScore;
+}

package/tests/accuracy/sdk/accuracyTestingClient.ts

@@ -0,0 +1,94 @@
+import { v4 as uuid } from "uuid";
+import { experimental_createMCPClient as createMCPClient, tool as createVercelTool } from "ai";
+import { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
+import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
+
+import { MCP_SERVER_CLI_SCRIPT } from "./constants.js";
+import { LLMToolCall } from "./accuracyResultStorage/resultStorage.js";
+import { VercelMCPClient, VercelMCPClientTools } from "./agent.js";
+
+type ToolResultGeneratorFn = (...parameters: unknown[]) => CallToolResult | Promise<CallToolResult>;
+export type MockedTools = Record<string, ToolResultGeneratorFn>;
+
+/**
+ * AccuracyTestingClient is a bridge between actual MCP client connected to our
+ * MCP server and our Tool calling agent. Its serves the following purposes:
+ * 1. Captures actual tools provided by our MCP server
+ * 2. Translates captured MCP tools to tool definitions that can be consumed by
+ *    Tool Calling agent (Ref: `vercelTools`)
+ * 3. Allow dynamic mocking and resetting of mocks of individual tool calls.
+ * 4. Records and provides tool calls made by LLMs with their parameters.
+ */
+export class AccuracyTestingClient {
+    private mockedTools: MockedTools = {};
+    private llmToolCalls: LLMToolCall[] = [];
+
+    private constructor(private readonly vercelMCPClient: VercelMCPClient) {}
+
+    async close(): Promise<void> {
+        await this.vercelMCPClient?.close();
+    }
+
+    async vercelTools(): Promise<VercelMCPClientTools> {
+        const vercelTools = (await this.vercelMCPClient?.tools()) ?? {};
+        const rewrappedVercelTools: VercelMCPClientTools = {};
+        for (const [toolName, tool] of Object.entries(vercelTools)) {
+            rewrappedVercelTools[toolName] = createVercelTool({
+                ...tool,
+                execute: async (args, options) => {
+                    this.llmToolCalls.push({
+                        toolCallId: uuid(),
+                        toolName: toolName,
+                        parameters: args as Record<string, unknown>,
+                    });
+                    try {
+                        const toolResultGeneratorFn = this.mockedTools[toolName];
+                        if (toolResultGeneratorFn) {
+                            return await toolResultGeneratorFn(args);
+                        }
+
+                        return await tool.execute(args, options);
+                    } catch (error) {
+                        // There are cases when LLM calls the tools incorrectly
+                        // and the schema definition check fails. In production,
+                        // the tool calling agents are deployed with this fail
+                        // safe to allow LLM to course correct themselves. That
+                        // is exactly what we do here as well.
+                        return {
+                            isError: true,
+                            content: JSON.stringify(error),
+                        };
+                    }
+                },
+            });
+        }
+
+        return rewrappedVercelTools;
+    }
+
+    getLLMToolCalls(): LLMToolCall[] {
+        return this.llmToolCalls;
+    }
+
+    mockTools(mockedTools: MockedTools): void {
+        this.mockedTools = mockedTools;
+    }
+
+    resetForTests(): void {
+        this.mockTools({});
+        this.llmToolCalls = [];
+    }
+
+    static async initializeClient(mdbConnectionString: string): Promise<AccuracyTestingClient> {
+        const clientTransport = new StdioClientTransport({
+            command: process.execPath,
+            args: [MCP_SERVER_CLI_SCRIPT, "--connectionString", mdbConnectionString],
+        });
+
+        const client = await createMCPClient({
+            transport: clientTransport,
+        });
+
+        return new AccuracyTestingClient(client);
+    }
+}

package/tests/accuracy/sdk/agent.ts

@@ -0,0 +1,56 @@
+import { generateText, LanguageModelV1, experimental_createMCPClient } from "ai";
+import { Model } from "./models.js";
+
+const systemPrompt = [
+    'The keywords "MUST", "MUST NOT", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in RFC 2119',
+    "You are an expert AI assistant with access to a set of tools for MongoDB database operations.",
+    "You MUST use the most relevant tool to answer the user's request",
+    "When calling a tool, you MUST strictly follow its input schema and MUST provide all required arguments",
+    "If a task requires multiple tool calls, you MUST call all the necessary tools in sequence, following the requirements mentioned above for each tool called.",
+    'If you do not know the answer or the request cannot be fulfilled, you MUST reply with "I don\'t know"',
+];
+
+// These types are not exported by Vercel SDK so we derive them here to be
+// re-used again.
+export type VercelMCPClient = Awaited<ReturnType<typeof experimental_createMCPClient>>;
+export type VercelMCPClientTools = Awaited<ReturnType<VercelMCPClient["tools"]>>;
+export type VercelAgent = ReturnType<typeof getVercelToolCallingAgent>;
+
+export interface VercelAgentPromptResult {
+    respondingModel: string;
+    tokensUsage?: {
+        promptTokens?: number;
+        completionTokens?: number;
+        totalTokens?: number;
+    };
+    text: string;
+    messages: Record<string, unknown>[];
+}
+
+// Generic interface for Agent, in case we need to switch to some other agent
+// development SDK
+export interface Agent<Model = unknown, Tools = unknown, Result = unknown> {
+    prompt(prompt: string, model: Model, tools: Tools): Promise<Result>;
+}
+
+export function getVercelToolCallingAgent(
+    requestedSystemPrompt?: string
+): Agent<Model<LanguageModelV1>, VercelMCPClientTools, VercelAgentPromptResult> {
+    return {
+        async prompt(prompt: string, model: Model<LanguageModelV1>, tools: VercelMCPClientTools) {
+            const result = await generateText({
+                model: model.getModel(),
+                system: [...systemPrompt, requestedSystemPrompt].filter(Boolean).join("\n"),
+                prompt,
+                tools,
+                maxSteps: 100,
+            });
+            return {
+                text: result.text,
+                messages: result.response.messages,
+                respondingModel: result.response.modelId,
+                tokensUsage: result.usage,
+            };
+        },
+    };
+}

package/tests/accuracy/sdk/constants.ts

@@ -0,0 +1,26 @@
+import path from "path";
+import { fileURLToPath } from "url";
+
+const __dirname = fileURLToPath(import.meta.url);
+
+export const ROOT_DIR = path.join(__dirname, "..", "..", "..", "..");
+
+export const DIST_DIR = path.join(ROOT_DIR, "dist");
+
+export const RESOURCES_DIR = path.join(ROOT_DIR, "resources");
+
+export const MCP_SERVER_CLI_SCRIPT = path.join(DIST_DIR, "index.js");
+
+export const TEST_DATA_DUMPS_DIR = path.join(__dirname, "test-data-dumps");
+
+export const GENERATED_ASSETS_DIR = path.join(ROOT_DIR, ".accuracy");
+
+export const ACCURACY_RESULTS_DIR = path.join(GENERATED_ASSETS_DIR, "results");
+
+export const LATEST_ACCURACY_RUN_NAME = "latest-run";
+
+export const HTML_TEST_SUMMARY_FILE = path.join(GENERATED_ASSETS_DIR, "test-summary.html");
+
+export const MARKDOWN_TEST_BRIEF_FILE = path.join(GENERATED_ASSETS_DIR, "test-brief.md");
+
+export const HTML_TESTS_SUMMARY_TEMPLATE = path.join(RESOURCES_DIR, "test-summary-template.html");

package/tests/accuracy/sdk/describeAccuracyTests.ts

@@ -0,0 +1,126 @@
+import { describe, it, beforeAll, beforeEach, afterAll } from "vitest";
+import { getAvailableModels } from "./models.js";
+import { calculateToolCallingAccuracy } from "./accuracyScorer.js";
+import { getVercelToolCallingAgent, VercelAgent } from "./agent.js";
+import { prepareTestData, setupMongoDBIntegrationTest } from "../../integration/tools/mongodb/mongodbHelpers.js";
+import { AccuracyTestingClient, MockedTools } from "./accuracyTestingClient.js";
+import { AccuracyResultStorage, ExpectedToolCall } from "./accuracyResultStorage/resultStorage.js";
+import { getAccuracyResultStorage } from "./accuracyResultStorage/getAccuracyResultStorage.js";
+import { getCommitSHA } from "./gitInfo.js";
+
+export interface AccuracyTestConfig {
+    /** The prompt to be provided to LLM for evaluation. */
+    prompt: string;
+
+    /**
+     * A list of tools and their parameters that we expect LLM to call based on
+     * how vague or detailed the prompt is. Ideally this should be a list of
+     * bare minimum and critical tool calls that are required to solve the
+     * problem mentioned in the prompt but because, for even a slightly vague
+     * prompt, LLM might decide to do additional confirmation by calling other
+     * tools, its fine to include those other tool calls as well to get a
+     * perfect 1 on the tool calling accuracy score. */
+    expectedToolCalls: ExpectedToolCall[];
+
+    /**
+     * The additional system prompt to be appended to already injected system
+     * prompt. */
+    systemPrompt?: string;
+
+    /**
+     * A small hint appended to the actual prompt in test, which is supposed to
+     * hint LLM to assume that the MCP server is already connected so that it
+     * does not call the connect tool.
+     * By default it is assumed to be true */
+    injectConnectedAssumption?: boolean;
+
+    /**
+     * A map of tool names to their mocked implementation. When the mocked
+     * implementations are available, the testing client will prefer those over
+     * actual MCP tool calls. */
+    mockedTools?: MockedTools;
+}
+
+export function describeAccuracyTests(accuracyTestConfigs: AccuracyTestConfig[]) {
+    if (!process.env.MDB_ACCURACY_RUN_ID) {
+        throw new Error("MDB_ACCURACY_RUN_ID env variable is required for accuracy test runs!");
+    }
+
+    const models = getAvailableModels();
+    if (!models.length) {
+        throw new Error("No models available to test. Ensure that the API keys are properly setup!");
+    }
+
+    const eachModel = describe.each(models);
+
+    eachModel(`$displayName`, function (model) {
+        const accuracyRunId = `${process.env.MDB_ACCURACY_RUN_ID}`;
+        const mdbIntegration = setupMongoDBIntegrationTest();
+        const { populateTestData, cleanupTestDatabases } = prepareTestData(mdbIntegration);
+
+        let commitSHA: string;
+        let accuracyResultStorage: AccuracyResultStorage;
+        let testMCPClient: AccuracyTestingClient;
+        let agent: VercelAgent;
+
+        beforeAll(async () => {
+            const retrievedCommitSHA = await getCommitSHA();
+            if (!retrievedCommitSHA) {
+                throw new Error("Could not derive commitSHA, exiting accuracy tests!");
+            }
+            commitSHA = retrievedCommitSHA;
+
+            accuracyResultStorage = getAccuracyResultStorage();
+            testMCPClient = await AccuracyTestingClient.initializeClient(mdbIntegration.connectionString());
+            agent = getVercelToolCallingAgent();
+        });
+
+        beforeEach(async () => {
+            await cleanupTestDatabases(mdbIntegration);
+            await populateTestData();
+            testMCPClient.resetForTests();
+        });
+
+        afterAll(async () => {
+            await accuracyResultStorage?.close();
+            await testMCPClient?.close();
+        });
+
+        const eachTest = it.each(accuracyTestConfigs);
+
+        eachTest("$prompt", async function (testConfig) {
+            testMCPClient.mockTools(testConfig.mockedTools ?? {});
+            const toolsForModel = await testMCPClient.vercelTools();
+            const promptForModel =
+                testConfig.injectConnectedAssumption === false
+                    ? testConfig.prompt
+                    : [testConfig.prompt, "(Assume that you are already connected to a MongoDB cluster!)"].join(" ");
+
+            const timeBeforePrompt = Date.now();
+            const result = await agent.prompt(promptForModel, model, toolsForModel);
+            const timeAfterPrompt = Date.now();
+
+            const llmToolCalls = testMCPClient.getLLMToolCalls();
+            const toolCallingAccuracy = calculateToolCallingAccuracy(testConfig.expectedToolCalls, llmToolCalls);
+
+            const responseTime = timeAfterPrompt - timeBeforePrompt;
+            await accuracyResultStorage.saveModelResponseForPrompt({
+                commitSHA,
+                runId: accuracyRunId,
+                prompt: testConfig.prompt,
+                expectedToolCalls: testConfig.expectedToolCalls,
+                modelResponse: {
+                    provider: model.provider,
+                    requestedModel: model.modelName,
+                    respondingModel: result.respondingModel,
+                    llmResponseTime: responseTime,
+                    toolCallingAccuracy: toolCallingAccuracy,
+                    llmToolCalls: llmToolCalls,
+                    tokensUsed: result.tokensUsage,
+                    text: result.text,
+                    messages: result.messages,
+                },
+            });
+        });
+    });
+}

package/tests/accuracy/sdk/gitInfo.ts

@@ -0,0 +1,7 @@
+import { simpleGit } from "simple-git";
+
+export async function getCommitSHA(): Promise<string | undefined> {
+    const commitLogs = await simpleGit().log();
+    const lastCommit = commitLogs.latest;
+    return lastCommit?.hash;
+}