mongodb-mcp-server 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/accuracy-tests.yml +55 -0
- package/.github/workflows/check.yml +1 -1
- package/.github/workflows/code_health.yaml +1 -1
- package/.github/workflows/code_health_fork.yaml +0 -14
- package/.github/workflows/dependabot_pr.yaml +26 -0
- package/.github/workflows/jira-issue.yml +72 -0
- package/.vscode/extensions.json +1 -1
- package/.vscode/launch.json +11 -1
- package/.vscode/settings.json +1 -11
- package/Dockerfile +1 -0
- package/README.md +118 -31
- package/dist/common/atlas/accessListUtils.js +36 -0
- package/dist/common/atlas/accessListUtils.js.map +1 -0
- package/dist/common/atlas/apiClient.js +25 -6
- package/dist/common/atlas/apiClient.js.map +1 -1
- package/dist/common/atlas/cluster.js +1 -1
- package/dist/common/atlas/cluster.js.map +1 -1
- package/dist/{config.js → common/config.js} +7 -1
- package/dist/common/config.js.map +1 -0
- package/dist/common/errors.js.map +1 -0
- package/dist/{logger.js → common/logger.js} +20 -18
- package/dist/common/logger.js.map +1 -0
- package/dist/common/managedTimeout.js +20 -0
- package/dist/common/managedTimeout.js.map +1 -0
- package/dist/common/packageInfo.js.map +1 -0
- package/dist/{session.js → common/session.js} +20 -21
- package/dist/common/session.js.map +1 -0
- package/dist/common/sessionStore.js +73 -0
- package/dist/common/sessionStore.js.map +1 -0
- package/dist/helpers/container.js +28 -0
- package/dist/helpers/container.js.map +1 -0
- package/dist/helpers/generatePassword.js.map +1 -0
- package/dist/helpers/indexCheck.js +1 -1
- package/dist/helpers/indexCheck.js.map +1 -1
- package/dist/index.js +30 -37
- package/dist/index.js.map +1 -1
- package/dist/server.js +43 -6
- package/dist/server.js.map +1 -1
- package/dist/telemetry/constants.js +1 -1
- package/dist/telemetry/constants.js.map +1 -1
- package/dist/telemetry/telemetry.js +28 -21
- package/dist/telemetry/telemetry.js.map +1 -1
- package/dist/tools/atlas/atlasTool.js +3 -3
- package/dist/tools/atlas/atlasTool.js.map +1 -1
- package/dist/tools/atlas/connect/connectCluster.js +198 -0
- package/dist/tools/atlas/connect/connectCluster.js.map +1 -0
- package/dist/tools/atlas/create/createAccessList.js +9 -10
- package/dist/tools/atlas/create/createAccessList.js.map +1 -1
- package/dist/tools/atlas/create/createDBUser.js +3 -1
- package/dist/tools/atlas/create/createDBUser.js.map +1 -1
- package/dist/tools/atlas/create/createFreeCluster.js +2 -0
- package/dist/tools/atlas/create/createFreeCluster.js.map +1 -1
- package/dist/tools/atlas/create/createProject.js.map +1 -1
- package/dist/tools/atlas/read/inspectAccessList.js.map +1 -1
- package/dist/tools/atlas/read/inspectCluster.js.map +1 -1
- package/dist/tools/atlas/read/listAlerts.js.map +1 -1
- package/dist/tools/atlas/read/listClusters.js.map +1 -1
- package/dist/tools/atlas/read/listDBUsers.js.map +1 -1
- package/dist/tools/atlas/read/listOrgs.js.map +1 -1
- package/dist/tools/atlas/read/listProjects.js.map +1 -1
- package/dist/tools/atlas/tools.js +1 -1
- package/dist/tools/atlas/tools.js.map +1 -1
- package/dist/tools/mongodb/{metadata → connect}/connect.js +7 -4
- package/dist/tools/mongodb/connect/connect.js.map +1 -0
- package/dist/tools/mongodb/create/createCollection.js.map +1 -1
- package/dist/tools/mongodb/create/createIndex.js +1 -1
- package/dist/tools/mongodb/create/createIndex.js.map +1 -1
- package/dist/tools/mongodb/create/insertMany.js +1 -1
- package/dist/tools/mongodb/create/insertMany.js.map +1 -1
- package/dist/tools/mongodb/delete/deleteMany.js +2 -1
- package/dist/tools/mongodb/delete/deleteMany.js.map +1 -1
- package/dist/tools/mongodb/delete/dropCollection.js.map +1 -1
- package/dist/tools/mongodb/delete/dropDatabase.js.map +1 -1
- package/dist/tools/mongodb/metadata/collectionSchema.js.map +1 -1
- package/dist/tools/mongodb/metadata/collectionStorageSize.js.map +1 -1
- package/dist/tools/mongodb/metadata/dbStats.js.map +1 -1
- package/dist/tools/mongodb/metadata/explain.js +1 -1
- package/dist/tools/mongodb/metadata/explain.js.map +1 -1
- package/dist/tools/mongodb/metadata/listCollections.js.map +1 -1
- package/dist/tools/mongodb/metadata/listDatabases.js.map +1 -1
- package/dist/tools/mongodb/metadata/logs.js.map +1 -1
- package/dist/tools/mongodb/mongodbTool.js +37 -10
- package/dist/tools/mongodb/mongodbTool.js.map +1 -1
- package/dist/tools/mongodb/read/aggregate.js +1 -1
- package/dist/tools/mongodb/read/aggregate.js.map +1 -1
- package/dist/tools/mongodb/read/collectionIndexes.js.map +1 -1
- package/dist/tools/mongodb/read/count.js +2 -1
- package/dist/tools/mongodb/read/count.js.map +1 -1
- package/dist/tools/mongodb/read/find.js +7 -4
- package/dist/tools/mongodb/read/find.js.map +1 -1
- package/dist/tools/mongodb/tools.js +1 -1
- package/dist/tools/mongodb/tools.js.map +1 -1
- package/dist/tools/mongodb/update/renameCollection.js.map +1 -1
- package/dist/tools/mongodb/update/updateMany.js +4 -2
- package/dist/tools/mongodb/update/updateMany.js.map +1 -1
- package/dist/tools/tool.js +8 -5
- package/dist/tools/tool.js.map +1 -1
- package/dist/transports/base.js +26 -0
- package/dist/transports/base.js.map +1 -0
- package/dist/{helpers/EJsonTransport.js → transports/stdio.js} +24 -2
- package/dist/transports/stdio.js.map +1 -0
- package/dist/transports/streamableHttp.js +140 -0
- package/dist/transports/streamableHttp.js.map +1 -0
- package/eslint.config.js +13 -4
- package/package.json +43 -33
- package/resources/test-summary-template.html +415 -0
- package/scripts/accuracy/generateTestSummary.ts +335 -0
- package/scripts/accuracy/runAccuracyTests.sh +45 -0
- package/scripts/accuracy/updateAccuracyRunStatus.ts +21 -0
- package/src/common/atlas/accessListUtils.ts +54 -0
- package/src/common/atlas/apiClient.ts +25 -6
- package/src/common/atlas/cluster.ts +1 -1
- package/src/{config.ts → common/config.ts} +14 -2
- package/src/{logger.ts → common/logger.ts} +21 -23
- package/src/common/managedTimeout.ts +27 -0
- package/src/{session.ts → common/session.ts} +24 -26
- package/src/common/sessionStore.ts +111 -0
- package/src/helpers/container.ts +35 -0
- package/src/helpers/indexCheck.ts +1 -1
- package/src/index.ts +30 -40
- package/src/server.ts +54 -10
- package/src/telemetry/constants.ts +1 -1
- package/src/telemetry/telemetry.ts +34 -26
- package/src/telemetry/types.ts +2 -0
- package/src/tools/atlas/atlasTool.ts +4 -4
- package/src/tools/atlas/connect/connectCluster.ts +259 -0
- package/src/tools/atlas/create/createAccessList.ts +15 -13
- package/src/tools/atlas/create/createDBUser.ts +5 -3
- package/src/tools/atlas/create/createFreeCluster.ts +4 -2
- package/src/tools/atlas/create/createProject.ts +2 -2
- package/src/tools/atlas/read/inspectAccessList.ts +2 -2
- package/src/tools/atlas/read/inspectCluster.ts +2 -2
- package/src/tools/atlas/read/listAlerts.ts +2 -2
- package/src/tools/atlas/read/listClusters.ts +2 -2
- package/src/tools/atlas/read/listDBUsers.ts +2 -2
- package/src/tools/atlas/read/listOrgs.ts +2 -2
- package/src/tools/atlas/read/listProjects.ts +2 -2
- package/src/tools/atlas/tools.ts +1 -1
- package/src/tools/mongodb/{metadata → connect}/connect.ts +12 -9
- package/src/tools/mongodb/create/createCollection.ts +2 -2
- package/src/tools/mongodb/create/createIndex.ts +3 -3
- package/src/tools/mongodb/create/insertMany.ts +3 -3
- package/src/tools/mongodb/delete/deleteMany.ts +4 -3
- package/src/tools/mongodb/delete/dropCollection.ts +2 -2
- package/src/tools/mongodb/delete/dropDatabase.ts +2 -2
- package/src/tools/mongodb/metadata/collectionSchema.ts +2 -2
- package/src/tools/mongodb/metadata/collectionStorageSize.ts +2 -2
- package/src/tools/mongodb/metadata/dbStats.ts +2 -2
- package/src/tools/mongodb/metadata/explain.ts +3 -3
- package/src/tools/mongodb/metadata/listCollections.ts +2 -2
- package/src/tools/mongodb/metadata/listDatabases.ts +2 -2
- package/src/tools/mongodb/metadata/logs.ts +2 -2
- package/src/tools/mongodb/mongodbTool.ts +50 -14
- package/src/tools/mongodb/read/aggregate.ts +3 -3
- package/src/tools/mongodb/read/collectionIndexes.ts +2 -2
- package/src/tools/mongodb/read/count.ts +4 -3
- package/src/tools/mongodb/read/find.ts +11 -6
- package/src/tools/mongodb/tools.ts +1 -1
- package/src/tools/mongodb/update/renameCollection.ts +2 -2
- package/src/tools/mongodb/update/updateMany.ts +6 -4
- package/src/tools/tool.ts +18 -13
- package/src/transports/base.ts +34 -0
- package/src/{helpers/EJsonTransport.ts → transports/stdio.ts} +30 -1
- package/src/transports/streamableHttp.ts +178 -0
- package/tests/accuracy/aggregate.test.ts +27 -0
- package/tests/accuracy/collectionIndexes.test.ts +40 -0
- package/tests/accuracy/collectionSchema.test.ts +28 -0
- package/tests/accuracy/collectionStorageSize.test.ts +41 -0
- package/tests/accuracy/count.test.ts +44 -0
- package/tests/accuracy/createCollection.test.ts +46 -0
- package/tests/accuracy/createIndex.test.ts +37 -0
- package/tests/accuracy/dbStats.test.ts +15 -0
- package/tests/accuracy/deleteMany.test.ts +44 -0
- package/tests/accuracy/dropCollection.test.ts +74 -0
- package/tests/accuracy/dropDatabase.test.ts +41 -0
- package/tests/accuracy/explain.test.ts +73 -0
- package/tests/accuracy/find.test.ts +114 -0
- package/tests/accuracy/insertMany.test.ts +48 -0
- package/tests/accuracy/listCollections.test.ts +60 -0
- package/tests/accuracy/listDatabases.test.ts +31 -0
- package/tests/accuracy/logs.test.ts +28 -0
- package/tests/accuracy/renameCollection.test.ts +31 -0
- package/tests/accuracy/sdk/accuracyResultStorage/diskStorage.ts +189 -0
- package/tests/accuracy/sdk/accuracyResultStorage/getAccuracyResultStorage.ts +11 -0
- package/tests/accuracy/sdk/accuracyResultStorage/mongodbStorage.ts +151 -0
- package/tests/accuracy/sdk/accuracyResultStorage/resultStorage.ts +117 -0
- package/tests/accuracy/sdk/accuracyScorer.ts +93 -0
- package/tests/accuracy/sdk/accuracyTestingClient.ts +94 -0
- package/tests/accuracy/sdk/agent.ts +56 -0
- package/tests/accuracy/sdk/constants.ts +26 -0
- package/tests/accuracy/sdk/describeAccuracyTests.ts +126 -0
- package/tests/accuracy/sdk/gitInfo.ts +7 -0
- package/tests/accuracy/sdk/matcher.ts +193 -0
- package/tests/accuracy/sdk/models.ts +95 -0
- package/tests/accuracy/test-data-dumps/comics.books.json +417 -0
- package/tests/accuracy/test-data-dumps/comics.characters.json +402 -0
- package/tests/accuracy/test-data-dumps/mflix.movies.json +496 -0
- package/tests/accuracy/test-data-dumps/mflix.shows.json +572 -0
- package/tests/accuracy/updateMany.test.ts +42 -0
- package/tests/integration/helpers.ts +9 -9
- package/tests/integration/indexCheck.test.ts +1 -0
- package/tests/integration/server.test.ts +1 -0
- package/tests/integration/telemetry.test.ts +4 -3
- package/tests/integration/tools/atlas/accessLists.test.ts +22 -2
- package/tests/integration/tools/atlas/alerts.test.ts +3 -2
- package/tests/integration/tools/atlas/atlasHelpers.ts +3 -0
- package/tests/integration/tools/atlas/clusters.test.ts +68 -16
- package/tests/integration/tools/atlas/dbUsers.test.ts +14 -1
- package/tests/integration/tools/atlas/orgs.test.ts +2 -1
- package/tests/integration/tools/atlas/projects.test.ts +4 -3
- package/tests/integration/tools/mongodb/{metadata → connect}/connect.test.ts +34 -3
- package/tests/integration/tools/mongodb/create/createCollection.test.ts +1 -0
- package/tests/integration/tools/mongodb/create/createIndex.test.ts +1 -0
- package/tests/integration/tools/mongodb/create/insertMany.test.ts +1 -0
- package/tests/integration/tools/mongodb/delete/deleteMany.test.ts +1 -0
- package/tests/integration/tools/mongodb/delete/dropCollection.test.ts +1 -1
- package/tests/integration/tools/mongodb/delete/dropDatabase.test.ts +1 -0
- package/tests/integration/tools/mongodb/metadata/collectionSchema.test.ts +1 -0
- package/tests/integration/tools/mongodb/metadata/collectionStorageSize.test.ts +1 -0
- package/tests/integration/tools/mongodb/metadata/dbStats.test.ts +1 -0
- package/tests/integration/tools/mongodb/metadata/explain.test.ts +1 -0
- package/tests/integration/tools/mongodb/metadata/listCollections.test.ts +1 -0
- package/tests/integration/tools/mongodb/metadata/listDatabases.test.ts +3 -2
- package/tests/integration/tools/mongodb/metadata/logs.test.ts +1 -0
- package/tests/integration/tools/mongodb/mongodbHelpers.ts +66 -2
- package/tests/integration/tools/mongodb/read/aggregate.test.ts +2 -1
- package/tests/integration/tools/mongodb/read/collectionIndexes.test.ts +1 -0
- package/tests/integration/tools/mongodb/read/count.test.ts +1 -0
- package/tests/integration/tools/mongodb/read/find.test.ts +2 -1
- package/tests/integration/tools/mongodb/update/renameCollection.test.ts +1 -0
- package/tests/integration/tools/mongodb/update/updateMany.test.ts +1 -0
- package/tests/integration/transports/stdio.test.ts +40 -0
- package/tests/integration/transports/streamableHttp.test.ts +56 -0
- package/tests/matchers/toIncludeSameMembers.test.ts +59 -0
- package/tests/matchers/toIncludeSameMembers.ts +12 -0
- package/tests/setup.ts +7 -0
- package/tests/unit/accessListUtils.test.ts +39 -0
- package/tests/unit/accuracyScorer.test.ts +390 -0
- package/tests/unit/{apiClient.test.ts → common/apiClient.test.ts} +15 -15
- package/tests/unit/common/managedTimeout.test.ts +67 -0
- package/tests/unit/{session.test.ts → common/session.test.ts} +7 -12
- package/tests/unit/{indexCheck.test.ts → helpers/indexCheck.test.ts} +2 -1
- package/tests/unit/telemetry.test.ts +52 -42
- package/tests/unit/{EJsonTransport.test.ts → transports/stdio.test.ts} +4 -4
- package/tests/vitest.d.ts +11 -0
- package/tsconfig.json +0 -1
- package/{tsconfig.jest.json → tsconfig.test.json} +1 -2
- package/vitest.config.ts +41 -0
- package/dist/common/atlas/generatePassword.js.map +0 -1
- package/dist/config.js.map +0 -1
- package/dist/errors.js.map +0 -1
- package/dist/helpers/EJsonTransport.js.map +0 -1
- package/dist/helpers/packageInfo.js.map +0 -1
- package/dist/logger.js.map +0 -1
- package/dist/session.js.map +0 -1
- package/dist/tools/atlas/metadata/connectCluster.js +0 -100
- package/dist/tools/atlas/metadata/connectCluster.js.map +0 -1
- package/dist/tools/mongodb/metadata/connect.js.map +0 -1
- package/global.d.ts +0 -1
- package/jest.config.cjs +0 -22
- package/src/tools/atlas/metadata/connectCluster.ts +0 -121
- /package/dist/{errors.js → common/errors.js} +0 -0
- /package/dist/{helpers → common}/packageInfo.js +0 -0
- /package/dist/{common/atlas → helpers}/generatePassword.js +0 -0
- /package/src/{errors.ts → common/errors.ts} +0 -0
- /package/src/{helpers → common}/packageInfo.ts +0 -0
- /package/src/{common/atlas → helpers}/generatePassword.ts +0 -0
|
@@ -0,0 +1,335 @@
|
|
|
1
|
+
import path from "path";
|
|
2
|
+
import { readFile, writeFile, mkdir } from "fs/promises";
|
|
3
|
+
import { getAccuracyResultStorage } from "../../tests/accuracy/sdk/accuracyResultStorage/getAccuracyResultStorage.js";
|
|
4
|
+
import {
|
|
5
|
+
AccuracyResult,
|
|
6
|
+
AccuracyRunStatuses,
|
|
7
|
+
ExpectedToolCall,
|
|
8
|
+
LLMToolCall,
|
|
9
|
+
ModelResponse,
|
|
10
|
+
} from "../../tests/accuracy/sdk/accuracyResultStorage/resultStorage.js";
|
|
11
|
+
import { getCommitSHA } from "../../tests/accuracy/sdk/gitInfo.js";
|
|
12
|
+
import {
|
|
13
|
+
HTML_TEST_SUMMARY_FILE,
|
|
14
|
+
HTML_TESTS_SUMMARY_TEMPLATE,
|
|
15
|
+
MARKDOWN_TEST_BRIEF_FILE,
|
|
16
|
+
} from "../../tests/accuracy/sdk/constants.js";
|
|
17
|
+
|
|
18
|
+
type ComparableAccuracyResult = Omit<AccuracyResult, "promptResults"> & {
|
|
19
|
+
promptAndModelResponses: PromptAndModelResponse[];
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
interface PromptAndModelResponse extends ModelResponse {
|
|
23
|
+
prompt: string;
|
|
24
|
+
expectedToolCalls: ExpectedToolCall[];
|
|
25
|
+
baselineToolAccuracy?: number;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
interface BaselineRunInfo {
|
|
29
|
+
commitSHA: string;
|
|
30
|
+
accuracyRunId: string;
|
|
31
|
+
accuracyRunStatus: AccuracyRunStatuses;
|
|
32
|
+
createdOn: string;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function populateTemplate(template: string, data: Record<string, string>): string {
|
|
36
|
+
return template.replace(/\{\{(\w+)\}\}/g, (_, key: string) => data[key] ?? "");
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function formatRunStatus(status: AccuracyRunStatuses) {
|
|
40
|
+
const statusClasses = ["chip", "run-status"];
|
|
41
|
+
if (status === "done") {
|
|
42
|
+
statusClasses.push("perfect");
|
|
43
|
+
} else if (status === "in-progress" || status === "failed") {
|
|
44
|
+
statusClasses.push("poor");
|
|
45
|
+
}
|
|
46
|
+
return `<span class="${statusClasses.join(" ")}">${status}</span>`;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function formatAccuracy(accuracy: number): string {
|
|
50
|
+
return (accuracy * 100).toFixed(1) + "%";
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function getAccuracyClass(accuracy: number): string {
|
|
54
|
+
if (accuracy === 1) return "chip perfect";
|
|
55
|
+
if (accuracy >= 0.75) return "chip good";
|
|
56
|
+
return "chip poor";
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function formatToolCallsWithTooltip(toolCalls: ExpectedToolCall[] | LLMToolCall[]): string {
|
|
60
|
+
return toolCalls
|
|
61
|
+
.map((call) => {
|
|
62
|
+
const params = JSON.stringify(call.parameters, null, 2);
|
|
63
|
+
return `<span class="tool-call" title="${params.replace(/"/g, """)}">${call.toolName}</span>`;
|
|
64
|
+
})
|
|
65
|
+
.join(", ");
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function formatTokenUsage(tokensUsage: {
|
|
69
|
+
promptTokens?: number;
|
|
70
|
+
completionTokens?: number;
|
|
71
|
+
totalTokens?: number;
|
|
72
|
+
}): string {
|
|
73
|
+
const total = tokensUsage.totalTokens || "-";
|
|
74
|
+
const prompt = tokensUsage.promptTokens || "-";
|
|
75
|
+
const completion = tokensUsage.completionTokens || "-";
|
|
76
|
+
|
|
77
|
+
const tooltip = [`Prompt: ${prompt}`, `Completion: ${completion}`, `Total: ${total}`].join("\n");
|
|
78
|
+
return `<span class="tokens-usage" title="${tooltip}">${total}</span>`;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function formatMessages(messages: Array<Record<string, unknown>>): string {
|
|
82
|
+
return messages.map((msg) => JSON.stringify(msg, null, 2)).join("\n\n");
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function formatCurrentAccuracy(response: PromptAndModelResponse): string {
|
|
86
|
+
const currentAccuracyText = formatAccuracy(response.toolCallingAccuracy);
|
|
87
|
+
const comparisonClass = getAccuracyClass(response.toolCallingAccuracy);
|
|
88
|
+
let comparisonIcon = "";
|
|
89
|
+
|
|
90
|
+
if (typeof response.baselineToolAccuracy === "number") {
|
|
91
|
+
if (response.toolCallingAccuracy > response.baselineToolAccuracy) {
|
|
92
|
+
comparisonIcon = " ↗";
|
|
93
|
+
} else if (response.toolCallingAccuracy < response.baselineToolAccuracy) {
|
|
94
|
+
comparisonIcon = " ↘";
|
|
95
|
+
} else {
|
|
96
|
+
comparisonIcon = " →";
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
return `<span class="${comparisonClass}">${currentAccuracyText}${comparisonIcon}</span>`;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function formatBaselineAccuracy(response: PromptAndModelResponse): string {
|
|
104
|
+
if (response.baselineToolAccuracy === null || response.baselineToolAccuracy === undefined) {
|
|
105
|
+
return '<span class="accuracy-comparison">N/A</span>';
|
|
106
|
+
}
|
|
107
|
+
return `<span class="accuracy-comparison">${formatAccuracy(response.baselineToolAccuracy)}</span>`;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function getTestSummary(comparableResult: ComparableAccuracyResult) {
|
|
111
|
+
const responses = comparableResult.promptAndModelResponses;
|
|
112
|
+
return {
|
|
113
|
+
totalPrompts: new Set(responses.map((r) => r.prompt)).size,
|
|
114
|
+
totalModels: new Set(responses.map((r) => `${r.provider} ${r.requestedModel}`)).size,
|
|
115
|
+
responsesWithZeroAccuracy: responses.filter((r) => r.toolCallingAccuracy === 0),
|
|
116
|
+
responsesWith75Accuracy: responses.filter((r) => r.toolCallingAccuracy === 0.75),
|
|
117
|
+
responsesWith100Accuracy: responses.filter((r) => r.toolCallingAccuracy === 1),
|
|
118
|
+
averageAccuracy:
|
|
119
|
+
responses.length > 0 ? responses.reduce((sum, r) => sum + r.toolCallingAccuracy, 0) / responses.length : 0,
|
|
120
|
+
responsesImproved: responses.filter(
|
|
121
|
+
(r) => typeof r.baselineToolAccuracy === "number" && r.toolCallingAccuracy > r.baselineToolAccuracy
|
|
122
|
+
).length,
|
|
123
|
+
responsesRegressed: responses.filter(
|
|
124
|
+
(r) => typeof r.baselineToolAccuracy === "number" && r.toolCallingAccuracy < r.baselineToolAccuracy
|
|
125
|
+
).length,
|
|
126
|
+
reportGeneratedOn: new Date().toLocaleString(),
|
|
127
|
+
resultCreatedOn: new Date(comparableResult.createdOn).toLocaleString(),
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
async function generateHtmlReport(
|
|
132
|
+
comparableResult: ComparableAccuracyResult,
|
|
133
|
+
testSummary: ReturnType<typeof getTestSummary>,
|
|
134
|
+
baselineInfo: BaselineRunInfo | null
|
|
135
|
+
): Promise<string> {
|
|
136
|
+
const responses = comparableResult.promptAndModelResponses;
|
|
137
|
+
const tableRows = responses
|
|
138
|
+
.map(
|
|
139
|
+
(response, index) => `
|
|
140
|
+
<tr class="test-row" onclick="toggleDetails(${index})">
|
|
141
|
+
<td class="prompt-cell">
|
|
142
|
+
<span class="expand-indicator" id="indicator-${index}">▶</span>
|
|
143
|
+
${response.prompt}
|
|
144
|
+
</td>
|
|
145
|
+
<td class="model-cell">${response.provider} - ${response.requestedModel}</td>
|
|
146
|
+
<td class="tool-calls-cell">${formatToolCallsWithTooltip(response.expectedToolCalls)}</td>
|
|
147
|
+
<td class="tool-calls-cell">${formatToolCallsWithTooltip(response.llmToolCalls)}</td>
|
|
148
|
+
<td class="accuracy-cell">${formatCurrentAccuracy(response)}</td>
|
|
149
|
+
<td class="baseline-accuracy-cell">${formatBaselineAccuracy(response)}</td>
|
|
150
|
+
<td class="response-time-cell">${response.llmResponseTime.toFixed(2)}</td>
|
|
151
|
+
<td class="tokens-cell">${formatTokenUsage(response.tokensUsed || {})}</td>
|
|
152
|
+
</tr>
|
|
153
|
+
<tr class="details-row" id="details-${index}">
|
|
154
|
+
<td colspan="8">
|
|
155
|
+
<div class="details-content">
|
|
156
|
+
<div class="conversation-section">
|
|
157
|
+
<h4>🤖 LLM Response</h4>
|
|
158
|
+
<div class="conversation-content">${response.text || "N/A"}</div>
|
|
159
|
+
</div>
|
|
160
|
+
<div class="conversation-section">
|
|
161
|
+
<h4>💬 Conversation Messages</h4>
|
|
162
|
+
<div class="conversation-content">${formatMessages(response.messages || [])}</div>
|
|
163
|
+
</div>
|
|
164
|
+
</div>
|
|
165
|
+
</td>
|
|
166
|
+
</tr>
|
|
167
|
+
`
|
|
168
|
+
)
|
|
169
|
+
.join("");
|
|
170
|
+
|
|
171
|
+
const template = await readFile(HTML_TESTS_SUMMARY_TEMPLATE, "utf8");
|
|
172
|
+
return populateTemplate(template, {
|
|
173
|
+
commitSHA: comparableResult.commitSHA,
|
|
174
|
+
accuracyRunId: comparableResult.runId,
|
|
175
|
+
accuracyRunStatus: formatRunStatus(comparableResult.runStatus),
|
|
176
|
+
reportGeneratedOn: testSummary.reportGeneratedOn,
|
|
177
|
+
createdOn: testSummary.resultCreatedOn,
|
|
178
|
+
totalPrompts: String(testSummary.totalPrompts),
|
|
179
|
+
totalModels: String(testSummary.totalModels),
|
|
180
|
+
responsesWithZeroAccuracy: String(testSummary.responsesWithZeroAccuracy.length),
|
|
181
|
+
averageAccuracy: formatAccuracy(testSummary.averageAccuracy),
|
|
182
|
+
baselineCommitSHA: baselineInfo?.commitSHA || "-",
|
|
183
|
+
baselineAccuracyRunId: baselineInfo?.accuracyRunId || "-",
|
|
184
|
+
baselineAccuracyRunStatus: baselineInfo?.accuracyRunStatus
|
|
185
|
+
? formatRunStatus(baselineInfo?.accuracyRunStatus)
|
|
186
|
+
: "-",
|
|
187
|
+
baselineCreatedOn: baselineInfo?.createdOn || "-",
|
|
188
|
+
responsesImproved: baselineInfo ? String(testSummary.responsesImproved) : "-",
|
|
189
|
+
responsesRegressed: baselineInfo ? String(testSummary.responsesRegressed) : "-",
|
|
190
|
+
tableRows,
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function generateMarkdownBrief(
|
|
195
|
+
comparableResult: ComparableAccuracyResult,
|
|
196
|
+
testSummary: ReturnType<typeof getTestSummary>,
|
|
197
|
+
baselineInfo: BaselineRunInfo | null
|
|
198
|
+
): string {
|
|
199
|
+
const markdownTexts = [
|
|
200
|
+
"# 📊 Accuracy Test Results",
|
|
201
|
+
"## 📈 Summary",
|
|
202
|
+
"| Metric | Value |",
|
|
203
|
+
"|--------|-------|",
|
|
204
|
+
`| **Commit SHA** | \`${comparableResult.commitSHA}\` |`,
|
|
205
|
+
`| **Run ID** | \`${comparableResult.runId}\` |`,
|
|
206
|
+
`| **Status** | ${comparableResult.runStatus} |`,
|
|
207
|
+
`| **Total Prompts Evaluated** | ${testSummary.totalPrompts} |`,
|
|
208
|
+
`| **Models Tested** | ${testSummary.totalModels} |`,
|
|
209
|
+
`| **Average Accuracy** | ${formatAccuracy(testSummary.averageAccuracy)} |`,
|
|
210
|
+
`| **Responses with 0% Accuracy** | ${testSummary.responsesWithZeroAccuracy.length} |`,
|
|
211
|
+
`| **Responses with 75% Accuracy** | ${testSummary.responsesWith75Accuracy.length} |`,
|
|
212
|
+
`| **Responses with 100% Accuracy** | ${testSummary.responsesWith100Accuracy.length} |`,
|
|
213
|
+
"",
|
|
214
|
+
];
|
|
215
|
+
|
|
216
|
+
if (baselineInfo) {
|
|
217
|
+
markdownTexts.push(
|
|
218
|
+
...[
|
|
219
|
+
"## 📊 Baseline Comparison",
|
|
220
|
+
"|--------|-------|",
|
|
221
|
+
`| **Baseline Commit** | \`${baselineInfo.commitSHA}\` |`,
|
|
222
|
+
`| **Baseline Run ID** | \`${baselineInfo.accuracyRunId}\` |`,
|
|
223
|
+
`| **Baseline Run Status** | \`${baselineInfo.accuracyRunStatus}\` |`,
|
|
224
|
+
`| **Responses Improved** | ${testSummary.responsesImproved} |`,
|
|
225
|
+
`| **Responses Regressed** | ${testSummary.responsesRegressed} |`,
|
|
226
|
+
"",
|
|
227
|
+
]
|
|
228
|
+
);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
const { GITHUB_SERVER_URL, GITHUB_REPOSITORY, GITHUB_RUN_ID } = process.env;
|
|
232
|
+
const githubRunUrl =
|
|
233
|
+
GITHUB_SERVER_URL && GITHUB_REPOSITORY && GITHUB_RUN_ID
|
|
234
|
+
? `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}`
|
|
235
|
+
: null;
|
|
236
|
+
|
|
237
|
+
const reportLinkText = githubRunUrl
|
|
238
|
+
? `📎 **[Download Full HTML Report](${githubRunUrl})** - Look for the \`accuracy-test-summary\` artifact for detailed results.`
|
|
239
|
+
: `📎 **Full HTML Report**: \`${HTML_TEST_SUMMARY_FILE}\``;
|
|
240
|
+
|
|
241
|
+
markdownTexts.push(...["---", reportLinkText, "", `*Report generated on: ${testSummary.reportGeneratedOn}*`]);
|
|
242
|
+
|
|
243
|
+
return markdownTexts.join("\n");
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
async function generateTestSummary() {
|
|
247
|
+
const storage = getAccuracyResultStorage();
|
|
248
|
+
try {
|
|
249
|
+
const baselineCommit = process.env.MDB_ACCURACY_BASELINE_COMMIT;
|
|
250
|
+
const accuracyRunCommit = await getCommitSHA();
|
|
251
|
+
const accuracyRunId = process.env.MDB_ACCURACY_RUN_ID;
|
|
252
|
+
|
|
253
|
+
if (!accuracyRunCommit) {
|
|
254
|
+
throw new Error("Cannot generate summary without accuracyRunCommit");
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
const accuracyRunResult = await storage.getAccuracyResult(accuracyRunCommit, accuracyRunId);
|
|
258
|
+
if (!accuracyRunResult) {
|
|
259
|
+
throw new Error(
|
|
260
|
+
`No accuracy run result found for commitSHA - ${accuracyRunCommit}, runId - ${accuracyRunId}`
|
|
261
|
+
);
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
const baselineAccuracyRunResult = baselineCommit ? await storage.getAccuracyResult(baselineCommit) : null;
|
|
265
|
+
const baselineInfo: BaselineRunInfo | null =
|
|
266
|
+
baselineCommit && baselineAccuracyRunResult
|
|
267
|
+
? {
|
|
268
|
+
commitSHA: baselineCommit,
|
|
269
|
+
accuracyRunId: baselineAccuracyRunResult.runId,
|
|
270
|
+
accuracyRunStatus: baselineAccuracyRunResult.runStatus,
|
|
271
|
+
createdOn: new Date(baselineAccuracyRunResult.createdOn).toLocaleString(),
|
|
272
|
+
}
|
|
273
|
+
: null;
|
|
274
|
+
|
|
275
|
+
const comparableAccuracyResult: ComparableAccuracyResult = {
|
|
276
|
+
...accuracyRunResult,
|
|
277
|
+
promptAndModelResponses: accuracyRunResult.promptResults.flatMap<PromptAndModelResponse>(
|
|
278
|
+
(currentPromptResult) => {
|
|
279
|
+
const baselinePromptResult = baselineAccuracyRunResult?.promptResults.find((baselineResult) => {
|
|
280
|
+
return baselineResult.prompt === currentPromptResult.prompt;
|
|
281
|
+
});
|
|
282
|
+
|
|
283
|
+
return currentPromptResult.modelResponses.map<PromptAndModelResponse>((currentModelResponse) => {
|
|
284
|
+
const baselineModelResponse = baselinePromptResult?.modelResponses.find(
|
|
285
|
+
(baselineModelResponse) => {
|
|
286
|
+
return (
|
|
287
|
+
baselineModelResponse.provider === currentModelResponse.provider &&
|
|
288
|
+
baselineModelResponse.requestedModel === currentModelResponse.requestedModel
|
|
289
|
+
);
|
|
290
|
+
}
|
|
291
|
+
);
|
|
292
|
+
return {
|
|
293
|
+
...currentModelResponse,
|
|
294
|
+
prompt: currentPromptResult.prompt,
|
|
295
|
+
expectedToolCalls: currentPromptResult.expectedToolCalls,
|
|
296
|
+
baselineToolAccuracy: baselineModelResponse?.toolCallingAccuracy,
|
|
297
|
+
};
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
),
|
|
301
|
+
};
|
|
302
|
+
|
|
303
|
+
// Ensure that our writable path actually exist.
|
|
304
|
+
await mkdir(path.dirname(HTML_TEST_SUMMARY_FILE), { recursive: true });
|
|
305
|
+
|
|
306
|
+
console.log(`\n📊 Generating test summary for accuracy run: ${accuracyRunId}\n`);
|
|
307
|
+
const testSummary = getTestSummary(comparableAccuracyResult);
|
|
308
|
+
|
|
309
|
+
const htmlReport = await generateHtmlReport(comparableAccuracyResult, testSummary, baselineInfo);
|
|
310
|
+
await writeFile(HTML_TEST_SUMMARY_FILE, htmlReport, "utf8");
|
|
311
|
+
console.log(`✅ HTML report generated: ${HTML_TEST_SUMMARY_FILE}`);
|
|
312
|
+
|
|
313
|
+
const markdownBrief = generateMarkdownBrief(comparableAccuracyResult, testSummary, baselineInfo);
|
|
314
|
+
await writeFile(MARKDOWN_TEST_BRIEF_FILE, markdownBrief, "utf8");
|
|
315
|
+
console.log(`✅ Markdown brief generated: ${MARKDOWN_TEST_BRIEF_FILE}`);
|
|
316
|
+
|
|
317
|
+
console.log(`\n📈 Summary:`);
|
|
318
|
+
console.log(` Total prompts evaluated: ${testSummary.totalPrompts}`);
|
|
319
|
+
console.log(` Models tested: ${testSummary.totalModels}`);
|
|
320
|
+
console.log(` Responses with 0% accuracy: ${testSummary.responsesWithZeroAccuracy.length}`);
|
|
321
|
+
|
|
322
|
+
if (baselineCommit) {
|
|
323
|
+
console.log(` Baseline commit: ${baselineCommit}`);
|
|
324
|
+
console.log(` Responses improved vs baseline: ${testSummary.responsesImproved}`);
|
|
325
|
+
console.log(` Responses regressed vs baseline: ${testSummary.responsesRegressed}`);
|
|
326
|
+
}
|
|
327
|
+
} catch (error) {
|
|
328
|
+
console.error("Error generating test summary:", error);
|
|
329
|
+
process.exit(1);
|
|
330
|
+
} finally {
|
|
331
|
+
await storage.close();
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
void generateTestSummary();
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
# Variables necessary for the accuracy test runs
|
|
3
|
+
export MDB_ACCURACY_RUN_ID=$(npx uuid v4)
|
|
4
|
+
|
|
5
|
+
# For providing access tokens for different LLM providers
|
|
6
|
+
# export MDB_OPEN_AI_API_KEY=""
|
|
7
|
+
# export MDB_GEMINI_API_KEY=""
|
|
8
|
+
# export MDB_AZURE_OPEN_AI_API_KEY=""
|
|
9
|
+
# export MDB_AZURE_OPEN_AI_API_URL=""
|
|
10
|
+
|
|
11
|
+
# For providing a mongodb based storage to store accuracy result
|
|
12
|
+
# export MDB_ACCURACY_MDB_URL=""
|
|
13
|
+
# export MDB_ACCURACY_MDB_DB=""
|
|
14
|
+
# export MDB_ACCURACY_MDB_COLLECTION=""
|
|
15
|
+
|
|
16
|
+
# By default we run all the tests under tests/accuracy folder unless a path is
|
|
17
|
+
# specified in the command line. Such as:
|
|
18
|
+
# npm run test:accuracy -- tests/accuracy/some-test.test.ts
|
|
19
|
+
echo "Running accuracy tests with MDB_ACCURACY_RUN_ID '$MDB_ACCURACY_RUN_ID'"
|
|
20
|
+
vitest --config vitest.config.ts --project=accuracy --coverage=false --run "$@"
|
|
21
|
+
|
|
22
|
+
# Preserving the exit code from test run to correctly notify in the CI
|
|
23
|
+
# environments when the tests fail.
|
|
24
|
+
TEST_EXIT_CODE=$?
|
|
25
|
+
|
|
26
|
+
# Each test run submits an accuracy result with the accuracyRunStatus:
|
|
27
|
+
# "in-progress". When all the tests are done and jest exits with an exit code of
|
|
28
|
+
# 0, we can safely mark accuracy run as finished otherwise failed.
|
|
29
|
+
|
|
30
|
+
# This "outside-the-test-status-update" is arising out of the fact that each
|
|
31
|
+
# test suite stores their own accuracy run data in the storage and this setup
|
|
32
|
+
# might lead to data inconsistency when the tests fail. To overcome that each
|
|
33
|
+
# accuracy result entry has a status which by default is "in-progress" and is
|
|
34
|
+
# updated when the tests either pass (all our accuracy tests are supposed to
|
|
35
|
+
# pass unless some errors occurs during the test runs), or fail.
|
|
36
|
+
|
|
37
|
+
# This is necessary when comparing one accuracy run with another as we wouldn't
|
|
38
|
+
# want to compare against an incomplete run.
|
|
39
|
+
export MDB_ACCURACY_RUN_STATUS=$([ $TEST_EXIT_CODE -eq 0 ] && echo "done" || echo "failed")
|
|
40
|
+
npx tsx scripts/accuracy/updateAccuracyRunStatus.ts || echo "Warning: Failed to update accuracy run status to '$MDB_ACCURACY_RUN_STATUS'"
|
|
41
|
+
|
|
42
|
+
# This is optional but we do it anyways to generate a readable summary of report.
|
|
43
|
+
npx tsx scripts/accuracy/generateTestSummary.ts || echo "Warning: Failed to generate test summary HTML report"
|
|
44
|
+
|
|
45
|
+
exit $TEST_EXIT_CODE
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { getAccuracyResultStorage } from "../../tests/accuracy/sdk/accuracyResultStorage/getAccuracyResultStorage.js";
|
|
2
|
+
import { AccuracyRunStatus } from "../../tests/accuracy/sdk/accuracyResultStorage/resultStorage.js";
|
|
3
|
+
import { getCommitSHA } from "../../tests/accuracy/sdk/gitInfo.js";
|
|
4
|
+
|
|
5
|
+
const envAccuracyRunId = process.env.MDB_ACCURACY_RUN_ID;
|
|
6
|
+
const envAccuracyRunStatus = process.env.MDB_ACCURACY_RUN_STATUS;
|
|
7
|
+
const commitSHA = await getCommitSHA();
|
|
8
|
+
|
|
9
|
+
if (
|
|
10
|
+
!envAccuracyRunId ||
|
|
11
|
+
!commitSHA ||
|
|
12
|
+
(envAccuracyRunStatus !== AccuracyRunStatus.Done && envAccuracyRunStatus !== AccuracyRunStatus.Failed)
|
|
13
|
+
) {
|
|
14
|
+
process.exit(1);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
console.time(`Marked accuracy run id - ${envAccuracyRunId} as ${envAccuracyRunStatus} in`);
|
|
18
|
+
const storage = getAccuracyResultStorage();
|
|
19
|
+
await storage.updateRunStatus(commitSHA, envAccuracyRunId, envAccuracyRunStatus);
|
|
20
|
+
await storage.close();
|
|
21
|
+
console.timeEnd(`Marked accuracy run id - ${envAccuracyRunId} as ${envAccuracyRunStatus} in`);
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { ApiClient } from "./apiClient.js";
|
|
2
|
+
import logger, { LogId } from "../logger.js";
|
|
3
|
+
import { ApiClientError } from "./apiClientError.js";
|
|
4
|
+
|
|
5
|
+
export const DEFAULT_ACCESS_LIST_COMMENT = "Added by MongoDB MCP Server to enable tool access";
|
|
6
|
+
|
|
7
|
+
export async function makeCurrentIpAccessListEntry(
|
|
8
|
+
apiClient: ApiClient,
|
|
9
|
+
projectId: string,
|
|
10
|
+
comment: string = DEFAULT_ACCESS_LIST_COMMENT
|
|
11
|
+
) {
|
|
12
|
+
const { currentIpv4Address } = await apiClient.getIpInfo();
|
|
13
|
+
return {
|
|
14
|
+
groupId: projectId,
|
|
15
|
+
ipAddress: currentIpv4Address,
|
|
16
|
+
comment,
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Ensures the current public IP is in the access list for the given Atlas project.
|
|
22
|
+
* If the IP is already present, this is a no-op.
|
|
23
|
+
* @param apiClient The Atlas API client instance
|
|
24
|
+
* @param projectId The Atlas project ID
|
|
25
|
+
*/
|
|
26
|
+
export async function ensureCurrentIpInAccessList(apiClient: ApiClient, projectId: string): Promise<void> {
|
|
27
|
+
const entry = await makeCurrentIpAccessListEntry(apiClient, projectId, DEFAULT_ACCESS_LIST_COMMENT);
|
|
28
|
+
try {
|
|
29
|
+
await apiClient.createProjectIpAccessList({
|
|
30
|
+
params: { path: { groupId: projectId } },
|
|
31
|
+
body: [entry],
|
|
32
|
+
});
|
|
33
|
+
logger.debug(
|
|
34
|
+
LogId.atlasIpAccessListAdded,
|
|
35
|
+
"accessListUtils",
|
|
36
|
+
`IP access list created: ${JSON.stringify(entry)}`
|
|
37
|
+
);
|
|
38
|
+
} catch (err) {
|
|
39
|
+
if (err instanceof ApiClientError && err.response?.status === 409) {
|
|
40
|
+
// 409 Conflict: entry already exists, log info
|
|
41
|
+
logger.debug(
|
|
42
|
+
LogId.atlasIpAccessListAdded,
|
|
43
|
+
"accessListUtils",
|
|
44
|
+
`IP address ${entry.ipAddress} is already present in the access list for project ${projectId}.`
|
|
45
|
+
);
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
logger.warning(
|
|
49
|
+
LogId.atlasIpAccessListAddFailure,
|
|
50
|
+
"accessListUtils",
|
|
51
|
+
`Error adding IP access list: ${err instanceof Error ? err.message : String(err)}`
|
|
52
|
+
);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
@@ -4,7 +4,8 @@ import { AccessToken, ClientCredentials } from "simple-oauth2";
|
|
|
4
4
|
import { ApiClientError } from "./apiClientError.js";
|
|
5
5
|
import { paths, operations } from "./openapi.js";
|
|
6
6
|
import { CommonProperties, TelemetryEvent } from "../../telemetry/types.js";
|
|
7
|
-
import { packageInfo } from "
|
|
7
|
+
import { packageInfo } from "../packageInfo.js";
|
|
8
|
+
import logger, { LogId } from "../logger.js";
|
|
8
9
|
|
|
9
10
|
const ATLAS_API_VERSION = "2025-03-12";
|
|
10
11
|
|
|
@@ -34,9 +35,7 @@ export class ApiClient {
|
|
|
34
35
|
|
|
35
36
|
private getAccessToken = async () => {
|
|
36
37
|
if (this.oauth2Client && (!this.accessToken || this.accessToken.expired())) {
|
|
37
|
-
this.accessToken = await this.oauth2Client.getToken({
|
|
38
|
-
agent: this.options.userAgent,
|
|
39
|
-
});
|
|
38
|
+
this.accessToken = await this.oauth2Client.getToken({});
|
|
40
39
|
}
|
|
41
40
|
return this.accessToken?.token.access_token as string | undefined;
|
|
42
41
|
};
|
|
@@ -49,7 +48,9 @@ export class ApiClient {
|
|
|
49
48
|
|
|
50
49
|
try {
|
|
51
50
|
const accessToken = await this.getAccessToken();
|
|
52
|
-
|
|
51
|
+
if (accessToken) {
|
|
52
|
+
request.headers.set("Authorization", `Bearer ${accessToken}`);
|
|
53
|
+
}
|
|
53
54
|
return request;
|
|
54
55
|
} catch {
|
|
55
56
|
// ignore not availble tokens, API will return 401
|
|
@@ -81,6 +82,12 @@ export class ApiClient {
|
|
|
81
82
|
auth: {
|
|
82
83
|
tokenHost: this.options.baseUrl,
|
|
83
84
|
tokenPath: "/api/oauth/token",
|
|
85
|
+
revokePath: "/api/oauth/revoke",
|
|
86
|
+
},
|
|
87
|
+
http: {
|
|
88
|
+
headers: {
|
|
89
|
+
"User-Agent": this.options.userAgent,
|
|
90
|
+
},
|
|
84
91
|
},
|
|
85
92
|
});
|
|
86
93
|
this.client.use(this.authMiddleware);
|
|
@@ -88,13 +95,25 @@ export class ApiClient {
|
|
|
88
95
|
}
|
|
89
96
|
|
|
90
97
|
public hasCredentials(): boolean {
|
|
91
|
-
return !!
|
|
98
|
+
return !!this.oauth2Client;
|
|
92
99
|
}
|
|
93
100
|
|
|
94
101
|
public async validateAccessToken(): Promise<void> {
|
|
95
102
|
await this.getAccessToken();
|
|
96
103
|
}
|
|
97
104
|
|
|
105
|
+
public async close(): Promise<void> {
|
|
106
|
+
if (this.accessToken) {
|
|
107
|
+
try {
|
|
108
|
+
await this.accessToken.revoke("access_token");
|
|
109
|
+
} catch (error: unknown) {
|
|
110
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
111
|
+
logger.error(LogId.atlasApiRevokeFailure, "apiClient", `Failed to revoke access token: ${err.message}`);
|
|
112
|
+
}
|
|
113
|
+
this.accessToken = undefined;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
98
117
|
public async getIpInfo(): Promise<{
|
|
99
118
|
currentIpv4Address: string;
|
|
100
119
|
}> {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ClusterDescription20240805, FlexClusterDescription20241113 } from "./openapi.js";
|
|
2
2
|
import { ApiClient } from "./apiClient.js";
|
|
3
|
-
import logger, { LogId } from "
|
|
3
|
+
import logger, { LogId } from "../logger.js";
|
|
4
4
|
|
|
5
5
|
export interface Cluster {
|
|
6
6
|
name?: string;
|
|
@@ -17,13 +17,19 @@ export interface UserConfig {
|
|
|
17
17
|
apiBaseUrl: string;
|
|
18
18
|
apiClientId?: string;
|
|
19
19
|
apiClientSecret?: string;
|
|
20
|
-
telemetry
|
|
20
|
+
telemetry: "enabled" | "disabled";
|
|
21
21
|
logPath: string;
|
|
22
22
|
connectionString?: string;
|
|
23
23
|
connectOptions: ConnectOptions;
|
|
24
24
|
disabledTools: Array<string>;
|
|
25
25
|
readOnly?: boolean;
|
|
26
26
|
indexCheck?: boolean;
|
|
27
|
+
transport: "stdio" | "http";
|
|
28
|
+
httpPort: number;
|
|
29
|
+
httpHost: string;
|
|
30
|
+
loggers: Array<"stderr" | "disk" | "mcp">;
|
|
31
|
+
idleTimeoutMs: number;
|
|
32
|
+
notificationTimeoutMs: number;
|
|
27
33
|
}
|
|
28
34
|
|
|
29
35
|
const defaults: UserConfig = {
|
|
@@ -39,6 +45,12 @@ const defaults: UserConfig = {
|
|
|
39
45
|
telemetry: "enabled",
|
|
40
46
|
readOnly: false,
|
|
41
47
|
indexCheck: false,
|
|
48
|
+
transport: "stdio",
|
|
49
|
+
httpPort: 3000,
|
|
50
|
+
httpHost: "127.0.0.1",
|
|
51
|
+
loggers: ["disk", "mcp"],
|
|
52
|
+
idleTimeoutMs: 600000, // 10 minutes
|
|
53
|
+
notificationTimeoutMs: 540000, // 9 minutes
|
|
42
54
|
};
|
|
43
55
|
|
|
44
56
|
export const config = {
|
|
@@ -120,6 +132,6 @@ function SNAKE_CASE_toCamelCase(str: string): string {
|
|
|
120
132
|
// Reads the cli args and parses them into a UserConfig object.
|
|
121
133
|
function getCliConfig() {
|
|
122
134
|
return argv(process.argv.slice(2), {
|
|
123
|
-
array: ["disabledTools"],
|
|
135
|
+
array: ["disabledTools", "loggers"],
|
|
124
136
|
}) as unknown as Partial<UserConfig>;
|
|
125
137
|
}
|
|
@@ -12,11 +12,17 @@ export const LogId = {
|
|
|
12
12
|
serverCloseRequested: mongoLogId(1_000_003),
|
|
13
13
|
serverClosed: mongoLogId(1_000_004),
|
|
14
14
|
serverCloseFailure: mongoLogId(1_000_005),
|
|
15
|
+
serverDuplicateLoggers: mongoLogId(1_000_006),
|
|
15
16
|
|
|
16
17
|
atlasCheckCredentials: mongoLogId(1_001_001),
|
|
17
18
|
atlasDeleteDatabaseUserFailure: mongoLogId(1_001_002),
|
|
18
19
|
atlasConnectFailure: mongoLogId(1_001_003),
|
|
19
20
|
atlasInspectFailure: mongoLogId(1_001_004),
|
|
21
|
+
atlasConnectAttempt: mongoLogId(1_001_005),
|
|
22
|
+
atlasConnectSucceeded: mongoLogId(1_001_006),
|
|
23
|
+
atlasApiRevokeFailure: mongoLogId(1_001_007),
|
|
24
|
+
atlasIpAccessListAdded: mongoLogId(1_001_008),
|
|
25
|
+
atlasIpAccessListAddFailure: mongoLogId(1_001_009),
|
|
20
26
|
|
|
21
27
|
telemetryDisabled: mongoLogId(1_002_001),
|
|
22
28
|
telemetryEmitFailure: mongoLogId(1_002_002),
|
|
@@ -34,9 +40,16 @@ export const LogId = {
|
|
|
34
40
|
mongodbDisconnectFailure: mongoLogId(1_004_002),
|
|
35
41
|
|
|
36
42
|
toolUpdateFailure: mongoLogId(1_005_001),
|
|
43
|
+
|
|
44
|
+
streamableHttpTransportStarted: mongoLogId(1_006_001),
|
|
45
|
+
streamableHttpTransportSessionCloseFailure: mongoLogId(1_006_002),
|
|
46
|
+
streamableHttpTransportSessionCloseNotification: mongoLogId(1_006_003),
|
|
47
|
+
streamableHttpTransportSessionCloseNotificationFailure: mongoLogId(1_006_004),
|
|
48
|
+
streamableHttpTransportRequestFailure: mongoLogId(1_006_005),
|
|
49
|
+
streamableHttpTransportCloseFailure: mongoLogId(1_006_006),
|
|
37
50
|
} as const;
|
|
38
51
|
|
|
39
|
-
abstract class LoggerBase {
|
|
52
|
+
export abstract class LoggerBase {
|
|
40
53
|
abstract log(level: LogLevel, id: MongoLogId, context: string, message: string): void;
|
|
41
54
|
|
|
42
55
|
info(id: MongoLogId, context: string, message: string): void {
|
|
@@ -71,14 +84,14 @@ abstract class LoggerBase {
|
|
|
71
84
|
}
|
|
72
85
|
}
|
|
73
86
|
|
|
74
|
-
class ConsoleLogger extends LoggerBase {
|
|
87
|
+
export class ConsoleLogger extends LoggerBase {
|
|
75
88
|
log(level: LogLevel, id: MongoLogId, context: string, message: string): void {
|
|
76
89
|
message = redact(message);
|
|
77
|
-
console.error(`[${level.toUpperCase()}] ${id.__value} - ${context}: ${message}`);
|
|
90
|
+
console.error(`[${level.toUpperCase()}] ${id.__value} - ${context}: ${message} (${process.pid})`);
|
|
78
91
|
}
|
|
79
92
|
}
|
|
80
93
|
|
|
81
|
-
class DiskLogger extends LoggerBase {
|
|
94
|
+
export class DiskLogger extends LoggerBase {
|
|
82
95
|
private constructor(private logWriter: MongoLogWriter) {
|
|
83
96
|
super();
|
|
84
97
|
}
|
|
@@ -130,7 +143,7 @@ class DiskLogger extends LoggerBase {
|
|
|
130
143
|
}
|
|
131
144
|
}
|
|
132
145
|
|
|
133
|
-
class McpLogger extends LoggerBase {
|
|
146
|
+
export class McpLogger extends LoggerBase {
|
|
134
147
|
constructor(private server: McpServer) {
|
|
135
148
|
super();
|
|
136
149
|
}
|
|
@@ -149,18 +162,12 @@ class McpLogger extends LoggerBase {
|
|
|
149
162
|
}
|
|
150
163
|
|
|
151
164
|
class CompositeLogger extends LoggerBase {
|
|
152
|
-
private loggers: LoggerBase[];
|
|
165
|
+
private loggers: LoggerBase[] = [];
|
|
153
166
|
|
|
154
167
|
constructor(...loggers: LoggerBase[]) {
|
|
155
168
|
super();
|
|
156
169
|
|
|
157
|
-
|
|
158
|
-
// default to ConsoleLogger
|
|
159
|
-
this.loggers = [new ConsoleLogger()];
|
|
160
|
-
return;
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
this.loggers = [...loggers];
|
|
170
|
+
this.setLoggers(...loggers);
|
|
164
171
|
}
|
|
165
172
|
|
|
166
173
|
setLoggers(...loggers: LoggerBase[]): void {
|
|
@@ -177,14 +184,5 @@ class CompositeLogger extends LoggerBase {
|
|
|
177
184
|
}
|
|
178
185
|
}
|
|
179
186
|
|
|
180
|
-
const logger = new CompositeLogger();
|
|
187
|
+
const logger = new CompositeLogger(new ConsoleLogger());
|
|
181
188
|
export default logger;
|
|
182
|
-
|
|
183
|
-
export async function initializeLogger(server: McpServer, logPath: string): Promise<LoggerBase> {
|
|
184
|
-
const diskLogger = await DiskLogger.fromPath(logPath);
|
|
185
|
-
const mcpLogger = new McpLogger(server);
|
|
186
|
-
|
|
187
|
-
logger.setLoggers(mcpLogger, diskLogger);
|
|
188
|
-
|
|
189
|
-
return logger;
|
|
190
|
-
}
|