mongodb-mcp-server 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (270) hide show
  1. package/.github/pull_request_template.md +5 -0
  2. package/.github/workflows/accuracy-tests.yml +55 -0
  3. package/.github/workflows/check.yml +1 -1
  4. package/.github/workflows/code_health.yaml +4 -4
  5. package/.github/workflows/code_health_fork.yaml +0 -14
  6. package/.github/workflows/dependabot_pr.yaml +26 -0
  7. package/.github/workflows/docker.yaml +1 -1
  8. package/.github/workflows/jira-issue.yml +72 -0
  9. package/.smithery/smithery.yaml +10 -0
  10. package/.vscode/extensions.json +1 -1
  11. package/.vscode/launch.json +11 -1
  12. package/.vscode/settings.json +1 -11
  13. package/Dockerfile +1 -0
  14. package/README.md +132 -31
  15. package/dist/common/atlas/accessListUtils.js +36 -0
  16. package/dist/common/atlas/accessListUtils.js.map +1 -0
  17. package/dist/common/atlas/apiClient.js +25 -6
  18. package/dist/common/atlas/apiClient.js.map +1 -1
  19. package/dist/common/atlas/cluster.js +1 -1
  20. package/dist/common/atlas/cluster.js.map +1 -1
  21. package/dist/{config.js → common/config.js} +8 -1
  22. package/dist/common/config.js.map +1 -0
  23. package/dist/{errors.js → common/errors.js} +1 -0
  24. package/dist/common/errors.js.map +1 -0
  25. package/dist/{logger.js → common/logger.js} +20 -19
  26. package/dist/common/logger.js.map +1 -0
  27. package/dist/common/managedTimeout.js +20 -0
  28. package/dist/common/managedTimeout.js.map +1 -0
  29. package/dist/common/packageInfo.js.map +1 -0
  30. package/dist/{session.js → common/session.js} +20 -21
  31. package/dist/common/session.js.map +1 -0
  32. package/dist/common/sessionStore.js +73 -0
  33. package/dist/common/sessionStore.js.map +1 -0
  34. package/dist/helpers/container.js +28 -0
  35. package/dist/helpers/container.js.map +1 -0
  36. package/dist/helpers/generatePassword.js.map +1 -0
  37. package/dist/helpers/indexCheck.js +63 -0
  38. package/dist/helpers/indexCheck.js.map +1 -0
  39. package/dist/index.js +30 -37
  40. package/dist/index.js.map +1 -1
  41. package/dist/server.js +44 -7
  42. package/dist/server.js.map +1 -1
  43. package/dist/telemetry/constants.js +1 -1
  44. package/dist/telemetry/constants.js.map +1 -1
  45. package/dist/telemetry/telemetry.js +86 -116
  46. package/dist/telemetry/telemetry.js.map +1 -1
  47. package/dist/tools/atlas/atlasTool.js +3 -3
  48. package/dist/tools/atlas/atlasTool.js.map +1 -1
  49. package/dist/tools/atlas/connect/connectCluster.js +198 -0
  50. package/dist/tools/atlas/connect/connectCluster.js.map +1 -0
  51. package/dist/tools/atlas/create/createAccessList.js +9 -10
  52. package/dist/tools/atlas/create/createAccessList.js.map +1 -1
  53. package/dist/tools/atlas/create/createDBUser.js +3 -1
  54. package/dist/tools/atlas/create/createDBUser.js.map +1 -1
  55. package/dist/tools/atlas/create/createFreeCluster.js +2 -0
  56. package/dist/tools/atlas/create/createFreeCluster.js.map +1 -1
  57. package/dist/tools/atlas/create/createProject.js.map +1 -1
  58. package/dist/tools/atlas/read/inspectAccessList.js.map +1 -1
  59. package/dist/tools/atlas/read/inspectCluster.js.map +1 -1
  60. package/dist/tools/atlas/read/listAlerts.js.map +1 -1
  61. package/dist/tools/atlas/read/listClusters.js.map +1 -1
  62. package/dist/tools/atlas/read/listDBUsers.js.map +1 -1
  63. package/dist/tools/atlas/read/listOrgs.js.map +1 -1
  64. package/dist/tools/atlas/read/listProjects.js.map +1 -1
  65. package/dist/tools/atlas/tools.js +1 -1
  66. package/dist/tools/atlas/tools.js.map +1 -1
  67. package/dist/tools/mongodb/{metadata → connect}/connect.js +7 -4
  68. package/dist/tools/mongodb/connect/connect.js.map +1 -0
  69. package/dist/tools/mongodb/create/createCollection.js.map +1 -1
  70. package/dist/tools/mongodb/create/createIndex.js +1 -1
  71. package/dist/tools/mongodb/create/createIndex.js.map +1 -1
  72. package/dist/tools/mongodb/create/insertMany.js +1 -1
  73. package/dist/tools/mongodb/create/insertMany.js.map +1 -1
  74. package/dist/tools/mongodb/delete/deleteMany.js +20 -1
  75. package/dist/tools/mongodb/delete/deleteMany.js.map +1 -1
  76. package/dist/tools/mongodb/delete/dropCollection.js.map +1 -1
  77. package/dist/tools/mongodb/delete/dropDatabase.js.map +1 -1
  78. package/dist/tools/mongodb/metadata/collectionSchema.js.map +1 -1
  79. package/dist/tools/mongodb/metadata/collectionStorageSize.js.map +1 -1
  80. package/dist/tools/mongodb/metadata/dbStats.js.map +1 -1
  81. package/dist/tools/mongodb/metadata/explain.js +2 -2
  82. package/dist/tools/mongodb/metadata/explain.js.map +1 -1
  83. package/dist/tools/mongodb/metadata/listCollections.js.map +1 -1
  84. package/dist/tools/mongodb/metadata/listDatabases.js.map +1 -1
  85. package/dist/tools/mongodb/metadata/logs.js.map +1 -1
  86. package/dist/tools/mongodb/mongodbTool.js +47 -10
  87. package/dist/tools/mongodb/mongodbTool.js.map +1 -1
  88. package/dist/tools/mongodb/read/aggregate.js +10 -1
  89. package/dist/tools/mongodb/read/aggregate.js.map +1 -1
  90. package/dist/tools/mongodb/read/collectionIndexes.js.map +1 -1
  91. package/dist/tools/mongodb/read/count.js +15 -1
  92. package/dist/tools/mongodb/read/count.js.map +1 -1
  93. package/dist/tools/mongodb/read/find.js +14 -4
  94. package/dist/tools/mongodb/read/find.js.map +1 -1
  95. package/dist/tools/mongodb/tools.js +1 -1
  96. package/dist/tools/mongodb/tools.js.map +1 -1
  97. package/dist/tools/mongodb/update/renameCollection.js.map +1 -1
  98. package/dist/tools/mongodb/update/updateMany.js +24 -2
  99. package/dist/tools/mongodb/update/updateMany.js.map +1 -1
  100. package/dist/tools/tool.js +12 -9
  101. package/dist/tools/tool.js.map +1 -1
  102. package/dist/transports/base.js +26 -0
  103. package/dist/transports/base.js.map +1 -0
  104. package/dist/{helpers/EJsonTransport.js → transports/stdio.js} +24 -2
  105. package/dist/transports/stdio.js.map +1 -0
  106. package/dist/transports/streamableHttp.js +140 -0
  107. package/dist/transports/streamableHttp.js.map +1 -0
  108. package/eslint.config.js +13 -4
  109. package/package.json +43 -33
  110. package/resources/test-summary-template.html +415 -0
  111. package/scripts/accuracy/generateTestSummary.ts +335 -0
  112. package/scripts/accuracy/runAccuracyTests.sh +45 -0
  113. package/scripts/accuracy/updateAccuracyRunStatus.ts +21 -0
  114. package/src/common/atlas/accessListUtils.ts +54 -0
  115. package/src/common/atlas/apiClient.ts +25 -6
  116. package/src/common/atlas/cluster.ts +1 -1
  117. package/src/{config.ts → common/config.ts} +16 -2
  118. package/src/{errors.ts → common/errors.ts} +1 -0
  119. package/src/{logger.ts → common/logger.ts} +21 -24
  120. package/src/common/managedTimeout.ts +27 -0
  121. package/src/{session.ts → common/session.ts} +24 -26
  122. package/src/common/sessionStore.ts +111 -0
  123. package/src/helpers/container.ts +35 -0
  124. package/src/helpers/indexCheck.ts +83 -0
  125. package/src/index.ts +30 -40
  126. package/src/server.ts +55 -11
  127. package/src/telemetry/constants.ts +1 -1
  128. package/src/telemetry/telemetry.ts +109 -153
  129. package/src/telemetry/types.ts +2 -1
  130. package/src/tools/atlas/atlasTool.ts +4 -4
  131. package/src/tools/atlas/connect/connectCluster.ts +259 -0
  132. package/src/tools/atlas/create/createAccessList.ts +15 -13
  133. package/src/tools/atlas/create/createDBUser.ts +5 -3
  134. package/src/tools/atlas/create/createFreeCluster.ts +4 -2
  135. package/src/tools/atlas/create/createProject.ts +2 -2
  136. package/src/tools/atlas/read/inspectAccessList.ts +2 -2
  137. package/src/tools/atlas/read/inspectCluster.ts +2 -2
  138. package/src/tools/atlas/read/listAlerts.ts +2 -2
  139. package/src/tools/atlas/read/listClusters.ts +2 -2
  140. package/src/tools/atlas/read/listDBUsers.ts +2 -2
  141. package/src/tools/atlas/read/listOrgs.ts +2 -2
  142. package/src/tools/atlas/read/listProjects.ts +2 -2
  143. package/src/tools/atlas/tools.ts +1 -1
  144. package/src/tools/mongodb/{metadata → connect}/connect.ts +12 -9
  145. package/src/tools/mongodb/create/createCollection.ts +2 -2
  146. package/src/tools/mongodb/create/createIndex.ts +3 -3
  147. package/src/tools/mongodb/create/insertMany.ts +3 -3
  148. package/src/tools/mongodb/delete/deleteMany.ts +24 -3
  149. package/src/tools/mongodb/delete/dropCollection.ts +2 -2
  150. package/src/tools/mongodb/delete/dropDatabase.ts +2 -2
  151. package/src/tools/mongodb/metadata/collectionSchema.ts +2 -2
  152. package/src/tools/mongodb/metadata/collectionStorageSize.ts +2 -2
  153. package/src/tools/mongodb/metadata/dbStats.ts +2 -2
  154. package/src/tools/mongodb/metadata/explain.ts +4 -4
  155. package/src/tools/mongodb/metadata/listCollections.ts +2 -2
  156. package/src/tools/mongodb/metadata/listDatabases.ts +2 -2
  157. package/src/tools/mongodb/metadata/logs.ts +2 -2
  158. package/src/tools/mongodb/mongodbTool.ts +60 -14
  159. package/src/tools/mongodb/read/aggregate.ts +14 -3
  160. package/src/tools/mongodb/read/collectionIndexes.ts +2 -2
  161. package/src/tools/mongodb/read/count.ts +19 -3
  162. package/src/tools/mongodb/read/find.ts +20 -6
  163. package/src/tools/mongodb/tools.ts +1 -1
  164. package/src/tools/mongodb/update/renameCollection.ts +2 -2
  165. package/src/tools/mongodb/update/updateMany.ts +28 -4
  166. package/src/tools/tool.ts +23 -18
  167. package/src/transports/base.ts +34 -0
  168. package/src/{helpers/EJsonTransport.ts → transports/stdio.ts} +30 -1
  169. package/src/transports/streamableHttp.ts +178 -0
  170. package/tests/accuracy/aggregate.test.ts +27 -0
  171. package/tests/accuracy/collectionIndexes.test.ts +40 -0
  172. package/tests/accuracy/collectionSchema.test.ts +28 -0
  173. package/tests/accuracy/collectionStorageSize.test.ts +41 -0
  174. package/tests/accuracy/count.test.ts +44 -0
  175. package/tests/accuracy/createCollection.test.ts +46 -0
  176. package/tests/accuracy/createIndex.test.ts +37 -0
  177. package/tests/accuracy/dbStats.test.ts +15 -0
  178. package/tests/accuracy/deleteMany.test.ts +44 -0
  179. package/tests/accuracy/dropCollection.test.ts +74 -0
  180. package/tests/accuracy/dropDatabase.test.ts +41 -0
  181. package/tests/accuracy/explain.test.ts +73 -0
  182. package/tests/accuracy/find.test.ts +114 -0
  183. package/tests/accuracy/insertMany.test.ts +48 -0
  184. package/tests/accuracy/listCollections.test.ts +60 -0
  185. package/tests/accuracy/listDatabases.test.ts +31 -0
  186. package/tests/accuracy/logs.test.ts +28 -0
  187. package/tests/accuracy/renameCollection.test.ts +31 -0
  188. package/tests/accuracy/sdk/accuracyResultStorage/diskStorage.ts +189 -0
  189. package/tests/accuracy/sdk/accuracyResultStorage/getAccuracyResultStorage.ts +11 -0
  190. package/tests/accuracy/sdk/accuracyResultStorage/mongodbStorage.ts +151 -0
  191. package/tests/accuracy/sdk/accuracyResultStorage/resultStorage.ts +117 -0
  192. package/tests/accuracy/sdk/accuracyScorer.ts +93 -0
  193. package/tests/accuracy/sdk/accuracyTestingClient.ts +94 -0
  194. package/tests/accuracy/sdk/agent.ts +56 -0
  195. package/tests/accuracy/sdk/constants.ts +26 -0
  196. package/tests/accuracy/sdk/describeAccuracyTests.ts +126 -0
  197. package/tests/accuracy/sdk/gitInfo.ts +7 -0
  198. package/tests/accuracy/sdk/matcher.ts +193 -0
  199. package/tests/accuracy/sdk/models.ts +95 -0
  200. package/tests/accuracy/test-data-dumps/comics.books.json +417 -0
  201. package/tests/accuracy/test-data-dumps/comics.characters.json +402 -0
  202. package/tests/accuracy/test-data-dumps/mflix.movies.json +496 -0
  203. package/tests/accuracy/test-data-dumps/mflix.shows.json +572 -0
  204. package/tests/accuracy/updateMany.test.ts +42 -0
  205. package/tests/integration/helpers.ts +9 -9
  206. package/tests/integration/indexCheck.test.ts +464 -0
  207. package/tests/integration/server.test.ts +6 -4
  208. package/tests/integration/telemetry.test.ts +29 -0
  209. package/tests/integration/tools/atlas/accessLists.test.ts +22 -2
  210. package/tests/integration/tools/atlas/alerts.test.ts +3 -2
  211. package/tests/integration/tools/atlas/atlasHelpers.ts +3 -0
  212. package/tests/integration/tools/atlas/clusters.test.ts +68 -16
  213. package/tests/integration/tools/atlas/dbUsers.test.ts +14 -1
  214. package/tests/integration/tools/atlas/orgs.test.ts +2 -1
  215. package/tests/integration/tools/atlas/projects.test.ts +4 -3
  216. package/tests/integration/tools/mongodb/{metadata → connect}/connect.test.ts +34 -3
  217. package/tests/integration/tools/mongodb/create/createCollection.test.ts +1 -0
  218. package/tests/integration/tools/mongodb/create/createIndex.test.ts +1 -0
  219. package/tests/integration/tools/mongodb/create/insertMany.test.ts +1 -0
  220. package/tests/integration/tools/mongodb/delete/deleteMany.test.ts +1 -0
  221. package/tests/integration/tools/mongodb/delete/dropCollection.test.ts +1 -1
  222. package/tests/integration/tools/mongodb/delete/dropDatabase.test.ts +1 -0
  223. package/tests/integration/tools/mongodb/metadata/collectionSchema.test.ts +1 -0
  224. package/tests/integration/tools/mongodb/metadata/collectionStorageSize.test.ts +1 -0
  225. package/tests/integration/tools/mongodb/metadata/dbStats.test.ts +1 -0
  226. package/tests/integration/tools/mongodb/metadata/explain.test.ts +1 -0
  227. package/tests/integration/tools/mongodb/metadata/listCollections.test.ts +1 -0
  228. package/tests/integration/tools/mongodb/metadata/listDatabases.test.ts +3 -2
  229. package/tests/integration/tools/mongodb/metadata/logs.test.ts +1 -0
  230. package/tests/integration/tools/mongodb/mongodbHelpers.ts +67 -2
  231. package/tests/integration/tools/mongodb/read/aggregate.test.ts +2 -1
  232. package/tests/integration/tools/mongodb/read/collectionIndexes.test.ts +1 -0
  233. package/tests/integration/tools/mongodb/read/count.test.ts +1 -0
  234. package/tests/integration/tools/mongodb/read/find.test.ts +2 -1
  235. package/tests/integration/tools/mongodb/update/renameCollection.test.ts +1 -0
  236. package/tests/integration/tools/mongodb/update/updateMany.test.ts +1 -0
  237. package/tests/integration/transports/stdio.test.ts +40 -0
  238. package/tests/integration/transports/streamableHttp.test.ts +56 -0
  239. package/tests/matchers/toIncludeSameMembers.test.ts +59 -0
  240. package/tests/matchers/toIncludeSameMembers.ts +12 -0
  241. package/tests/setup.ts +7 -0
  242. package/tests/unit/accessListUtils.test.ts +39 -0
  243. package/tests/unit/accuracyScorer.test.ts +390 -0
  244. package/tests/unit/{apiClient.test.ts → common/apiClient.test.ts} +15 -15
  245. package/tests/unit/common/managedTimeout.test.ts +67 -0
  246. package/tests/unit/{session.test.ts → common/session.test.ts} +7 -12
  247. package/tests/unit/helpers/indexCheck.test.ts +150 -0
  248. package/tests/unit/telemetry.test.ts +99 -137
  249. package/tests/unit/{EJsonTransport.test.ts → transports/stdio.test.ts} +4 -4
  250. package/tests/vitest.d.ts +11 -0
  251. package/tsconfig.json +0 -1
  252. package/{tsconfig.jest.json → tsconfig.test.json} +1 -2
  253. package/vitest.config.ts +41 -0
  254. package/dist/common/atlas/generatePassword.js.map +0 -1
  255. package/dist/config.js.map +0 -1
  256. package/dist/errors.js.map +0 -1
  257. package/dist/helpers/EJsonTransport.js.map +0 -1
  258. package/dist/helpers/packageInfo.js.map +0 -1
  259. package/dist/logger.js.map +0 -1
  260. package/dist/session.js.map +0 -1
  261. package/dist/tools/atlas/metadata/connectCluster.js +0 -100
  262. package/dist/tools/atlas/metadata/connectCluster.js.map +0 -1
  263. package/dist/tools/mongodb/metadata/connect.js.map +0 -1
  264. package/global.d.ts +0 -1
  265. package/jest.config.cjs +0 -22
  266. package/src/tools/atlas/metadata/connectCluster.ts +0 -121
  267. /package/dist/{helpers → common}/packageInfo.js +0 -0
  268. /package/dist/{common/atlas → helpers}/generatePassword.js +0 -0
  269. /package/src/{helpers → common}/packageInfo.ts +0 -0
  270. /package/src/{common/atlas → helpers}/generatePassword.ts +0 -0
@@ -0,0 +1,56 @@
1
+ import { StreamableHttpRunner } from "../../../src/transports/streamableHttp.js";
2
+ import { Client } from "@modelcontextprotocol/sdk/client/index.js";
3
+ import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
4
+ import { describe, expect, it, beforeAll, afterAll } from "vitest";
5
+ import { config } from "../../../src/common/config.js";
6
+
7
+ describe("StreamableHttpRunner", () => {
8
+ let runner: StreamableHttpRunner;
9
+ let oldTelemetry: "enabled" | "disabled";
10
+ let oldLoggers: ("stderr" | "disk" | "mcp")[];
11
+
12
+ beforeAll(async () => {
13
+ oldTelemetry = config.telemetry;
14
+ oldLoggers = config.loggers;
15
+ config.telemetry = "disabled";
16
+ config.loggers = ["stderr"];
17
+ runner = new StreamableHttpRunner(config);
18
+ await runner.start();
19
+ });
20
+
21
+ afterAll(async () => {
22
+ await runner.close();
23
+ config.telemetry = oldTelemetry;
24
+ config.loggers = oldLoggers;
25
+ });
26
+
27
+ describe("client connects successfully", () => {
28
+ let client: Client;
29
+ let transport: StreamableHTTPClientTransport;
30
+ beforeAll(async () => {
31
+ transport = new StreamableHTTPClientTransport(new URL("http://127.0.0.1:3000/mcp"));
32
+
33
+ client = new Client({
34
+ name: "test",
35
+ version: "0.0.0",
36
+ });
37
+ await client.connect(transport);
38
+ });
39
+
40
+ afterAll(async () => {
41
+ await client.close();
42
+ await transport.close();
43
+ });
44
+
45
+ it("handles requests and sends responses", async () => {
46
+ const response = await client.listTools();
47
+ expect(response).toBeDefined();
48
+ expect(response.tools).toBeDefined();
49
+ expect(response.tools.length).toBeGreaterThan(0);
50
+
51
+ const sortedTools = response.tools.sort((a, b) => a.name.localeCompare(b.name));
52
+ expect(sortedTools[0]?.name).toBe("aggregate");
53
+ expect(sortedTools[0]?.description).toBe("Run an aggregation against a MongoDB collection");
54
+ });
55
+ });
56
+ });
@@ -0,0 +1,59 @@
1
+ import { describe, expect, it } from "vitest";
2
+
3
+ describe("toIncludeSameMembers matcher", () => {
4
+ it("should pass when arrays contain the same elements in different order", () => {
5
+ const array1 = [1, 2, 3];
6
+ const array2 = [3, 1, 2];
7
+
8
+ expect(array1).toIncludeSameMembers(array2);
9
+ });
10
+
11
+ it("should pass when arrays contain the same elements in same order", () => {
12
+ const array1 = [1, 2, 3];
13
+ const array2 = [1, 2, 3];
14
+
15
+ expect(array1).toIncludeSameMembers(array2);
16
+ });
17
+
18
+ it("should fail when arrays have different lengths", () => {
19
+ const array1 = [1, 2, 3];
20
+ const array2 = [1, 2];
21
+
22
+ expect(() => expect(array1).toIncludeSameMembers(array2)).toThrow();
23
+ });
24
+
25
+ it("should fail when arrays contain different elements", () => {
26
+ const array1 = [1, 2, 3];
27
+ const array2 = [4, 5, 6];
28
+
29
+ expect(() => expect(array1).toIncludeSameMembers(array2)).toThrow();
30
+ });
31
+
32
+ it("should work with string arrays", () => {
33
+ const array1 = ["apple", "banana", "cherry"];
34
+ const array2 = ["cherry", "apple", "banana"];
35
+
36
+ expect(array1).toIncludeSameMembers(array2);
37
+ });
38
+
39
+ it("should work with object arrays", () => {
40
+ const array1 = [{ name: "Alice" }, { name: "Bob" }];
41
+ const array2 = [{ name: "Bob" }, { name: "Alice" }];
42
+
43
+ expect(array1).toIncludeSameMembers(array2);
44
+ });
45
+
46
+ it("should work with mixed type arrays", () => {
47
+ const array1 = [1, "hello", { key: "value" }];
48
+ const array2 = [{ key: "value" }, 1, "hello"];
49
+
50
+ expect(array1).toIncludeSameMembers(array2);
51
+ });
52
+
53
+ it("should work with empty arrays", () => {
54
+ const array1: unknown[] = [];
55
+ const array2: unknown[] = [];
56
+
57
+ expect(array1).toIncludeSameMembers(array2);
58
+ });
59
+ });
@@ -0,0 +1,12 @@
1
+ import { expect } from "vitest";
2
+
3
+ export function toIncludeSameMembers<T>(actual: T[], expected: T[]): { pass: boolean; message: () => string } {
4
+ expect(actual).toEqual(expect.arrayContaining(expected as unknown[]));
5
+ expect(expected).toEqual(expect.arrayContaining(actual as unknown[]));
6
+
7
+ return {
8
+ pass: true,
9
+ message: () =>
10
+ `Expected arrays to include the same members.\nExpected: ${JSON.stringify(expected)}\nReceived: ${JSON.stringify(actual)}`,
11
+ };
12
+ }
package/tests/setup.ts ADDED
@@ -0,0 +1,7 @@
1
+ import { expect } from "vitest";
2
+ import { toIncludeSameMembers } from "./matchers/toIncludeSameMembers.js";
3
+
4
+ // Extend vitest's expect with custom matchers
5
+ expect.extend({
6
+ toIncludeSameMembers,
7
+ });
@@ -0,0 +1,39 @@
1
+ import { describe, it, expect, vi } from "vitest";
2
+ import { ApiClient } from "../../src/common/atlas/apiClient.js";
3
+ import { ensureCurrentIpInAccessList, DEFAULT_ACCESS_LIST_COMMENT } from "../../src/common/atlas/accessListUtils.js";
4
+ import { ApiClientError } from "../../src/common/atlas/apiClientError.js";
5
+
6
+ describe("accessListUtils", () => {
7
+ it("should add the current IP to the access list", async () => {
8
+ const apiClient = {
9
+ getIpInfo: vi.fn().mockResolvedValue({ currentIpv4Address: "127.0.0.1" } as never),
10
+ createProjectIpAccessList: vi.fn().mockResolvedValue(undefined as never),
11
+ } as unknown as ApiClient;
12
+ await ensureCurrentIpInAccessList(apiClient, "projectId");
13
+ // eslint-disable-next-line @typescript-eslint/unbound-method
14
+ expect(apiClient.createProjectIpAccessList).toHaveBeenCalledWith({
15
+ params: { path: { groupId: "projectId" } },
16
+ body: [{ groupId: "projectId", ipAddress: "127.0.0.1", comment: DEFAULT_ACCESS_LIST_COMMENT }],
17
+ });
18
+ });
19
+
20
+ it("should not fail if the current IP is already in the access list", async () => {
21
+ const apiClient = {
22
+ getIpInfo: vi.fn().mockResolvedValue({ currentIpv4Address: "127.0.0.1" } as never),
23
+ createProjectIpAccessList: vi
24
+ .fn()
25
+ .mockRejectedValue(
26
+ ApiClientError.fromError(
27
+ { status: 409, statusText: "Conflict" } as Response,
28
+ { message: "Conflict" } as never
29
+ ) as never
30
+ ),
31
+ } as unknown as ApiClient;
32
+ await ensureCurrentIpInAccessList(apiClient, "projectId");
33
+ // eslint-disable-next-line @typescript-eslint/unbound-method
34
+ expect(apiClient.createProjectIpAccessList).toHaveBeenCalledWith({
35
+ params: { path: { groupId: "projectId" } },
36
+ body: [{ groupId: "projectId", ipAddress: "127.0.0.1", comment: DEFAULT_ACCESS_LIST_COMMENT }],
37
+ });
38
+ });
39
+ });
@@ -0,0 +1,390 @@
1
+ import { describe, expect, it } from "vitest";
2
+ import { calculateToolCallingAccuracy } from "../accuracy/sdk/accuracyScorer.js";
3
+ import { ExpectedToolCall, LLMToolCall } from "../accuracy/sdk/accuracyResultStorage/resultStorage.js";
4
+ import { Matcher } from "../accuracy/sdk/matcher.js";
5
+
6
+ describe("calculateToolCallingAccuracy", () => {
7
+ describe("edge cases", () => {
8
+ it("should return 1 when both expected and actual are empty", () => {
9
+ const result = calculateToolCallingAccuracy([], []);
10
+ expect(result).toBe(1);
11
+ });
12
+
13
+ it("should return 0.75 when expected is empty but actual has tool calls", () => {
14
+ const actualToolCalls: LLMToolCall[] = [{ toolCallId: "1", toolName: "find", parameters: { db: "test" } }];
15
+ const result = calculateToolCallingAccuracy([], actualToolCalls);
16
+ expect(result).toBe(0.75);
17
+ });
18
+
19
+ it("should return 0 when expected has tool calls but actual is empty", () => {
20
+ const expectedToolCalls: ExpectedToolCall[] = [{ toolName: "find", parameters: { db: "test" } }];
21
+ const result = calculateToolCallingAccuracy(expectedToolCalls, []);
22
+ expect(result).toBe(0);
23
+ });
24
+ });
25
+
26
+ describe("perfect matches", () => {
27
+ it("should return 1 for exact match with nested parameters", () => {
28
+ const expected: ExpectedToolCall[] = [
29
+ {
30
+ toolName: "find",
31
+ parameters: { db: "test", collection: "users", filter: { age: { $gte: 18 }, status: "active" } },
32
+ },
33
+ ];
34
+ const actual: LLMToolCall[] = [
35
+ {
36
+ toolCallId: "1",
37
+ toolName: "find",
38
+ parameters: { db: "test", collection: "users", filter: { age: { $gte: 18 }, status: "active" } },
39
+ },
40
+ ];
41
+ const result = calculateToolCallingAccuracy(expected, actual);
42
+ expect(result).toBe(1);
43
+ });
44
+
45
+ it("should return 1 for exact match with multiple diverse tool calls", () => {
46
+ const expected: ExpectedToolCall[] = [
47
+ { toolName: "find", parameters: { db: "test", collection: "users", filter: { status: "active" } } },
48
+ {
49
+ toolName: "aggregate",
50
+ parameters: { db: "test", collection: "orders", pipeline: [{ $match: { total: { $gt: 100 } } }] },
51
+ },
52
+ { toolName: "count", parameters: { db: "test", collection: "products" } },
53
+ ];
54
+ const actual: LLMToolCall[] = [
55
+ {
56
+ toolCallId: "1",
57
+ toolName: "find",
58
+ parameters: { db: "test", collection: "users", filter: { status: "active" } },
59
+ },
60
+ {
61
+ toolCallId: "2",
62
+ toolName: "aggregate",
63
+ parameters: { db: "test", collection: "orders", pipeline: [{ $match: { total: { $gt: 100 } } }] },
64
+ },
65
+ { toolCallId: "3", toolName: "count", parameters: { db: "test", collection: "products" } },
66
+ ];
67
+ const result = calculateToolCallingAccuracy(expected, actual);
68
+ expect(result).toBe(1);
69
+ });
70
+ });
71
+
72
+ describe("additional parameters", () => {
73
+ it("should return 0 when tool call has additional nested parameters (default behavior)", () => {
74
+ const expected: ExpectedToolCall[] = [
75
+ { toolName: "find", parameters: { db: "test", collection: "users", filter: { status: "active" } } },
76
+ ];
77
+ const actual: LLMToolCall[] = [
78
+ {
79
+ toolCallId: "1",
80
+ toolName: "find",
81
+ parameters: {
82
+ db: "test",
83
+ collection: "users",
84
+ filter: { status: "active", age: { $gte: 18 } },
85
+ limit: 10,
86
+ },
87
+ },
88
+ ];
89
+ const result = calculateToolCallingAccuracy(expected, actual);
90
+ expect(result).toBe(0);
91
+ });
92
+
93
+ it("should return 1 when expected has no filter but actual has empty filter", () => {
94
+ const expected: ExpectedToolCall[] = [
95
+ {
96
+ toolName: "find",
97
+ parameters: {
98
+ database: "mflix",
99
+ collection: "movies",
100
+ filter: Matcher.emptyObjectOrUndefined,
101
+ },
102
+ },
103
+ ];
104
+ const actual: LLMToolCall[] = [
105
+ {
106
+ toolCallId: "1",
107
+ toolName: "find",
108
+ parameters: {
109
+ database: "mflix",
110
+ collection: "movies",
111
+ filter: {},
112
+ },
113
+ },
114
+ ];
115
+ const result = calculateToolCallingAccuracy(expected, actual);
116
+ expect(result).toBe(1);
117
+ });
118
+
119
+ it("should return 1 when expected has no filter and actual has no filter", () => {
120
+ const expected: ExpectedToolCall[] = [
121
+ {
122
+ toolName: "find",
123
+ parameters: {
124
+ database: "mflix",
125
+ collection: "movies",
126
+ filter: Matcher.emptyObjectOrUndefined,
127
+ },
128
+ },
129
+ ];
130
+ const actual: LLMToolCall[] = [
131
+ {
132
+ toolCallId: "1",
133
+ toolName: "find",
134
+ parameters: {
135
+ database: "mflix",
136
+ collection: "movies",
137
+ },
138
+ },
139
+ ];
140
+ const result = calculateToolCallingAccuracy(expected, actual);
141
+ expect(result).toBe(1);
142
+ });
143
+
144
+ it("should return 0 when expected has no filter but actual has non-empty filter", () => {
145
+ const expected: ExpectedToolCall[] = [
146
+ {
147
+ toolName: "find",
148
+ parameters: {
149
+ database: "mflix",
150
+ collection: "movies",
151
+ filter: Matcher.emptyObjectOrUndefined,
152
+ },
153
+ },
154
+ ];
155
+ const actual: LLMToolCall[] = [
156
+ {
157
+ toolCallId: "1",
158
+ toolName: "find",
159
+ parameters: {
160
+ database: "mflix",
161
+ collection: "movies",
162
+ filter: { genre: "Horror" },
163
+ },
164
+ },
165
+ ];
166
+ const result = calculateToolCallingAccuracy(expected, actual);
167
+ expect(result).toBe(0);
168
+ });
169
+
170
+ it("should return 0 when there are additional nested fields", () => {
171
+ const expected: ExpectedToolCall[] = [
172
+ {
173
+ toolName: "find",
174
+ parameters: {
175
+ database: "mflix",
176
+ collection: "movies",
177
+ filter: { runtime: { $lt: 100 } },
178
+ },
179
+ },
180
+ ];
181
+ const actual: LLMToolCall[] = [
182
+ {
183
+ toolCallId: "1",
184
+ toolName: "find",
185
+ parameters: {
186
+ database: "mflix",
187
+ collection: "movies",
188
+ filter: { runtime: { $lt: 100 }, genre: "Horror" },
189
+ },
190
+ },
191
+ ];
192
+ const result = calculateToolCallingAccuracy(expected, actual);
193
+ expect(result).toBe(0);
194
+ });
195
+
196
+ it("should return 1 when ignored additional fields are provided", () => {
197
+ const expected: ExpectedToolCall[] = [
198
+ {
199
+ toolName: "find",
200
+ parameters: {
201
+ database: "mflix",
202
+ collection: "movies",
203
+ filter: { runtime: { $lt: 100 } },
204
+ limit: Matcher.number(),
205
+ sort: Matcher.anyValue,
206
+ },
207
+ },
208
+ ];
209
+ const actual: LLMToolCall[] = [
210
+ {
211
+ toolCallId: "1",
212
+ toolName: "find",
213
+ parameters: {
214
+ database: "mflix",
215
+ collection: "movies",
216
+ filter: { runtime: { $lt: 100 } },
217
+ limit: 10,
218
+ sort: { title: 1 },
219
+ },
220
+ },
221
+ ];
222
+ const result = calculateToolCallingAccuracy(expected, actual);
223
+ expect(result).toBe(1);
224
+ });
225
+
226
+ it("should return 1 for array where additional elements are allowed", () => {
227
+ const expected: ExpectedToolCall[] = [
228
+ {
229
+ toolName: "aggregate",
230
+ parameters: {
231
+ database: "mflix",
232
+ collection: "movies",
233
+ pipeline: [{ $match: { genre: "Horror" } }, Matcher.anyOf(Matcher.undefined, Matcher.anyValue)],
234
+ },
235
+ },
236
+ ];
237
+
238
+ const actual: LLMToolCall[] = [
239
+ {
240
+ toolCallId: "1",
241
+ toolName: "aggregate",
242
+ parameters: {
243
+ database: "mflix",
244
+ collection: "movies",
245
+ pipeline: [{ $match: { genre: "Horror" } }, { $sort: { title: 1 } }],
246
+ },
247
+ },
248
+ ];
249
+
250
+ const result = calculateToolCallingAccuracy(expected, actual);
251
+ expect(result).toBe(1);
252
+ });
253
+
254
+ it("should return 1 for array where additional elements are allowed but not provided", () => {
255
+ const expected: ExpectedToolCall[] = [
256
+ {
257
+ toolName: "aggregate",
258
+ parameters: {
259
+ database: "mflix",
260
+ collection: "movies",
261
+ pipeline: [{ $match: { genre: "Horror" } }, Matcher.anyOf(Matcher.undefined, Matcher.anyValue)],
262
+ },
263
+ },
264
+ ];
265
+
266
+ const actual: LLMToolCall[] = [
267
+ {
268
+ toolCallId: "1",
269
+ toolName: "aggregate",
270
+ parameters: {
271
+ database: "mflix",
272
+ collection: "movies",
273
+ pipeline: [{ $match: { genre: "Horror" } }],
274
+ },
275
+ },
276
+ ];
277
+
278
+ const result = calculateToolCallingAccuracy(expected, actual);
279
+ expect(result).toBe(1);
280
+ });
281
+ });
282
+
283
+ describe("missing or incorrect parameters", () => {
284
+ it("should return 0 when tool call has missing nested parameters", () => {
285
+ const expected: ExpectedToolCall[] = [
286
+ {
287
+ toolName: "find",
288
+ parameters: { db: "test", collection: "users", filter: { status: "active", age: { $gte: 18 } } },
289
+ },
290
+ ];
291
+ const actual: LLMToolCall[] = [
292
+ {
293
+ toolCallId: "1",
294
+ toolName: "find",
295
+ parameters: { db: "test", collection: "users", filter: { status: "active" } },
296
+ },
297
+ ];
298
+ const result = calculateToolCallingAccuracy(expected, actual);
299
+ expect(result).toBe(0);
300
+ });
301
+
302
+ it("should return 0 when aggregate tool call has incorrect pipeline", () => {
303
+ const expected: ExpectedToolCall[] = [
304
+ {
305
+ toolName: "aggregate",
306
+ parameters: { db: "test", collection: "orders", pipeline: [{ $match: { total: { $gt: 100 } } }] },
307
+ },
308
+ ];
309
+ const actual: LLMToolCall[] = [
310
+ {
311
+ toolCallId: "1",
312
+ toolName: "aggregate",
313
+ parameters: { db: "test", collection: "orders", pipeline: [{ $match: { total: { $lt: 50 } } }] },
314
+ },
315
+ ];
316
+ const result = calculateToolCallingAccuracy(expected, actual);
317
+ expect(result).toBe(0);
318
+ });
319
+ });
320
+
321
+ describe("additional tool calls", () => {
322
+ it("should cap accuracy at 0.75 when LLM calls extra tools", () => {
323
+ const expected: ExpectedToolCall[] = [
324
+ { toolName: "find", parameters: { db: "test", collection: "users", filter: { status: "active" } } },
325
+ ];
326
+ const actual: LLMToolCall[] = [
327
+ {
328
+ toolCallId: "1",
329
+ toolName: "find",
330
+ parameters: { db: "test", collection: "users", filter: { status: "active" } },
331
+ },
332
+ { toolCallId: "2", toolName: "count", parameters: { db: "test", collection: "orders" } },
333
+ {
334
+ toolCallId: "3",
335
+ toolName: "aggregate",
336
+ parameters: {
337
+ db: "test",
338
+ collection: "products",
339
+ pipeline: [{ $group: { _id: "$category", total: { $sum: 1 } } }],
340
+ },
341
+ },
342
+ ];
343
+ const result = calculateToolCallingAccuracy(expected, actual);
344
+ expect(result).toBe(0.75);
345
+ });
346
+
347
+ it("should cap accuracy at 0.75 when LLM calls same tool multiple times with variations", () => {
348
+ const expected: ExpectedToolCall[] = [
349
+ { toolName: "find", parameters: { db: "test", collection: "users", filter: { status: "active" } } },
350
+ ];
351
+ const actual: LLMToolCall[] = [
352
+ {
353
+ toolCallId: "1",
354
+ toolName: "find",
355
+ parameters: { db: "test", collection: "users", filter: { status: "active" } },
356
+ },
357
+ {
358
+ toolCallId: "2",
359
+ toolName: "find",
360
+ parameters: { db: "test", collection: "users", filter: { status: "active", age: { $gte: 18 } } },
361
+ },
362
+ { toolCallId: "3", toolName: "find", parameters: { db: "test", collection: "users", limit: 10 } },
363
+ ];
364
+ const result = calculateToolCallingAccuracy(expected, actual);
365
+ expect(result).toBe(0.75);
366
+ });
367
+ });
368
+
369
+ describe("missing tool calls", () => {
370
+ it("should return 0 if any expected tool call was not called", () => {
371
+ const expected: ExpectedToolCall[] = [
372
+ { toolName: "find", parameters: { db: "test", collection: "users", filter: { status: "active" } } },
373
+ {
374
+ toolName: "aggregate",
375
+ parameters: { db: "test", collection: "orders", pipeline: [{ $match: { total: { $gt: 100 } } }] },
376
+ },
377
+ ];
378
+ const actual: LLMToolCall[] = [
379
+ {
380
+ toolCallId: "1",
381
+ toolName: "find",
382
+ parameters: { db: "test", collection: "users", filter: { status: "active" } },
383
+ },
384
+ // Missing the aggregate tool call
385
+ ];
386
+ const result = calculateToolCallingAccuracy(expected, actual);
387
+ expect(result).toBe(0); // One expected tool call was not called
388
+ });
389
+ });
390
+ });
@@ -1,6 +1,6 @@
1
- import { jest } from "@jest/globals";
2
- import { ApiClient } from "../../src/common/atlas/apiClient.js";
3
- import { CommonProperties, TelemetryEvent, TelemetryResult } from "../../src/telemetry/types.js";
1
+ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
2
+ import { ApiClient } from "../../../src/common/atlas/apiClient.js";
3
+ import { CommonProperties, TelemetryEvent, TelemetryResult } from "../../../src/telemetry/types.js";
4
4
 
5
5
  describe("ApiClient", () => {
6
6
  let apiClient: ApiClient;
@@ -36,11 +36,11 @@ describe("ApiClient", () => {
36
36
  });
37
37
 
38
38
  // @ts-expect-error accessing private property for testing
39
- apiClient.getAccessToken = jest.fn().mockResolvedValue("mockToken");
39
+ apiClient.getAccessToken = vi.fn().mockResolvedValue("mockToken");
40
40
  });
41
41
 
42
42
  afterEach(() => {
43
- jest.clearAllMocks();
43
+ vi.clearAllMocks();
44
44
  });
45
45
 
46
46
  describe("constructor", () => {
@@ -60,7 +60,7 @@ describe("ApiClient", () => {
60
60
  totalCount: 2,
61
61
  };
62
62
 
63
- const mockGet = jest.fn().mockImplementation(() => ({
63
+ const mockGet = vi.fn().mockImplementation(() => ({
64
64
  data: mockProjects,
65
65
  error: null,
66
66
  response: new Response(),
@@ -81,7 +81,7 @@ describe("ApiClient", () => {
81
81
  detail: "Something went wrong",
82
82
  };
83
83
 
84
- const mockGet = jest.fn().mockImplementation(() => ({
84
+ const mockGet = vi.fn().mockImplementation(() => ({
85
85
  data: null,
86
86
  error: mockError,
87
87
  response: new Response(),
@@ -96,7 +96,7 @@ describe("ApiClient", () => {
96
96
 
97
97
  describe("sendEvents", () => {
98
98
  it("should send events to authenticated endpoint when token is available and valid", async () => {
99
- const mockFetch = jest.spyOn(global, "fetch");
99
+ const mockFetch = vi.spyOn(global, "fetch");
100
100
  mockFetch.mockResolvedValueOnce(new Response(null, { status: 200 }));
101
101
 
102
102
  await apiClient.sendEvents(mockEvents);
@@ -115,11 +115,11 @@ describe("ApiClient", () => {
115
115
  });
116
116
 
117
117
  it("should fall back to unauthenticated endpoint when token is not available via exception", async () => {
118
- const mockFetch = jest.spyOn(global, "fetch");
118
+ const mockFetch = vi.spyOn(global, "fetch");
119
119
  mockFetch.mockResolvedValueOnce(new Response(null, { status: 200 }));
120
120
 
121
121
  // @ts-expect-error accessing private property for testing
122
- apiClient.getAccessToken = jest.fn().mockRejectedValue(new Error("No access token available"));
122
+ apiClient.getAccessToken = vi.fn().mockRejectedValue(new Error("No access token available"));
123
123
 
124
124
  await apiClient.sendEvents(mockEvents);
125
125
 
@@ -136,11 +136,11 @@ describe("ApiClient", () => {
136
136
  });
137
137
 
138
138
  it("should fall back to unauthenticated endpoint when token is undefined", async () => {
139
- const mockFetch = jest.spyOn(global, "fetch");
139
+ const mockFetch = vi.spyOn(global, "fetch");
140
140
  mockFetch.mockResolvedValueOnce(new Response(null, { status: 200 }));
141
141
 
142
142
  // @ts-expect-error accessing private property for testing
143
- apiClient.getAccessToken = jest.fn().mockReturnValueOnce(undefined);
143
+ apiClient.getAccessToken = vi.fn().mockReturnValueOnce(undefined);
144
144
 
145
145
  await apiClient.sendEvents(mockEvents);
146
146
 
@@ -157,7 +157,7 @@ describe("ApiClient", () => {
157
157
  });
158
158
 
159
159
  it("should fall back to unauthenticated endpoint on 401 error", async () => {
160
- const mockFetch = jest.spyOn(global, "fetch");
160
+ const mockFetch = vi.spyOn(global, "fetch");
161
161
  mockFetch
162
162
  .mockResolvedValueOnce(new Response(null, { status: 401 }))
163
163
  .mockResolvedValueOnce(new Response(null, { status: 200 }));
@@ -178,14 +178,14 @@ describe("ApiClient", () => {
178
178
  });
179
179
 
180
180
  it("should throw error when both authenticated and unauthenticated requests fail", async () => {
181
- const mockFetch = jest.spyOn(global, "fetch");
181
+ const mockFetch = vi.spyOn(global, "fetch");
182
182
  mockFetch
183
183
  .mockResolvedValueOnce(new Response(null, { status: 401 }))
184
184
  .mockResolvedValueOnce(new Response(null, { status: 500 }));
185
185
 
186
186
  const mockToken = "test-token";
187
187
  // @ts-expect-error accessing private property for testing
188
- apiClient.getAccessToken = jest.fn().mockResolvedValue(mockToken);
188
+ apiClient.getAccessToken = vi.fn().mockResolvedValue(mockToken);
189
189
 
190
190
  await expect(apiClient.sendEvents(mockEvents)).rejects.toThrow();
191
191
  });