mongodb-mcp-server 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (270) hide show
  1. package/.github/pull_request_template.md +5 -0
  2. package/.github/workflows/accuracy-tests.yml +55 -0
  3. package/.github/workflows/check.yml +1 -1
  4. package/.github/workflows/code_health.yaml +4 -4
  5. package/.github/workflows/code_health_fork.yaml +0 -14
  6. package/.github/workflows/dependabot_pr.yaml +26 -0
  7. package/.github/workflows/docker.yaml +1 -1
  8. package/.github/workflows/jira-issue.yml +72 -0
  9. package/.smithery/smithery.yaml +10 -0
  10. package/.vscode/extensions.json +1 -1
  11. package/.vscode/launch.json +11 -1
  12. package/.vscode/settings.json +1 -11
  13. package/Dockerfile +1 -0
  14. package/README.md +132 -31
  15. package/dist/common/atlas/accessListUtils.js +36 -0
  16. package/dist/common/atlas/accessListUtils.js.map +1 -0
  17. package/dist/common/atlas/apiClient.js +25 -6
  18. package/dist/common/atlas/apiClient.js.map +1 -1
  19. package/dist/common/atlas/cluster.js +1 -1
  20. package/dist/common/atlas/cluster.js.map +1 -1
  21. package/dist/{config.js → common/config.js} +8 -1
  22. package/dist/common/config.js.map +1 -0
  23. package/dist/{errors.js → common/errors.js} +1 -0
  24. package/dist/common/errors.js.map +1 -0
  25. package/dist/{logger.js → common/logger.js} +20 -19
  26. package/dist/common/logger.js.map +1 -0
  27. package/dist/common/managedTimeout.js +20 -0
  28. package/dist/common/managedTimeout.js.map +1 -0
  29. package/dist/common/packageInfo.js.map +1 -0
  30. package/dist/{session.js → common/session.js} +20 -21
  31. package/dist/common/session.js.map +1 -0
  32. package/dist/common/sessionStore.js +73 -0
  33. package/dist/common/sessionStore.js.map +1 -0
  34. package/dist/helpers/container.js +28 -0
  35. package/dist/helpers/container.js.map +1 -0
  36. package/dist/helpers/generatePassword.js.map +1 -0
  37. package/dist/helpers/indexCheck.js +63 -0
  38. package/dist/helpers/indexCheck.js.map +1 -0
  39. package/dist/index.js +30 -37
  40. package/dist/index.js.map +1 -1
  41. package/dist/server.js +44 -7
  42. package/dist/server.js.map +1 -1
  43. package/dist/telemetry/constants.js +1 -1
  44. package/dist/telemetry/constants.js.map +1 -1
  45. package/dist/telemetry/telemetry.js +86 -116
  46. package/dist/telemetry/telemetry.js.map +1 -1
  47. package/dist/tools/atlas/atlasTool.js +3 -3
  48. package/dist/tools/atlas/atlasTool.js.map +1 -1
  49. package/dist/tools/atlas/connect/connectCluster.js +198 -0
  50. package/dist/tools/atlas/connect/connectCluster.js.map +1 -0
  51. package/dist/tools/atlas/create/createAccessList.js +9 -10
  52. package/dist/tools/atlas/create/createAccessList.js.map +1 -1
  53. package/dist/tools/atlas/create/createDBUser.js +3 -1
  54. package/dist/tools/atlas/create/createDBUser.js.map +1 -1
  55. package/dist/tools/atlas/create/createFreeCluster.js +2 -0
  56. package/dist/tools/atlas/create/createFreeCluster.js.map +1 -1
  57. package/dist/tools/atlas/create/createProject.js.map +1 -1
  58. package/dist/tools/atlas/read/inspectAccessList.js.map +1 -1
  59. package/dist/tools/atlas/read/inspectCluster.js.map +1 -1
  60. package/dist/tools/atlas/read/listAlerts.js.map +1 -1
  61. package/dist/tools/atlas/read/listClusters.js.map +1 -1
  62. package/dist/tools/atlas/read/listDBUsers.js.map +1 -1
  63. package/dist/tools/atlas/read/listOrgs.js.map +1 -1
  64. package/dist/tools/atlas/read/listProjects.js.map +1 -1
  65. package/dist/tools/atlas/tools.js +1 -1
  66. package/dist/tools/atlas/tools.js.map +1 -1
  67. package/dist/tools/mongodb/{metadata → connect}/connect.js +7 -4
  68. package/dist/tools/mongodb/connect/connect.js.map +1 -0
  69. package/dist/tools/mongodb/create/createCollection.js.map +1 -1
  70. package/dist/tools/mongodb/create/createIndex.js +1 -1
  71. package/dist/tools/mongodb/create/createIndex.js.map +1 -1
  72. package/dist/tools/mongodb/create/insertMany.js +1 -1
  73. package/dist/tools/mongodb/create/insertMany.js.map +1 -1
  74. package/dist/tools/mongodb/delete/deleteMany.js +20 -1
  75. package/dist/tools/mongodb/delete/deleteMany.js.map +1 -1
  76. package/dist/tools/mongodb/delete/dropCollection.js.map +1 -1
  77. package/dist/tools/mongodb/delete/dropDatabase.js.map +1 -1
  78. package/dist/tools/mongodb/metadata/collectionSchema.js.map +1 -1
  79. package/dist/tools/mongodb/metadata/collectionStorageSize.js.map +1 -1
  80. package/dist/tools/mongodb/metadata/dbStats.js.map +1 -1
  81. package/dist/tools/mongodb/metadata/explain.js +2 -2
  82. package/dist/tools/mongodb/metadata/explain.js.map +1 -1
  83. package/dist/tools/mongodb/metadata/listCollections.js.map +1 -1
  84. package/dist/tools/mongodb/metadata/listDatabases.js.map +1 -1
  85. package/dist/tools/mongodb/metadata/logs.js.map +1 -1
  86. package/dist/tools/mongodb/mongodbTool.js +47 -10
  87. package/dist/tools/mongodb/mongodbTool.js.map +1 -1
  88. package/dist/tools/mongodb/read/aggregate.js +10 -1
  89. package/dist/tools/mongodb/read/aggregate.js.map +1 -1
  90. package/dist/tools/mongodb/read/collectionIndexes.js.map +1 -1
  91. package/dist/tools/mongodb/read/count.js +15 -1
  92. package/dist/tools/mongodb/read/count.js.map +1 -1
  93. package/dist/tools/mongodb/read/find.js +14 -4
  94. package/dist/tools/mongodb/read/find.js.map +1 -1
  95. package/dist/tools/mongodb/tools.js +1 -1
  96. package/dist/tools/mongodb/tools.js.map +1 -1
  97. package/dist/tools/mongodb/update/renameCollection.js.map +1 -1
  98. package/dist/tools/mongodb/update/updateMany.js +24 -2
  99. package/dist/tools/mongodb/update/updateMany.js.map +1 -1
  100. package/dist/tools/tool.js +12 -9
  101. package/dist/tools/tool.js.map +1 -1
  102. package/dist/transports/base.js +26 -0
  103. package/dist/transports/base.js.map +1 -0
  104. package/dist/{helpers/EJsonTransport.js → transports/stdio.js} +24 -2
  105. package/dist/transports/stdio.js.map +1 -0
  106. package/dist/transports/streamableHttp.js +140 -0
  107. package/dist/transports/streamableHttp.js.map +1 -0
  108. package/eslint.config.js +13 -4
  109. package/package.json +43 -33
  110. package/resources/test-summary-template.html +415 -0
  111. package/scripts/accuracy/generateTestSummary.ts +335 -0
  112. package/scripts/accuracy/runAccuracyTests.sh +45 -0
  113. package/scripts/accuracy/updateAccuracyRunStatus.ts +21 -0
  114. package/src/common/atlas/accessListUtils.ts +54 -0
  115. package/src/common/atlas/apiClient.ts +25 -6
  116. package/src/common/atlas/cluster.ts +1 -1
  117. package/src/{config.ts → common/config.ts} +16 -2
  118. package/src/{errors.ts → common/errors.ts} +1 -0
  119. package/src/{logger.ts → common/logger.ts} +21 -24
  120. package/src/common/managedTimeout.ts +27 -0
  121. package/src/{session.ts → common/session.ts} +24 -26
  122. package/src/common/sessionStore.ts +111 -0
  123. package/src/helpers/container.ts +35 -0
  124. package/src/helpers/indexCheck.ts +83 -0
  125. package/src/index.ts +30 -40
  126. package/src/server.ts +55 -11
  127. package/src/telemetry/constants.ts +1 -1
  128. package/src/telemetry/telemetry.ts +109 -153
  129. package/src/telemetry/types.ts +2 -1
  130. package/src/tools/atlas/atlasTool.ts +4 -4
  131. package/src/tools/atlas/connect/connectCluster.ts +259 -0
  132. package/src/tools/atlas/create/createAccessList.ts +15 -13
  133. package/src/tools/atlas/create/createDBUser.ts +5 -3
  134. package/src/tools/atlas/create/createFreeCluster.ts +4 -2
  135. package/src/tools/atlas/create/createProject.ts +2 -2
  136. package/src/tools/atlas/read/inspectAccessList.ts +2 -2
  137. package/src/tools/atlas/read/inspectCluster.ts +2 -2
  138. package/src/tools/atlas/read/listAlerts.ts +2 -2
  139. package/src/tools/atlas/read/listClusters.ts +2 -2
  140. package/src/tools/atlas/read/listDBUsers.ts +2 -2
  141. package/src/tools/atlas/read/listOrgs.ts +2 -2
  142. package/src/tools/atlas/read/listProjects.ts +2 -2
  143. package/src/tools/atlas/tools.ts +1 -1
  144. package/src/tools/mongodb/{metadata → connect}/connect.ts +12 -9
  145. package/src/tools/mongodb/create/createCollection.ts +2 -2
  146. package/src/tools/mongodb/create/createIndex.ts +3 -3
  147. package/src/tools/mongodb/create/insertMany.ts +3 -3
  148. package/src/tools/mongodb/delete/deleteMany.ts +24 -3
  149. package/src/tools/mongodb/delete/dropCollection.ts +2 -2
  150. package/src/tools/mongodb/delete/dropDatabase.ts +2 -2
  151. package/src/tools/mongodb/metadata/collectionSchema.ts +2 -2
  152. package/src/tools/mongodb/metadata/collectionStorageSize.ts +2 -2
  153. package/src/tools/mongodb/metadata/dbStats.ts +2 -2
  154. package/src/tools/mongodb/metadata/explain.ts +4 -4
  155. package/src/tools/mongodb/metadata/listCollections.ts +2 -2
  156. package/src/tools/mongodb/metadata/listDatabases.ts +2 -2
  157. package/src/tools/mongodb/metadata/logs.ts +2 -2
  158. package/src/tools/mongodb/mongodbTool.ts +60 -14
  159. package/src/tools/mongodb/read/aggregate.ts +14 -3
  160. package/src/tools/mongodb/read/collectionIndexes.ts +2 -2
  161. package/src/tools/mongodb/read/count.ts +19 -3
  162. package/src/tools/mongodb/read/find.ts +20 -6
  163. package/src/tools/mongodb/tools.ts +1 -1
  164. package/src/tools/mongodb/update/renameCollection.ts +2 -2
  165. package/src/tools/mongodb/update/updateMany.ts +28 -4
  166. package/src/tools/tool.ts +23 -18
  167. package/src/transports/base.ts +34 -0
  168. package/src/{helpers/EJsonTransport.ts → transports/stdio.ts} +30 -1
  169. package/src/transports/streamableHttp.ts +178 -0
  170. package/tests/accuracy/aggregate.test.ts +27 -0
  171. package/tests/accuracy/collectionIndexes.test.ts +40 -0
  172. package/tests/accuracy/collectionSchema.test.ts +28 -0
  173. package/tests/accuracy/collectionStorageSize.test.ts +41 -0
  174. package/tests/accuracy/count.test.ts +44 -0
  175. package/tests/accuracy/createCollection.test.ts +46 -0
  176. package/tests/accuracy/createIndex.test.ts +37 -0
  177. package/tests/accuracy/dbStats.test.ts +15 -0
  178. package/tests/accuracy/deleteMany.test.ts +44 -0
  179. package/tests/accuracy/dropCollection.test.ts +74 -0
  180. package/tests/accuracy/dropDatabase.test.ts +41 -0
  181. package/tests/accuracy/explain.test.ts +73 -0
  182. package/tests/accuracy/find.test.ts +114 -0
  183. package/tests/accuracy/insertMany.test.ts +48 -0
  184. package/tests/accuracy/listCollections.test.ts +60 -0
  185. package/tests/accuracy/listDatabases.test.ts +31 -0
  186. package/tests/accuracy/logs.test.ts +28 -0
  187. package/tests/accuracy/renameCollection.test.ts +31 -0
  188. package/tests/accuracy/sdk/accuracyResultStorage/diskStorage.ts +189 -0
  189. package/tests/accuracy/sdk/accuracyResultStorage/getAccuracyResultStorage.ts +11 -0
  190. package/tests/accuracy/sdk/accuracyResultStorage/mongodbStorage.ts +151 -0
  191. package/tests/accuracy/sdk/accuracyResultStorage/resultStorage.ts +117 -0
  192. package/tests/accuracy/sdk/accuracyScorer.ts +93 -0
  193. package/tests/accuracy/sdk/accuracyTestingClient.ts +94 -0
  194. package/tests/accuracy/sdk/agent.ts +56 -0
  195. package/tests/accuracy/sdk/constants.ts +26 -0
  196. package/tests/accuracy/sdk/describeAccuracyTests.ts +126 -0
  197. package/tests/accuracy/sdk/gitInfo.ts +7 -0
  198. package/tests/accuracy/sdk/matcher.ts +193 -0
  199. package/tests/accuracy/sdk/models.ts +95 -0
  200. package/tests/accuracy/test-data-dumps/comics.books.json +417 -0
  201. package/tests/accuracy/test-data-dumps/comics.characters.json +402 -0
  202. package/tests/accuracy/test-data-dumps/mflix.movies.json +496 -0
  203. package/tests/accuracy/test-data-dumps/mflix.shows.json +572 -0
  204. package/tests/accuracy/updateMany.test.ts +42 -0
  205. package/tests/integration/helpers.ts +9 -9
  206. package/tests/integration/indexCheck.test.ts +464 -0
  207. package/tests/integration/server.test.ts +6 -4
  208. package/tests/integration/telemetry.test.ts +29 -0
  209. package/tests/integration/tools/atlas/accessLists.test.ts +22 -2
  210. package/tests/integration/tools/atlas/alerts.test.ts +3 -2
  211. package/tests/integration/tools/atlas/atlasHelpers.ts +3 -0
  212. package/tests/integration/tools/atlas/clusters.test.ts +68 -16
  213. package/tests/integration/tools/atlas/dbUsers.test.ts +14 -1
  214. package/tests/integration/tools/atlas/orgs.test.ts +2 -1
  215. package/tests/integration/tools/atlas/projects.test.ts +4 -3
  216. package/tests/integration/tools/mongodb/{metadata → connect}/connect.test.ts +34 -3
  217. package/tests/integration/tools/mongodb/create/createCollection.test.ts +1 -0
  218. package/tests/integration/tools/mongodb/create/createIndex.test.ts +1 -0
  219. package/tests/integration/tools/mongodb/create/insertMany.test.ts +1 -0
  220. package/tests/integration/tools/mongodb/delete/deleteMany.test.ts +1 -0
  221. package/tests/integration/tools/mongodb/delete/dropCollection.test.ts +1 -1
  222. package/tests/integration/tools/mongodb/delete/dropDatabase.test.ts +1 -0
  223. package/tests/integration/tools/mongodb/metadata/collectionSchema.test.ts +1 -0
  224. package/tests/integration/tools/mongodb/metadata/collectionStorageSize.test.ts +1 -0
  225. package/tests/integration/tools/mongodb/metadata/dbStats.test.ts +1 -0
  226. package/tests/integration/tools/mongodb/metadata/explain.test.ts +1 -0
  227. package/tests/integration/tools/mongodb/metadata/listCollections.test.ts +1 -0
  228. package/tests/integration/tools/mongodb/metadata/listDatabases.test.ts +3 -2
  229. package/tests/integration/tools/mongodb/metadata/logs.test.ts +1 -0
  230. package/tests/integration/tools/mongodb/mongodbHelpers.ts +67 -2
  231. package/tests/integration/tools/mongodb/read/aggregate.test.ts +2 -1
  232. package/tests/integration/tools/mongodb/read/collectionIndexes.test.ts +1 -0
  233. package/tests/integration/tools/mongodb/read/count.test.ts +1 -0
  234. package/tests/integration/tools/mongodb/read/find.test.ts +2 -1
  235. package/tests/integration/tools/mongodb/update/renameCollection.test.ts +1 -0
  236. package/tests/integration/tools/mongodb/update/updateMany.test.ts +1 -0
  237. package/tests/integration/transports/stdio.test.ts +40 -0
  238. package/tests/integration/transports/streamableHttp.test.ts +56 -0
  239. package/tests/matchers/toIncludeSameMembers.test.ts +59 -0
  240. package/tests/matchers/toIncludeSameMembers.ts +12 -0
  241. package/tests/setup.ts +7 -0
  242. package/tests/unit/accessListUtils.test.ts +39 -0
  243. package/tests/unit/accuracyScorer.test.ts +390 -0
  244. package/tests/unit/{apiClient.test.ts → common/apiClient.test.ts} +15 -15
  245. package/tests/unit/common/managedTimeout.test.ts +67 -0
  246. package/tests/unit/{session.test.ts → common/session.test.ts} +7 -12
  247. package/tests/unit/helpers/indexCheck.test.ts +150 -0
  248. package/tests/unit/telemetry.test.ts +99 -137
  249. package/tests/unit/{EJsonTransport.test.ts → transports/stdio.test.ts} +4 -4
  250. package/tests/vitest.d.ts +11 -0
  251. package/tsconfig.json +0 -1
  252. package/{tsconfig.jest.json → tsconfig.test.json} +1 -2
  253. package/vitest.config.ts +41 -0
  254. package/dist/common/atlas/generatePassword.js.map +0 -1
  255. package/dist/config.js.map +0 -1
  256. package/dist/errors.js.map +0 -1
  257. package/dist/helpers/EJsonTransport.js.map +0 -1
  258. package/dist/helpers/packageInfo.js.map +0 -1
  259. package/dist/logger.js.map +0 -1
  260. package/dist/session.js.map +0 -1
  261. package/dist/tools/atlas/metadata/connectCluster.js +0 -100
  262. package/dist/tools/atlas/metadata/connectCluster.js.map +0 -1
  263. package/dist/tools/mongodb/metadata/connect.js.map +0 -1
  264. package/global.d.ts +0 -1
  265. package/jest.config.cjs +0 -22
  266. package/src/tools/atlas/metadata/connectCluster.ts +0 -121
  267. /package/dist/{helpers → common}/packageInfo.js +0 -0
  268. /package/dist/{common/atlas → helpers}/generatePassword.js +0 -0
  269. /package/src/{helpers → common}/packageInfo.ts +0 -0
  270. /package/src/{common/atlas → helpers}/generatePassword.ts +0 -0
@@ -0,0 +1,193 @@
1
+ const MATCHER_SYMBOL = Symbol("match");
2
+
3
+ export abstract class Matcher {
4
+ [MATCHER_SYMBOL] = true;
5
+ public abstract match(actual: unknown): number;
6
+
7
+ public static get emptyObjectOrUndefined(): Matcher {
8
+ return new EmptyObjectOrUndefinedMatcher();
9
+ }
10
+
11
+ public static get anyValue(): Matcher {
12
+ return new AnyValueMatcher();
13
+ }
14
+
15
+ public static number(additionalFilter: (value: number) => boolean = () => true): Matcher {
16
+ return new NumberMatcher(additionalFilter);
17
+ }
18
+
19
+ public static anyOf(...matchers: Matcher[]): Matcher {
20
+ return new CompositeMatcher(matchers);
21
+ }
22
+
23
+ public static get undefined(): Matcher {
24
+ return new UndefinedMatcher();
25
+ }
26
+
27
+ public static boolean(expected?: boolean): Matcher {
28
+ return new BooleanMatcher(expected);
29
+ }
30
+
31
+ public static string(): Matcher {
32
+ return new StringMatcher();
33
+ }
34
+
35
+ public static value(expected: unknown): Matcher {
36
+ if (typeof expected === "object" && expected !== null && MATCHER_SYMBOL in expected) {
37
+ return expected as Matcher;
38
+ }
39
+
40
+ return new ValueMatcher(expected);
41
+ }
42
+ }
43
+
44
+ export const PARAMETER_SCORER_SYMBOL = Symbol("parameterScorer");
45
+
46
+ class EmptyObjectOrUndefinedMatcher extends Matcher {
47
+ public match(actual: unknown): number {
48
+ if (
49
+ actual === undefined ||
50
+ actual === null ||
51
+ (typeof actual === "object" && Object.keys(actual).length === 0)
52
+ ) {
53
+ return 1; // Match if actual is undefined, null, or an empty object
54
+ }
55
+
56
+ return 0; // No match
57
+ }
58
+ }
59
+
60
+ class AnyValueMatcher extends Matcher {
61
+ public match(): number {
62
+ return 1;
63
+ }
64
+ }
65
+
66
+ class NumberMatcher extends Matcher {
67
+ constructor(private additionalFilter: (value: number) => boolean = () => true) {
68
+ super();
69
+ }
70
+ public match(actual: unknown): number {
71
+ return typeof actual === "number" && this.additionalFilter(actual) ? 1 : 0;
72
+ }
73
+ }
74
+
75
+ class UndefinedMatcher extends Matcher {
76
+ public match(actual: unknown): number {
77
+ return actual === undefined ? 1 : 0;
78
+ }
79
+ }
80
+
81
+ class CompositeMatcher extends Matcher {
82
+ constructor(private matchers: Matcher[]) {
83
+ super();
84
+ }
85
+
86
+ public match(actual: unknown): number {
87
+ let currentScore = 0;
88
+
89
+ for (const matcher of this.matchers) {
90
+ const score = matcher.match(actual);
91
+ if (score === 1) {
92
+ return 1; // If one of the matchers is perfect score, return immediately
93
+ }
94
+ currentScore = Math.max(currentScore, score);
95
+ }
96
+
97
+ return currentScore;
98
+ }
99
+ }
100
+
101
+ class BooleanMatcher extends Matcher {
102
+ constructor(private expected?: boolean) {
103
+ super();
104
+ }
105
+
106
+ public match(actual: unknown): number {
107
+ return typeof actual === "boolean" && (this.expected === undefined || this.expected === actual) ? 1 : 0;
108
+ }
109
+ }
110
+
111
+ class StringMatcher extends Matcher {
112
+ public match(actual: unknown): number {
113
+ return typeof actual === "string" ? 1 : 0;
114
+ }
115
+ }
116
+
117
+ class ValueMatcher extends Matcher {
118
+ constructor(private expected: unknown) {
119
+ super();
120
+ }
121
+
122
+ public match(actual: unknown): number {
123
+ if (this.expected === actual) {
124
+ // If both are the same, just return immediately.
125
+ return 1;
126
+ }
127
+
128
+ if (this.expected === undefined || this.expected === null) {
129
+ // We expect null/undefined - return 1 if actual is also null/undefined
130
+ return actual === undefined || actual === null ? 1 : 0;
131
+ }
132
+
133
+ let currentScore = 1;
134
+
135
+ if (Array.isArray(this.expected)) {
136
+ if (!Array.isArray(actual)) {
137
+ // One is an array, the other is not
138
+ return 0;
139
+ }
140
+
141
+ if (actual.length > this.expected.length) {
142
+ // Actual array has more elements - this is likely an error (e.g. an aggregation pipeline with extra stages)
143
+ // If we want to allow extra elements, we should add matchers to the array
144
+ return 0;
145
+ }
146
+
147
+ for (let i = 0; i < this.expected.length; i++) {
148
+ currentScore = Math.min(currentScore, Matcher.value(this.expected[i]).match(actual[i]));
149
+ if (currentScore === 0) {
150
+ // If we already found a mismatch, we can stop early
151
+ return 0;
152
+ }
153
+ }
154
+ } else if (typeof this.expected === "object") {
155
+ if (MATCHER_SYMBOL in this.expected) {
156
+ return (this.expected as Matcher).match(actual);
157
+ }
158
+
159
+ if (typeof actual !== "object" || actual === null) {
160
+ // One is an object, the other is not
161
+ return 0;
162
+ }
163
+
164
+ const expectedKeys = Object.keys(this.expected);
165
+ const actualKeys = Object.keys(actual);
166
+
167
+ if (actualKeys.length > expectedKeys.length) {
168
+ // The model provided more keys than expected - this should not happen.
169
+ // If we want to allow some extra keys, we should specify that in the test definition
170
+ // by adding matchers for those keys.
171
+ return 0;
172
+ }
173
+
174
+ for (const key of expectedKeys) {
175
+ currentScore = Math.min(
176
+ currentScore,
177
+ Matcher.value((this.expected as Record<string, unknown>)[key]).match(
178
+ (actual as Record<string, unknown>)[key]
179
+ )
180
+ );
181
+
182
+ if (currentScore === 0) {
183
+ // If we already found a mismatch, we can stop early
184
+ return 0;
185
+ }
186
+ }
187
+ } else {
188
+ return 0;
189
+ }
190
+
191
+ return currentScore;
192
+ }
193
+ }
@@ -0,0 +1,95 @@
1
+ import { LanguageModelV1 } from "ai";
2
+ import { createGoogleGenerativeAI } from "@ai-sdk/google";
3
+ import { createAzure } from "@ai-sdk/azure";
4
+ import { createOpenAI } from "@ai-sdk/openai";
5
+ import { ollama } from "ollama-ai-provider";
6
+
7
+ export interface Model<VercelModel extends LanguageModelV1 = LanguageModelV1> {
8
+ readonly modelName: string;
9
+ readonly provider: string;
10
+ readonly displayName: string;
11
+ isAvailable(): boolean;
12
+ getModel(): VercelModel;
13
+ }
14
+
15
+ export class OpenAIModel implements Model {
16
+ readonly provider = "OpenAI";
17
+ readonly displayName: string;
18
+
19
+ constructor(readonly modelName: string) {
20
+ this.displayName = `${this.provider} - ${modelName}`;
21
+ }
22
+
23
+ isAvailable(): boolean {
24
+ return !!process.env.MDB_OPEN_AI_API_KEY;
25
+ }
26
+
27
+ getModel() {
28
+ return createOpenAI({
29
+ apiKey: process.env.MDB_OPEN_AI_API_KEY,
30
+ })(this.modelName);
31
+ }
32
+ }
33
+
34
+ export class AzureOpenAIModel implements Model {
35
+ readonly provider = "Azure";
36
+ readonly displayName: string;
37
+
38
+ constructor(readonly modelName: string) {
39
+ this.displayName = `${this.provider} - ${modelName}`;
40
+ }
41
+
42
+ isAvailable(): boolean {
43
+ return !!process.env.MDB_AZURE_OPEN_AI_API_KEY && !!process.env.MDB_AZURE_OPEN_AI_API_URL;
44
+ }
45
+
46
+ getModel() {
47
+ return createAzure({
48
+ baseURL: process.env.MDB_AZURE_OPEN_AI_API_URL,
49
+ apiKey: process.env.MDB_AZURE_OPEN_AI_API_KEY,
50
+ apiVersion: "2024-12-01-preview",
51
+ })(this.modelName);
52
+ }
53
+ }
54
+
55
+ export class GeminiModel implements Model {
56
+ readonly provider = "Google";
57
+ readonly displayName: string;
58
+
59
+ constructor(readonly modelName: string) {
60
+ this.displayName = `${this.provider} - ${modelName}`;
61
+ }
62
+
63
+ isAvailable(): boolean {
64
+ return !!process.env.MDB_GEMINI_API_KEY;
65
+ }
66
+
67
+ getModel() {
68
+ return createGoogleGenerativeAI({
69
+ apiKey: process.env.MDB_GEMINI_API_KEY,
70
+ })(this.modelName);
71
+ }
72
+ }
73
+
74
+ export class OllamaModel implements Model {
75
+ readonly provider = "Ollama";
76
+ readonly displayName: string;
77
+
78
+ constructor(readonly modelName: string) {
79
+ this.displayName = `${this.provider} - ${modelName}`;
80
+ }
81
+
82
+ isAvailable(): boolean {
83
+ return true;
84
+ }
85
+
86
+ getModel() {
87
+ return ollama(this.modelName);
88
+ }
89
+ }
90
+
91
+ const ALL_TESTABLE_MODELS: Model[] = [new AzureOpenAIModel("gpt-4o")];
92
+
93
+ export function getAvailableModels(): Model[] {
94
+ return ALL_TESTABLE_MODELS.filter((model) => model.isAvailable());
95
+ }
@@ -0,0 +1,417 @@
1
+ [
2
+ {
3
+ "_id": "fa53ead3-36f3-414c-9b3a-53aa9cf5038a",
4
+ "title": "Configurable dedicated project",
5
+ "publisher": "Dark Horse Comics",
6
+ "release_date": "2007-03-02T00:00:00",
7
+ "issues": 118,
8
+ "main_characters": ["Stephen Shaw"],
9
+ "genre": ["Sci-Fi"]
10
+ },
11
+ {
12
+ "_id": "b2e993fb-2688-4ab0-9512-f8ada5faa948",
13
+ "title": "Focused intangible service-desk",
14
+ "publisher": "Image Comics",
15
+ "release_date": "1998-12-07T00:00:00",
16
+ "issues": 137,
17
+ "main_characters": ["Margaret Hogan"],
18
+ "genre": ["Adventure", "Horror"]
19
+ },
20
+ {
21
+ "_id": "f674a05a-12c8-4344-875c-6cd1fcba8f9d",
22
+ "title": "Expanded secondary system engine",
23
+ "publisher": "DC Comics",
24
+ "release_date": "2012-12-01T00:00:00",
25
+ "issues": 227,
26
+ "main_characters": ["Joseph Cook", "Tammy Bishop"],
27
+ "genre": ["Superhero"]
28
+ },
29
+ {
30
+ "_id": "bb72b493-2a61-41d7-9406-dfaf6e51a425",
31
+ "title": "Customizable zero-defect Graphic Interface",
32
+ "publisher": "DC Comics",
33
+ "release_date": "2011-02-24T00:00:00",
34
+ "issues": 270,
35
+ "main_characters": ["Sandra Moss"],
36
+ "genre": ["Fantasy"]
37
+ },
38
+ {
39
+ "_id": "ea85131f-dfc8-4997-b3b0-996138185d73",
40
+ "title": "Reduced eco-centric help-desk",
41
+ "publisher": "Dark Horse Comics",
42
+ "release_date": "2021-03-12T00:00:00",
43
+ "issues": 202,
44
+ "main_characters": [
45
+ "Margaret Hogan",
46
+ "Angelica Stein",
47
+ "Tammy Murphy",
48
+ "Larry Hensley"
49
+ ],
50
+ "genre": ["Adventure", "Horror"]
51
+ },
52
+ {
53
+ "_id": "fdd56270-eb31-4456-8bf4-df81371eb290",
54
+ "title": "Triple-buffered dedicated help-desk",
55
+ "publisher": "Image Comics",
56
+ "release_date": "1964-09-20T00:00:00",
57
+ "issues": 36,
58
+ "main_characters": [
59
+ "Richard Cooper",
60
+ "James Sanchez",
61
+ "Micheal Brown",
62
+ "Jeremy Rice"
63
+ ],
64
+ "genre": ["Fantasy", "Action"]
65
+ },
66
+ {
67
+ "_id": "6de66ba4-3975-4055-824c-cda5caf517d2",
68
+ "title": "Operative logistical secured line",
69
+ "publisher": "Marvel Comics",
70
+ "release_date": "2007-11-19T00:00:00",
71
+ "issues": 55,
72
+ "main_characters": ["Joseph Bowman", "Robert Logan", "Ashley Watkins"],
73
+ "genre": ["Sci-Fi", "Horror"]
74
+ },
75
+ {
76
+ "_id": "e3cafdbf-e97a-47c9-a848-bdd82e12f8f7",
77
+ "title": "Multi-lateral multi-state framework",
78
+ "publisher": "IDW Publishing",
79
+ "release_date": "2011-09-14T00:00:00",
80
+ "issues": 250,
81
+ "main_characters": [
82
+ "Ashley Watkins",
83
+ "Virginia Watts",
84
+ "Lindsay Anderson",
85
+ "Scott Garcia"
86
+ ],
87
+ "genre": ["Action", "Horror"]
88
+ },
89
+ {
90
+ "_id": "547190cd-5c9e-44c5-b8f9-afeefd039001",
91
+ "title": "Re-engineered encompassing standardization",
92
+ "publisher": "Marvel Comics",
93
+ "release_date": "1987-04-16T00:00:00",
94
+ "issues": 235,
95
+ "main_characters": ["Julie Goodwin"],
96
+ "genre": ["Sci-Fi"]
97
+ },
98
+ {
99
+ "_id": "ba3d82f7-8edc-408c-8212-c0d6634624ee",
100
+ "title": "Fully-configurable local success",
101
+ "publisher": "Dark Horse Comics",
102
+ "release_date": "1979-09-13T00:00:00",
103
+ "issues": 239,
104
+ "main_characters": ["Chad Pham", "Lindsay Anderson", "Carlos Burton"],
105
+ "genre": ["Adventure"]
106
+ },
107
+ {
108
+ "_id": "a6bc8677-22ab-415a-bfe2-731a9f887cb9",
109
+ "title": "Realigned zero-defect capability",
110
+ "publisher": "Marvel Comics",
111
+ "release_date": "2023-10-01T00:00:00",
112
+ "issues": 163,
113
+ "main_characters": ["Kevin Humphrey", "Maria Wright", "Virginia Watts"],
114
+ "genre": ["Fantasy", "Action"]
115
+ },
116
+ {
117
+ "_id": "fb986790-df22-4db4-8168-c76e9e9471f8",
118
+ "title": "Sharable bottom-line frame",
119
+ "publisher": "IDW Publishing",
120
+ "release_date": "2016-09-28T00:00:00",
121
+ "issues": 14,
122
+ "main_characters": ["Brian Vincent"],
123
+ "genre": ["Sci-Fi", "Fantasy"]
124
+ },
125
+ {
126
+ "_id": "700aa115-dc5a-4be6-b275-bfb943c95ee0",
127
+ "title": "Centralized next generation middleware",
128
+ "publisher": "Image Comics",
129
+ "release_date": "1970-04-16T00:00:00",
130
+ "issues": 5,
131
+ "main_characters": ["Joseph Cook"],
132
+ "genre": ["Fantasy"]
133
+ },
134
+ {
135
+ "_id": "7959187e-9693-43a1-ae2d-c168431fceb2",
136
+ "title": "Re-engineered heuristic array",
137
+ "publisher": "IDW Publishing",
138
+ "release_date": "2019-02-15T00:00:00",
139
+ "issues": 121,
140
+ "main_characters": ["Angelica Stein", "Benjamin Morris", "Jeremy Rice"],
141
+ "genre": ["Fantasy", "Action"]
142
+ },
143
+ {
144
+ "_id": "d6018445-5149-42e7-9d87-eb1b181ce20c",
145
+ "title": "Programmable transitional collaboration",
146
+ "publisher": "DC Comics",
147
+ "release_date": "1999-08-10T00:00:00",
148
+ "issues": 235,
149
+ "main_characters": [
150
+ "Joseph Cook",
151
+ "Cynthia Brown",
152
+ "Carlos Burton",
153
+ "Micheal Brown"
154
+ ],
155
+ "genre": ["Adventure"]
156
+ },
157
+ {
158
+ "_id": "055507ff-7a48-4df8-9ba9-7b6c10e11836",
159
+ "title": "Object-based dynamic knowledgebase",
160
+ "publisher": "Image Comics",
161
+ "release_date": "1993-02-24T00:00:00",
162
+ "issues": 189,
163
+ "main_characters": [
164
+ "Cristian Oneal",
165
+ "Brian Vincent",
166
+ "Holly Green",
167
+ "James Sanchez"
168
+ ],
169
+ "genre": ["Sci-Fi", "Fantasy"]
170
+ },
171
+ {
172
+ "_id": "1add2da3-68e6-48a3-9703-b593c9e0bf2e",
173
+ "title": "Enhanced asynchronous matrices",
174
+ "publisher": "DC Comics",
175
+ "release_date": "2001-03-01T00:00:00",
176
+ "issues": 176,
177
+ "main_characters": ["Justin Martinez", "Tammy Murphy"],
178
+ "genre": ["Action", "Fantasy"]
179
+ },
180
+ {
181
+ "_id": "c0fe2869-eb7d-4f09-a773-028387a54969",
182
+ "title": "Synergized maximized artificial intelligence",
183
+ "publisher": "DC Comics",
184
+ "release_date": "1976-09-05T00:00:00",
185
+ "issues": 68,
186
+ "main_characters": ["Christopher Elliott", "Maria Wright"],
187
+ "genre": ["Superhero", "Adventure"]
188
+ },
189
+ {
190
+ "_id": "c2fafbf6-5f71-4f31-9775-803e8c77e467",
191
+ "title": "Switchable bottom-line complexity",
192
+ "publisher": "Marvel Comics",
193
+ "release_date": "2012-08-12T00:00:00",
194
+ "issues": 156,
195
+ "main_characters": [
196
+ "Lindsay Anderson",
197
+ "Virginia Watts",
198
+ "Robert Logan",
199
+ "Margaret Hogan"
200
+ ],
201
+ "genre": ["Adventure"]
202
+ },
203
+ {
204
+ "_id": "f72be3a7-d4be-40a1-ad66-370b44759047",
205
+ "title": "Triple-buffered impactful customer loyalty",
206
+ "publisher": "Marvel Comics",
207
+ "release_date": "1976-09-18T00:00:00",
208
+ "issues": 275,
209
+ "main_characters": ["Sandra Moss", "Charles Blair", "Justin Martinez"],
210
+ "genre": ["Fantasy", "Action"]
211
+ },
212
+ {
213
+ "_id": "da5be16e-13e8-42d5-8954-bd89919395af",
214
+ "title": "Programmable 24/7 website",
215
+ "publisher": "DC Comics",
216
+ "release_date": "2023-11-06T00:00:00",
217
+ "issues": 278,
218
+ "main_characters": [
219
+ "Luis Callahan",
220
+ "Carlos Burton",
221
+ "Cristian Oneal",
222
+ "Michelle Valdez"
223
+ ],
224
+ "genre": ["Horror", "Fantasy"]
225
+ },
226
+ {
227
+ "_id": "92afc1e6-f703-4aa7-9866-3b62f2784fec",
228
+ "title": "Advanced incremental framework",
229
+ "publisher": "Image Comics",
230
+ "release_date": "2008-07-21T00:00:00",
231
+ "issues": 109,
232
+ "main_characters": ["Holly Green", "Diana Mata", "Julie Goodwin"],
233
+ "genre": ["Horror", "Sci-Fi"]
234
+ },
235
+ {
236
+ "_id": "fec61fdd-bddb-431a-b14a-d81601a47cf8",
237
+ "title": "Front-line coherent system engine",
238
+ "publisher": "DC Comics",
239
+ "release_date": "2012-04-27T00:00:00",
240
+ "issues": 297,
241
+ "main_characters": ["Joshua Hicks"],
242
+ "genre": ["Action", "Horror"]
243
+ },
244
+ {
245
+ "_id": "9d37d0d7-1adc-4f54-8790-30f13472520c",
246
+ "title": "Progressive systematic superstructure",
247
+ "publisher": "Image Comics",
248
+ "release_date": "1996-02-20T00:00:00",
249
+ "issues": 295,
250
+ "main_characters": ["Margaret Hogan", "Christopher Elliott", "Joseph Cook"],
251
+ "genre": ["Fantasy", "Adventure"]
252
+ },
253
+ {
254
+ "_id": "338a83ad-06fc-42e1-a605-60a192ce5643",
255
+ "title": "Implemented national help-desk",
256
+ "publisher": "DC Comics",
257
+ "release_date": "2015-05-11T00:00:00",
258
+ "issues": 257,
259
+ "main_characters": [
260
+ "Lindsay Anderson",
261
+ "James Sanchez",
262
+ "Julie Goodwin",
263
+ "Charles Blair"
264
+ ],
265
+ "genre": ["Action"]
266
+ },
267
+ {
268
+ "_id": "5b07c17b-4df9-4b72-9c3e-b51d93def1fb",
269
+ "title": "Down-sized impactful workforce",
270
+ "publisher": "IDW Publishing",
271
+ "release_date": "2024-06-19T00:00:00",
272
+ "issues": 259,
273
+ "main_characters": ["Debbie Green"],
274
+ "genre": ["Sci-Fi", "Superhero"]
275
+ },
276
+ {
277
+ "_id": "625b11a5-bb45-4837-9cd6-50bfe2e3396c",
278
+ "title": "Re-engineered leadingedge structure",
279
+ "publisher": "DC Comics",
280
+ "release_date": "2011-04-14T00:00:00",
281
+ "issues": 282,
282
+ "main_characters": [
283
+ "Larry Hensley",
284
+ "Joseph Cook",
285
+ "Brian Vincent",
286
+ "Sandra Moss"
287
+ ],
288
+ "genre": ["Adventure"]
289
+ },
290
+ {
291
+ "_id": "71b845f3-4416-430a-81eb-8c208f824365",
292
+ "title": "Cloned 3rdgeneration contingency",
293
+ "publisher": "Dark Horse Comics",
294
+ "release_date": "2002-07-11T00:00:00",
295
+ "issues": 238,
296
+ "main_characters": [
297
+ "Larry Hensley",
298
+ "Margaret Hogan",
299
+ "Holly Green",
300
+ "Joseph Bowman"
301
+ ],
302
+ "genre": ["Superhero", "Fantasy"]
303
+ },
304
+ {
305
+ "_id": "14dbf3a6-d258-4c96-8883-336b60bc2112",
306
+ "title": "Secured zero tolerance monitoring",
307
+ "publisher": "DC Comics",
308
+ "release_date": "1969-11-30T00:00:00",
309
+ "issues": 104,
310
+ "main_characters": ["Micheal Brown"],
311
+ "genre": ["Horror", "Superhero"]
312
+ },
313
+ {
314
+ "_id": "091e16d8-d50c-4e7d-9b3a-545cf2596738",
315
+ "title": "Automated bifurcated access",
316
+ "publisher": "Image Comics",
317
+ "release_date": "1990-01-24T00:00:00",
318
+ "issues": 74,
319
+ "main_characters": ["Robert Logan"],
320
+ "genre": ["Sci-Fi"]
321
+ },
322
+ {
323
+ "_id": "c47ec96a-4d6e-43ea-9bb5-00e4c8058b53",
324
+ "title": "Universal high-level pricing structure",
325
+ "publisher": "DC Comics",
326
+ "release_date": "1971-04-21T00:00:00",
327
+ "issues": 135,
328
+ "main_characters": ["Jeremy Rice", "Elizabeth Robinson", "James Sanchez"],
329
+ "genre": ["Action", "Sci-Fi"]
330
+ },
331
+ {
332
+ "_id": "d446a8ca-5d01-4be9-a061-027ef1f7bfc6",
333
+ "title": "Reduced optimizing strategy",
334
+ "publisher": "Dark Horse Comics",
335
+ "release_date": "1984-06-24T00:00:00",
336
+ "issues": 111,
337
+ "main_characters": ["Joshua Hicks", "Jeremy Rice", "Micheal Brown"],
338
+ "genre": ["Fantasy", "Superhero"]
339
+ },
340
+ {
341
+ "_id": "09c734ff-2bf0-4cb6-bd42-4232209c00c9",
342
+ "title": "Virtual non-volatile groupware",
343
+ "publisher": "DC Comics",
344
+ "release_date": "2013-05-22T00:00:00",
345
+ "issues": 13,
346
+ "main_characters": ["Luis Callahan", "Tammy Bishop", "Cynthia Brown"],
347
+ "genre": ["Action"]
348
+ },
349
+ {
350
+ "_id": "691034fa-ad52-413e-96a2-a9a319fffe7b",
351
+ "title": "Horizontal disintermediate extranet",
352
+ "publisher": "DC Comics",
353
+ "release_date": "2021-12-03T00:00:00",
354
+ "issues": 129,
355
+ "main_characters": ["Margaret Hogan"],
356
+ "genre": ["Action"]
357
+ },
358
+ {
359
+ "_id": "07942b5a-f7c4-4fc1-bdeb-7eb46b0d57f8",
360
+ "title": "Cross-platform discrete framework",
361
+ "publisher": "Dark Horse Comics",
362
+ "release_date": "2001-08-02T00:00:00",
363
+ "issues": 38,
364
+ "main_characters": ["James Sanchez", "Larry Hensley"],
365
+ "genre": ["Superhero"]
366
+ },
367
+ {
368
+ "_id": "05d637ed-3942-4276-a885-7b3363dd48e2",
369
+ "title": "Cross-platform regional info-mediaries",
370
+ "publisher": "Image Comics",
371
+ "release_date": "2005-03-30T00:00:00",
372
+ "issues": 150,
373
+ "main_characters": ["Carlos Burton"],
374
+ "genre": ["Superhero", "Fantasy"]
375
+ },
376
+ {
377
+ "_id": "88904f06-50a6-44f1-bccc-f379a9788611",
378
+ "title": "Mandatory 6thgeneration secured line",
379
+ "publisher": "Image Comics",
380
+ "release_date": "2021-06-27T00:00:00",
381
+ "issues": 262,
382
+ "main_characters": ["Luis Callahan"],
383
+ "genre": ["Sci-Fi", "Superhero"]
384
+ },
385
+ {
386
+ "_id": "fc961fd6-2ec6-43e5-beae-7f58a6c25d9c",
387
+ "title": "Exclusive interactive concept",
388
+ "publisher": "IDW Publishing",
389
+ "release_date": "1969-06-03T00:00:00",
390
+ "issues": 264,
391
+ "main_characters": ["Scott Garcia", "Joseph Bowman"],
392
+ "genre": ["Fantasy", "Superhero"]
393
+ },
394
+ {
395
+ "_id": "481a3ea6-9629-4fe6-8a5a-eba846f0e62c",
396
+ "title": "Focused intermediate methodology",
397
+ "publisher": "DC Comics",
398
+ "release_date": "2004-03-19T00:00:00",
399
+ "issues": 210,
400
+ "main_characters": [
401
+ "Justin Martinez",
402
+ "Julie Goodwin",
403
+ "Benjamin Morris",
404
+ "Virginia Watts"
405
+ ],
406
+ "genre": ["Adventure", "Action"]
407
+ },
408
+ {
409
+ "_id": "6bab6bcd-2f6b-4dfb-a030-d63b32fc6250",
410
+ "title": "Right-sized contextually-based toolset",
411
+ "publisher": "IDW Publishing",
412
+ "release_date": "2007-12-27T00:00:00",
413
+ "issues": 117,
414
+ "main_characters": ["Debbie Green", "Christopher Elliott", "Joshua Hicks"],
415
+ "genre": ["Sci-Fi", "Action"]
416
+ }
417
+ ]