@skybridge/test 0.0.0-dev.7b38d74 → 0.0.0-dev.7cd4271

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/README.md +32 -2
  2. package/dist/chat.d.ts +16 -6
  3. package/dist/chat.d.ts.map +1 -1
  4. package/dist/chat.js +121 -24
  5. package/dist/chat.js.map +1 -1
  6. package/dist/eval-middlewares.d.ts +13 -0
  7. package/dist/eval-middlewares.d.ts.map +1 -0
  8. package/dist/eval-middlewares.js +18 -0
  9. package/dist/eval-middlewares.js.map +1 -0
  10. package/dist/index.d.ts +2 -2
  11. package/dist/index.d.ts.map +1 -1
  12. package/dist/index.js.map +1 -1
  13. package/dist/input-files.d.ts +14 -0
  14. package/dist/input-files.d.ts.map +1 -0
  15. package/dist/input-files.js +126 -0
  16. package/dist/input-files.js.map +1 -0
  17. package/dist/matchers/index.d.ts +4 -0
  18. package/dist/matchers/index.d.ts.map +1 -0
  19. package/dist/matchers/index.js +2 -0
  20. package/dist/matchers/index.js.map +1 -0
  21. package/dist/matchers/matchers.d.ts +54 -0
  22. package/dist/matchers/matchers.d.ts.map +1 -0
  23. package/dist/matchers/matchers.js +258 -0
  24. package/dist/matchers/matchers.js.map +1 -0
  25. package/dist/matchers/types.d.ts +44 -0
  26. package/dist/matchers/types.d.ts.map +1 -0
  27. package/dist/matchers/types.js +2 -0
  28. package/dist/matchers/types.js.map +1 -0
  29. package/dist/middleware/list-tools.d.ts +4 -0
  30. package/dist/middleware/list-tools.d.ts.map +1 -0
  31. package/dist/middleware/list-tools.js +67 -0
  32. package/dist/middleware/list-tools.js.map +1 -0
  33. package/dist/middleware/resolve-files.d.ts +4 -0
  34. package/dist/middleware/resolve-files.d.ts.map +1 -0
  35. package/dist/middleware/resolve-files.js +13 -0
  36. package/dist/middleware/resolve-files.js.map +1 -0
  37. package/dist/middleware/stub-tool.d.ts +4 -0
  38. package/dist/middleware/stub-tool.d.ts.map +1 -0
  39. package/dist/middleware/stub-tool.js +16 -0
  40. package/dist/middleware/stub-tool.js.map +1 -0
  41. package/dist/session-registry.d.ts +18 -13
  42. package/dist/session-registry.d.ts.map +1 -1
  43. package/dist/session-registry.js +33 -7
  44. package/dist/session-registry.js.map +1 -1
  45. package/package.json +12 -8
@@ -1,6 +1,7 @@
1
1
  import { createMcpHandler } from "@modelcontextprotocol/server";
2
2
  import { inject, onTestFinished } from "vitest";
3
3
  import { Chat } from "./chat.js";
4
+ import { EvalMiddlewares } from "./eval-middlewares.js";
4
5
  function sharedDefaults() {
5
6
  try {
6
7
  return inject("skybridgeEvals");
@@ -19,6 +20,11 @@ function sharedDefaults() {
19
20
  * The assertions are inferred from the app value itself, so `expect.chat` gets
20
21
  * the project's tool names and argument shapes with no type parameter.
21
22
  *
23
+ * `stubs` answers a tool from the scenario instead of the app, so a case that
24
+ * depends on today's date or on a live catalogue still reads the same weeks
25
+ * later. A stubbed call bypasses the application pipeline, and shows up in
26
+ * `chat.toolCalls` like any other. See {@link ToolStubs}.
27
+ *
22
28
  * `authInfo` claims an identity for the session: the app's per-tool scheme and
23
29
  * scope enforcement runs against it for real, only token verification is
24
30
  * skipped. Omit it to exercise the anonymous path, challenges included.
@@ -26,16 +32,36 @@ function sharedDefaults() {
26
32
  export async function start(options) {
27
33
  const config = sharedDefaults();
28
34
  const { app, authInfo } = options;
35
+ const middleware = new EvalMiddlewares((options.stubs ?? {}));
36
+ app
37
+ .withMcpMiddleware({
38
+ filter: "tools/list",
39
+ handler: middleware.listTools,
40
+ })
41
+ .withMcpMiddleware({
42
+ filter: "tools/call",
43
+ handler: middleware.resolveFiles,
44
+ })
45
+ .withMcpMiddleware({
46
+ filter: "tools/call",
47
+ handler: middleware.stubTool,
48
+ });
29
49
  const handler = createMcpHandler(() => app.createServerInstance());
30
- const chat = await Chat.open({
31
- model: options.model,
32
- temperature: options.temperature ?? config?.temperature,
33
- systemPrompt: options.systemPrompt ?? config?.systemPrompt,
34
- maxSteps: options.maxSteps ?? config?.maxSteps,
35
- }, (url, init) => handler.fetch(new Request(url, init), authInfo === undefined ? undefined : { authInfo }));
50
+ let chat;
51
+ try {
52
+ chat = await Chat.open({
53
+ model: options.model,
54
+ temperature: options.temperature ?? config?.temperature,
55
+ systemPrompt: options.systemPrompt ?? config?.systemPrompt,
56
+ maxSteps: options.maxSteps ?? config?.maxSteps,
57
+ }, (url, init) => handler.fetch(new Request(url, init), authInfo === undefined ? undefined : { authInfo }), middleware, () => handler.close());
58
+ }
59
+ catch (error) {
60
+ await Promise.allSettled([middleware.files.close(), handler.close()]);
61
+ throw error;
62
+ }
36
63
  onTestFinished(async () => {
37
64
  await chat.close();
38
- await handler.close();
39
65
  });
40
66
  return chat;
41
67
  }
@@ -1 +1 @@
1
- {"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAe,MAAM,8BAA8B,CAAC;AAE7E,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAmCjC,SAAS,cAAc;IACrB,IAAI,CAAC;QACH,OAAO,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,OAA6D;IAE7D,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;IAChC,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAClC,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,oBAAoB,EAAE,CAAC,CAAC;IAEnE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAC1B;QACE,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,MAAM,EAAE,WAAW;QACvD,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,MAAM,EAAE,YAAY;QAC1D,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,MAAM,EAAE,QAAQ;KAC/C,EACD,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CACZ,OAAO,CAAC,KAAK,CACX,IAAI,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,EACtB,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAClD,CACJ,CAAC;IAEF,cAAc,CAAC,KAAK,IAAI,EAAE;QACxB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC"}
1
+ {"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAQhE,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AA0CxD,SAAS,cAAc;IACrB,IAAI,CAAC;QACH,OAAO,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,OAIC;IAED,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;IAChC,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAClC,MAAM,UAAU,GAAG,IAAI,eAAe,CAAC,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,CAAc,CAAC,CAAC;IAC3E,GAAG;SACA,iBAAiB,CAAC;QACjB,MAAM,EAAE,YAAY;QACpB,OAAO,EAAE,UAAU,CAAC,SAAS;KAC9B,CAAC;SACD,iBAAiB,CAAC;QACjB,MAAM,EAAE,YAAY;QACpB,OAAO,EAAE,UAAU,CAAC,YAAY;KACjC,CAAC;SACD,iBAAiB,CAAC;QACjB,MAAM,EAAE,YAAY;QACpB,OAAO,EAAE,UAAU,CAAC,QAAQ;KAC7B,CAAC,CAAC;IACL,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,oBAAoB,EAAE,CAAC,CAAC;IACnE,IAAI,IAAe,CAAC;IACpB,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CACpB;YACE,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,MAAM,EAAE,WAAW;YACvD,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,MAAM,EAAE,YAAY;YAC1D,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,MAAM,EAAE,QAAQ;SAC/C,EACD,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CACZ,OAAO,CAAC,KAAK,CACX,IAAI,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,EACtB,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAClD,EACH,UAAU,EACV,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,CACtB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACtE,MAAM,KAAK,CAAC;IACd,CAAC;IACD,cAAc,CAAC,KAAK,IAAI,EAAE;QACxB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC"}
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@skybridge/test",
3
- "version": "0.0.0-dev.7b38d74",
4
- "description": "Eval runner for Skybridge MCP apps: does a prompt reach the right tool with the right arguments",
3
+ "version": "0.0.0-dev.7cd4271",
4
+ "description": "Eval runner for Skybridge MCP Apps: does a prompt reach the right tool with the right arguments",
5
5
  "license": "ISC",
6
6
  "type": "module",
7
7
  "repository": {
@@ -12,6 +12,10 @@
12
12
  ".": {
13
13
  "types": "./dist/index.d.ts",
14
14
  "default": "./dist/index.js"
15
+ },
16
+ "./matchers": {
17
+ "types": "./dist/matchers/index.d.ts",
18
+ "default": "./dist/matchers/index.js"
15
19
  }
16
20
  },
17
21
  "files": [
@@ -22,19 +26,19 @@
22
26
  "@modelcontextprotocol/server": "^2.0.0"
23
27
  },
24
28
  "peerDependencies": {
25
- "@skybridge/vite-plugin": "^2.0.0-beta.0",
26
- "ai": "^5.0.0",
27
- "skybridge": "^2.0.0-beta.0",
29
+ "@skybridge/vite-plugin": "^2.0.0",
30
+ "ai": "^7.0.0",
31
+ "skybridge": "^2.0.0",
28
32
  "vitest": "^4.1.0"
29
33
  },
30
34
  "devDependencies": {
31
35
  "@types/node": "^24.13.3",
32
- "ai": "^5.0.0",
36
+ "ai": "^7.0.0",
33
37
  "typescript": "^5.9.3",
34
38
  "vitest": "^4.1.10",
35
39
  "zod": "^4.4.3",
36
- "@skybridge/vite-plugin": "0.0.0-dev.7b38d74",
37
- "skybridge": "0.0.0-dev.7b38d74"
40
+ "@skybridge/vite-plugin": "0.0.0-dev.7cd4271",
41
+ "skybridge": "0.0.0-dev.7cd4271"
38
42
  },
39
43
  "engines": {
40
44
  "node": ">=22.23.1"