@skybridge/test 0.0.0-dev.b6f5d60 → 0.0.0-dev.b807968
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -15
- package/dist/chat.d.ts +26 -9
- package/dist/chat.d.ts.map +1 -1
- package/dist/chat.js +103 -34
- package/dist/chat.js.map +1 -1
- package/dist/eval-middlewares.d.ts +12 -0
- package/dist/eval-middlewares.d.ts.map +1 -0
- package/dist/eval-middlewares.js +18 -0
- package/dist/eval-middlewares.js.map +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/input-files.d.ts +14 -0
- package/dist/input-files.d.ts.map +1 -0
- package/dist/input-files.js +126 -0
- package/dist/input-files.js.map +1 -0
- package/dist/matchers/index.d.ts +4 -0
- package/dist/matchers/index.d.ts.map +1 -0
- package/dist/matchers/index.js +2 -0
- package/dist/matchers/index.js.map +1 -0
- package/dist/matchers/matchers.d.ts +54 -0
- package/dist/matchers/matchers.d.ts.map +1 -0
- package/dist/matchers/matchers.js +258 -0
- package/dist/matchers/matchers.js.map +1 -0
- package/dist/matchers/types.d.ts +44 -0
- package/dist/matchers/types.d.ts.map +1 -0
- package/dist/matchers/types.js +2 -0
- package/dist/matchers/types.js.map +1 -0
- package/dist/middleware/list-tools.d.ts +4 -0
- package/dist/middleware/list-tools.d.ts.map +1 -0
- package/dist/middleware/list-tools.js +67 -0
- package/dist/middleware/list-tools.js.map +1 -0
- package/dist/middleware/resolve-files.d.ts +4 -0
- package/dist/middleware/resolve-files.d.ts.map +1 -0
- package/dist/middleware/resolve-files.js +13 -0
- package/dist/middleware/resolve-files.js.map +1 -0
- package/dist/middleware/stub-tool.d.ts +4 -0
- package/dist/middleware/stub-tool.d.ts.map +1 -0
- package/dist/middleware/stub-tool.js +16 -0
- package/dist/middleware/stub-tool.js.map +1 -0
- package/dist/session-registry.d.ts +49 -9
- package/dist/session-registry.d.ts.map +1 -1
- package/dist/session-registry.js +48 -17
- package/dist/session-registry.js.map +1 -1
- package/package.json +14 -7
|
@@ -1,20 +1,60 @@
|
|
|
1
1
|
import type { LanguageModel } from "ai";
|
|
2
|
+
import type { Skybridge, ToolDef, ToolInput, ToolNames } from "skybridge/server";
|
|
2
3
|
import { Chat } from "./chat.js";
|
|
4
|
+
/** The Skybridge surface used by an in-process eval session. */
|
|
5
|
+
export type EvalApp = Pick<Skybridge, "$types" | "createServerInstance">;
|
|
3
6
|
/**
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
7
|
+
* The identity an in-process session claims, handed to the app as the
|
|
8
|
+
* request's `authInfo`. Mirrors the SDK's `AuthInfo` structurally; `extra`
|
|
9
|
+
* carries whatever claims the app's verifier would have produced.
|
|
10
|
+
*/
|
|
11
|
+
export interface EvalIdentity {
|
|
12
|
+
token: string;
|
|
13
|
+
clientId: string;
|
|
14
|
+
scopes: string[];
|
|
15
|
+
expiresAt?: number;
|
|
16
|
+
resource?: URL;
|
|
17
|
+
extra?: Record<string, unknown>;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Results the runner answers with instead of calling the app, keyed by tool
|
|
21
|
+
* name and typed against the project's own registry. A stub may be async, and
|
|
22
|
+
* one that returns `undefined` falls through to the real server, so a scenario
|
|
23
|
+
* can pin one set of arguments and leave the rest live.
|
|
12
24
|
*/
|
|
13
|
-
export
|
|
25
|
+
export type ToolStubs<App = Skybridge<Record<string, ToolDef<Record<string, unknown>>>>> = {
|
|
26
|
+
[Name in ToolNames<App>]?: (args: ToolInput<App, Name>) => unknown;
|
|
27
|
+
};
|
|
28
|
+
interface StartOptions {
|
|
14
29
|
/** Any AI SDK model instance, built by the project. */
|
|
15
30
|
model: LanguageModel;
|
|
16
31
|
systemPrompt?: string;
|
|
17
32
|
temperature?: number;
|
|
18
33
|
maxSteps?: number;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Opens a fresh MCP session and conversation against the app under test. The
|
|
37
|
+
* app is served in-process by a handler built on `app.createServerInstance`
|
|
38
|
+
* for this conversation alone, and both are closed when the current test
|
|
39
|
+
* finishes, so scenarios never leak sessions and never write teardown, and
|
|
40
|
+
* tests that run concurrently cannot close each other's sessions.
|
|
41
|
+
*
|
|
42
|
+
* The assertions are inferred from the app value itself, so `expect.chat` gets
|
|
43
|
+
* the project's tool names and argument shapes with no type parameter.
|
|
44
|
+
*
|
|
45
|
+
* `stubs` answers a tool from the scenario instead of the app, so a case that
|
|
46
|
+
* depends on today's date or on a live catalogue still reads the same weeks
|
|
47
|
+
* later. A stubbed call bypasses the application pipeline, and shows up in
|
|
48
|
+
* `chat.toolCalls` like any other. See {@link ToolStubs}.
|
|
49
|
+
*
|
|
50
|
+
* `authInfo` claims an identity for the session: the app's per-tool scheme and
|
|
51
|
+
* scope enforcement runs against it for real, only token verification is
|
|
52
|
+
* skipped. Omit it to exercise the anonymous path, challenges included.
|
|
53
|
+
*/
|
|
54
|
+
export declare function start<App extends EvalApp>(options: StartOptions & {
|
|
55
|
+
app: App;
|
|
56
|
+
authInfo?: EvalIdentity;
|
|
57
|
+
stubs?: ToolStubs<App>;
|
|
19
58
|
}): Promise<Chat<App>>;
|
|
59
|
+
export {};
|
|
20
60
|
//# sourceMappingURL=session-registry.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxC,OAAO,KAAK,EACV,SAAS,EACT,OAAO,EACP,SAAS,EACT,SAAS,EACV,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,gEAAgE;AAChE,MAAM,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,QAAQ,GAAG,sBAAsB,CAAC,CAAC;AAEzE;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,GAAG,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED;;;;;GAKG;AACH,MAAM,MAAM,SAAS,CACnB,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAC/D;KACD,IAAI,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,OAAO;CACnE,CAAC;AAEF,UAAU,YAAY;IACpB,uDAAuD;IACvD,KAAK,EAAE,aAAa,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAUD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,KAAK,CAAC,GAAG,SAAS,OAAO,EAC7C,OAAO,EAAE,YAAY,GAAG;IACtB,GAAG,EAAE,GAAG,CAAC;IACT,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,KAAK,CAAC,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC;CACxB,GACA,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAoCpB"}
|
package/dist/session-registry.js
CHANGED
|
@@ -1,28 +1,59 @@
|
|
|
1
|
+
import { createMcpHandler } from "@modelcontextprotocol/server";
|
|
1
2
|
import { inject, onTestFinished } from "vitest";
|
|
2
3
|
import { Chat } from "./chat.js";
|
|
4
|
+
import { createEvalMiddlewares } from "./eval-middlewares.js";
|
|
5
|
+
function sharedDefaults() {
|
|
6
|
+
try {
|
|
7
|
+
return inject("skybridgeEvals");
|
|
8
|
+
}
|
|
9
|
+
catch {
|
|
10
|
+
return undefined;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
3
13
|
/**
|
|
4
|
-
* Opens a fresh MCP session and conversation against the
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
14
|
+
* Opens a fresh MCP session and conversation against the app under test. The
|
|
15
|
+
* app is served in-process by a handler built on `app.createServerInstance`
|
|
16
|
+
* for this conversation alone, and both are closed when the current test
|
|
17
|
+
* finishes, so scenarios never leak sessions and never write teardown, and
|
|
18
|
+
* tests that run concurrently cannot close each other's sessions.
|
|
8
19
|
*
|
|
9
|
-
* The
|
|
10
|
-
*
|
|
11
|
-
*
|
|
20
|
+
* The assertions are inferred from the app value itself, so `expect.chat` gets
|
|
21
|
+
* the project's tool names and argument shapes with no type parameter.
|
|
22
|
+
*
|
|
23
|
+
* `stubs` answers a tool from the scenario instead of the app, so a case that
|
|
24
|
+
* depends on today's date or on a live catalogue still reads the same weeks
|
|
25
|
+
* later. A stubbed call bypasses the application pipeline, and shows up in
|
|
26
|
+
* `chat.toolCalls` like any other. See {@link ToolStubs}.
|
|
27
|
+
*
|
|
28
|
+
* `authInfo` claims an identity for the session: the app's per-tool scheme and
|
|
29
|
+
* scope enforcement runs against it for real, only token verification is
|
|
30
|
+
* skipped. Omit it to exercise the anonymous path, challenges included.
|
|
12
31
|
*/
|
|
13
32
|
export async function start(options) {
|
|
14
|
-
const
|
|
15
|
-
|
|
16
|
-
|
|
33
|
+
const config = sharedDefaults();
|
|
34
|
+
const { app, authInfo } = options;
|
|
35
|
+
const middleware = createEvalMiddlewares((options.stubs ?? {}));
|
|
36
|
+
const handler = createMcpHandler(() => app.createServerInstance([
|
|
37
|
+
{ filter: "tools/list", handler: middleware.listTools },
|
|
38
|
+
{ filter: "tools/call", handler: middleware.resolveFiles },
|
|
39
|
+
{ filter: "tools/call", handler: middleware.stubTool },
|
|
40
|
+
]));
|
|
41
|
+
let chat;
|
|
42
|
+
try {
|
|
43
|
+
chat = await Chat.open({
|
|
44
|
+
model: options.model,
|
|
45
|
+
temperature: options.temperature ?? config?.temperature,
|
|
46
|
+
systemPrompt: options.systemPrompt ?? config?.systemPrompt,
|
|
47
|
+
maxSteps: options.maxSteps ?? config?.maxSteps,
|
|
48
|
+
}, (url, init) => handler.fetch(new Request(url, init), authInfo === undefined ? undefined : { authInfo }), middleware, () => handler.close());
|
|
49
|
+
}
|
|
50
|
+
catch (error) {
|
|
51
|
+
await Promise.allSettled([middleware.files.close(), handler.close()]);
|
|
52
|
+
throw error;
|
|
17
53
|
}
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
model: options.model,
|
|
21
|
-
temperature: options.temperature ?? config?.temperature,
|
|
22
|
-
systemPrompt: options.systemPrompt ?? config?.systemPrompt,
|
|
23
|
-
maxSteps: options.maxSteps ?? config?.maxSteps,
|
|
54
|
+
onTestFinished(async () => {
|
|
55
|
+
await chat.close();
|
|
24
56
|
});
|
|
25
|
-
onTestFinished(() => chat.close());
|
|
26
57
|
return chat;
|
|
27
58
|
}
|
|
28
59
|
//# sourceMappingURL=session-registry.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAQhE,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAuC9D,SAAS,cAAc;IACrB,IAAI,CAAC;QACH,OAAO,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,OAIC;IAED,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;IAChC,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAClC,MAAM,UAAU,GAAG,qBAAqB,CAAC,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,CAAc,CAAC,CAAC;IAC7E,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,EAAE,CACpC,GAAG,CAAC,oBAAoB,CAAC;QACvB,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,UAAU,CAAC,SAAS,EAAE;QACvD,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,UAAU,CAAC,YAAY,EAAE;QAC1D,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,UAAU,CAAC,QAAQ,EAAE;KACvD,CAAC,CACH,CAAC;IACF,IAAI,IAAe,CAAC;IACpB,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CACpB;YACE,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,MAAM,EAAE,WAAW;YACvD,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,MAAM,EAAE,YAAY;YAC1D,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,MAAM,EAAE,QAAQ;SAC/C,EACD,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CACZ,OAAO,CAAC,KAAK,CACX,IAAI,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,EACtB,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAClD,EACH,UAAU,EACV,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,CACtB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACtE,MAAM,KAAK,CAAC;IACd,CAAC;IACD,cAAc,CAAC,KAAK,IAAI,EAAE;QACxB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skybridge/test",
|
|
3
|
-
"version": "0.0.0-dev.
|
|
4
|
-
"description": "Eval runner for Skybridge MCP
|
|
3
|
+
"version": "0.0.0-dev.b807968",
|
|
4
|
+
"description": "Eval runner for Skybridge MCP Apps: does a prompt reach the right tool with the right arguments",
|
|
5
5
|
"license": "ISC",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"repository": {
|
|
@@ -12,26 +12,33 @@
|
|
|
12
12
|
".": {
|
|
13
13
|
"types": "./dist/index.d.ts",
|
|
14
14
|
"default": "./dist/index.js"
|
|
15
|
+
},
|
|
16
|
+
"./matchers": {
|
|
17
|
+
"types": "./dist/matchers/index.d.ts",
|
|
18
|
+
"default": "./dist/matchers/index.js"
|
|
15
19
|
}
|
|
16
20
|
},
|
|
17
21
|
"files": [
|
|
18
22
|
"dist"
|
|
19
23
|
],
|
|
20
24
|
"dependencies": {
|
|
21
|
-
"@modelcontextprotocol/
|
|
22
|
-
"@
|
|
25
|
+
"@modelcontextprotocol/client": "^2.0.0",
|
|
26
|
+
"@modelcontextprotocol/server": "^2.0.0"
|
|
23
27
|
},
|
|
24
28
|
"peerDependencies": {
|
|
25
|
-
"
|
|
29
|
+
"@skybridge/vite-plugin": "^2.0.0",
|
|
30
|
+
"ai": "^7.0.0",
|
|
26
31
|
"skybridge": "^2.0.0",
|
|
27
32
|
"vitest": "^4.1.0"
|
|
28
33
|
},
|
|
29
34
|
"devDependencies": {
|
|
30
35
|
"@types/node": "^24.13.3",
|
|
31
|
-
"ai": "^
|
|
36
|
+
"ai": "^7.0.0",
|
|
32
37
|
"typescript": "^5.9.3",
|
|
33
38
|
"vitest": "^4.1.10",
|
|
34
|
-
"
|
|
39
|
+
"zod": "^4.4.3",
|
|
40
|
+
"@skybridge/vite-plugin": "0.0.0-dev.b807968",
|
|
41
|
+
"skybridge": "0.0.0-dev.b807968"
|
|
35
42
|
},
|
|
36
43
|
"engines": {
|
|
37
44
|
"node": ">=22.23.1"
|