@skybridge/test 0.0.0-dev.df0cfdd → 0.0.0-dev.e3d3217
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -7
- package/dist/chat.d.ts +18 -28
- package/dist/chat.d.ts.map +1 -1
- package/dist/chat.js +37 -15
- package/dist/chat.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +0 -1
- package/dist/index.js.map +1 -1
- package/dist/matchers/index.d.ts +4 -0
- package/dist/matchers/index.d.ts.map +1 -0
- package/dist/matchers/index.js +2 -0
- package/dist/matchers/index.js.map +1 -0
- package/dist/{matchers.d.ts → matchers/matchers.d.ts} +14 -2
- package/dist/matchers/matchers.d.ts.map +1 -0
- package/dist/matchers/matchers.js +208 -0
- package/dist/matchers/matchers.js.map +1 -0
- package/dist/matchers/types.d.ts +44 -0
- package/dist/matchers/types.d.ts.map +1 -0
- package/dist/matchers/types.js +2 -0
- package/dist/matchers/types.js.map +1 -0
- package/dist/session-registry.d.ts +48 -9
- package/dist/session-registry.d.ts.map +1 -1
- package/dist/session-registry.js +33 -16
- package/dist/session-registry.js.map +1 -1
- package/package.json +12 -12
- package/dist/config.d.ts +0 -13
- package/dist/config.d.ts.map +0 -1
- package/dist/config.js +0 -2
- package/dist/config.js.map +0 -1
- package/dist/global-setup.d.ts +0 -3
- package/dist/global-setup.d.ts.map +0 -1
- package/dist/global-setup.js +0 -24
- package/dist/global-setup.js.map +0 -1
- package/dist/matchers.d.ts.map +0 -1
- package/dist/matchers.js +0 -66
- package/dist/matchers.js.map +0 -1
- package/dist/model.d.ts +0 -9
- package/dist/model.d.ts.map +0 -1
- package/dist/model.js +0 -24
- package/dist/model.js.map +0 -1
- package/dist/plugin.d.ts +0 -10
- package/dist/plugin.d.ts.map +0 -1
- package/dist/plugin.js +0 -32
- package/dist/plugin.js.map +0 -1
- package/dist/server-harness.d.ts +0 -18
- package/dist/server-harness.d.ts.map +0 -1
- package/dist/server-harness.js +0 -79
- package/dist/server-harness.js.map +0 -1
- package/dist/setup.d.ts +0 -2
- package/dist/setup.d.ts.map +0 -1
- package/dist/setup.js +0 -2
- package/dist/setup.js.map +0 -1
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { EvalsOptions } from "@skybridge/vite-plugin";
|
|
2
|
+
import type { LanguageModel } from "ai";
|
|
3
|
+
import type { ToolInput, ToolNames } from "skybridge/server";
|
|
4
|
+
/**
|
|
5
|
+
* One recorded call, typed against the project's own registry: `name` is a
|
|
6
|
+
* union of the project's tool names and `arguments` is discriminated on it.
|
|
7
|
+
* `failed` carries the server's reason when the call did not go through.
|
|
8
|
+
*/
|
|
9
|
+
export type ToolCall<App> = {
|
|
10
|
+
[Name in ToolNames<App>]: {
|
|
11
|
+
name: Name;
|
|
12
|
+
arguments: ToolInput<App, Name>;
|
|
13
|
+
failed?: string;
|
|
14
|
+
};
|
|
15
|
+
}[ToolNames<App>];
|
|
16
|
+
/**
|
|
17
|
+
* One thing that happened in the conversation. A `tool` turn carries the call
|
|
18
|
+
* itself, so its failure reason lives on the call and `result` holds whatever
|
|
19
|
+
* the server sent back when it went through.
|
|
20
|
+
*/
|
|
21
|
+
export type Turn<App> = {
|
|
22
|
+
role: "user";
|
|
23
|
+
text: string;
|
|
24
|
+
} | {
|
|
25
|
+
role: "assistant";
|
|
26
|
+
text: string;
|
|
27
|
+
} | {
|
|
28
|
+
role: "tool";
|
|
29
|
+
call: ToolCall<App>;
|
|
30
|
+
result?: unknown;
|
|
31
|
+
};
|
|
32
|
+
/** The part of a conversation the matchers assert on. */
|
|
33
|
+
export interface ChatLike<App> {
|
|
34
|
+
/** Type-only anchor so `expect.chat` infers `App` from the conversation. */
|
|
35
|
+
readonly $app?: App;
|
|
36
|
+
readonly turns: Turn<App>[];
|
|
37
|
+
readonly model: LanguageModel;
|
|
38
|
+
}
|
|
39
|
+
declare module "vitest" {
|
|
40
|
+
interface ProvidedContext {
|
|
41
|
+
skybridgeEvals: EvalsOptions | undefined;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/matchers/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxC,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D;;;;GAIG;AACH,MAAM,MAAM,QAAQ,CAAC,GAAG,IAAI;KACzB,IAAI,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG;QACxB,IAAI,EAAE,IAAI,CAAC;QACX,SAAS,EAAE,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAChC,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB;CACF,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;AAElB;;;;GAIG;AACH,MAAM,MAAM,IAAI,CAAC,GAAG,IAChB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC9B;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;IAAC,MAAM,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC;AAE5D,yDAAyD;AACzD,MAAM,WAAW,QAAQ,CAAC,GAAG;IAC3B,4EAA4E;IAC5E,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;CAC/B;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,eAAe;QACvB,cAAc,EAAE,YAAY,GAAG,SAAS,CAAC;KAC1C;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/matchers/types.ts"],"names":[],"mappings":""}
|
|
@@ -1,16 +1,55 @@
|
|
|
1
|
+
import { type Server } from "@modelcontextprotocol/server";
|
|
2
|
+
import type { LanguageModel } from "ai";
|
|
1
3
|
import { Chat } from "./chat.js";
|
|
2
4
|
/**
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
5
|
+
* The minimum a Skybridge app has to expose to be served in-process: the
|
|
6
|
+
* `$types` marker the assertions infer tool names from, and the per-request
|
|
7
|
+
* server builder the session's handler is built on. Structural on purpose, so
|
|
8
|
+
* this package needs no runtime dependency on `skybridge`.
|
|
9
|
+
*/
|
|
10
|
+
export interface EvalApp {
|
|
11
|
+
readonly $types: {
|
|
12
|
+
readonly tools: object;
|
|
13
|
+
};
|
|
14
|
+
createServerInstance(): Promise<Server>;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* The identity an in-process session claims, handed to the app as the
|
|
18
|
+
* request's `authInfo`. Mirrors the SDK's `AuthInfo` structurally; `extra`
|
|
19
|
+
* carries whatever claims the app's verifier would have produced.
|
|
11
20
|
*/
|
|
12
|
-
export
|
|
21
|
+
export interface EvalIdentity {
|
|
22
|
+
token: string;
|
|
23
|
+
clientId: string;
|
|
24
|
+
scopes: string[];
|
|
25
|
+
expiresAt?: number;
|
|
26
|
+
resource?: URL;
|
|
27
|
+
extra?: Record<string, unknown>;
|
|
28
|
+
}
|
|
29
|
+
interface StartOptions {
|
|
30
|
+
/** Any AI SDK model instance, built by the project. */
|
|
31
|
+
model: LanguageModel;
|
|
13
32
|
systemPrompt?: string;
|
|
14
33
|
temperature?: number;
|
|
34
|
+
maxSteps?: number;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Opens a fresh MCP session and conversation against the app under test. The
|
|
38
|
+
* app is served in-process by a handler built on `app.createServerInstance`
|
|
39
|
+
* for this conversation alone, and both are closed when the current test
|
|
40
|
+
* finishes, so scenarios never leak sessions and never write teardown, and
|
|
41
|
+
* tests that run concurrently cannot close each other's sessions.
|
|
42
|
+
*
|
|
43
|
+
* The assertions are inferred from the app value itself, so `expect.chat` gets
|
|
44
|
+
* the project's tool names and argument shapes with no type parameter.
|
|
45
|
+
*
|
|
46
|
+
* `authInfo` claims an identity for the session: the app's per-tool scheme and
|
|
47
|
+
* scope enforcement runs against it for real, only token verification is
|
|
48
|
+
* skipped. Omit it to exercise the anonymous path, challenges included.
|
|
49
|
+
*/
|
|
50
|
+
export declare function start<App extends EvalApp>(options: StartOptions & {
|
|
51
|
+
app: App;
|
|
52
|
+
authInfo?: EvalIdentity;
|
|
15
53
|
}): Promise<Chat<App>>;
|
|
54
|
+
export {};
|
|
16
55
|
//# sourceMappingURL=session-registry.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAoB,KAAK,MAAM,EAAE,MAAM,8BAA8B,CAAC;AAC7E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAExC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC;;;;;GAKG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5C,oBAAoB,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;CACzC;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,GAAG,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED,UAAU,YAAY;IACpB,uDAAuD;IACvD,KAAK,EAAE,aAAa,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAUD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,KAAK,CAAC,GAAG,SAAS,OAAO,EAC7C,OAAO,EAAE,YAAY,GAAG;IAAE,GAAG,EAAE,GAAG,CAAC;IAAC,QAAQ,CAAC,EAAE,YAAY,CAAA;CAAE,GAC5D,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAwBpB"}
|
package/dist/session-registry.js
CHANGED
|
@@ -1,25 +1,42 @@
|
|
|
1
|
+
import { createMcpHandler } from "@modelcontextprotocol/server";
|
|
1
2
|
import { inject, onTestFinished } from "vitest";
|
|
2
3
|
import { Chat } from "./chat.js";
|
|
3
|
-
|
|
4
|
+
function sharedDefaults() {
|
|
5
|
+
try {
|
|
6
|
+
return inject("skybridgeEvals");
|
|
7
|
+
}
|
|
8
|
+
catch {
|
|
9
|
+
return undefined;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
4
12
|
/**
|
|
5
|
-
* Opens a fresh MCP session and conversation against the
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
13
|
+
* Opens a fresh MCP session and conversation against the app under test. The
|
|
14
|
+
* app is served in-process by a handler built on `app.createServerInstance`
|
|
15
|
+
* for this conversation alone, and both are closed when the current test
|
|
16
|
+
* finishes, so scenarios never leak sessions and never write teardown, and
|
|
17
|
+
* tests that run concurrently cannot close each other's sessions.
|
|
18
|
+
*
|
|
19
|
+
* The assertions are inferred from the app value itself, so `expect.chat` gets
|
|
20
|
+
* the project's tool names and argument shapes with no type parameter.
|
|
9
21
|
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
22
|
+
* `authInfo` claims an identity for the session: the app's per-tool scheme and
|
|
23
|
+
* scope enforcement runs against it for real, only token verification is
|
|
24
|
+
* skipped. Omit it to exercise the anonymous path, challenges included.
|
|
13
25
|
*/
|
|
14
|
-
export async function start(
|
|
15
|
-
const config =
|
|
16
|
-
const
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
26
|
+
export async function start(options) {
|
|
27
|
+
const config = sharedDefaults();
|
|
28
|
+
const { app, authInfo } = options;
|
|
29
|
+
const handler = createMcpHandler(() => app.createServerInstance());
|
|
30
|
+
const chat = await Chat.open({
|
|
31
|
+
model: options.model,
|
|
32
|
+
temperature: options.temperature ?? config?.temperature,
|
|
33
|
+
systemPrompt: options.systemPrompt ?? config?.systemPrompt,
|
|
34
|
+
maxSteps: options.maxSteps ?? config?.maxSteps,
|
|
35
|
+
}, (url, init) => handler.fetch(new Request(url, init), authInfo === undefined ? undefined : { authInfo }));
|
|
36
|
+
onTestFinished(async () => {
|
|
37
|
+
await chat.close();
|
|
38
|
+
await handler.close();
|
|
21
39
|
});
|
|
22
|
-
onTestFinished(() => chat.close());
|
|
23
40
|
return chat;
|
|
24
41
|
}
|
|
25
42
|
//# sourceMappingURL=session-registry.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAe,MAAM,8BAA8B,CAAC;AAE7E,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAmCjC,SAAS,cAAc;IACrB,IAAI,CAAC;QACH,OAAO,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,OAA6D;IAE7D,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;IAChC,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAClC,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,oBAAoB,EAAE,CAAC,CAAC;IAEnE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAC1B;QACE,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,MAAM,EAAE,WAAW;QACvD,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,MAAM,EAAE,YAAY;QAC1D,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,MAAM,EAAE,QAAQ;KAC/C,EACD,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CACZ,OAAO,CAAC,KAAK,CACX,IAAI,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,EACtB,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAClD,CACJ,CAAC;IAEF,cAAc,CAAC,KAAK,IAAI,EAAE;QACxB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skybridge/test",
|
|
3
|
-
"version": "0.0.0-dev.
|
|
3
|
+
"version": "0.0.0-dev.e3d3217",
|
|
4
4
|
"description": "Eval runner for Skybridge MCP apps: does a prompt reach the right tool with the right arguments",
|
|
5
5
|
"license": "ISC",
|
|
6
6
|
"type": "module",
|
|
@@ -13,36 +13,36 @@
|
|
|
13
13
|
"types": "./dist/index.d.ts",
|
|
14
14
|
"default": "./dist/index.js"
|
|
15
15
|
},
|
|
16
|
-
"./
|
|
17
|
-
"types": "./dist/
|
|
18
|
-
"default": "./dist/
|
|
16
|
+
"./matchers": {
|
|
17
|
+
"types": "./dist/matchers/index.d.ts",
|
|
18
|
+
"default": "./dist/matchers/index.js"
|
|
19
19
|
}
|
|
20
20
|
},
|
|
21
21
|
"files": [
|
|
22
22
|
"dist"
|
|
23
23
|
],
|
|
24
24
|
"dependencies": {
|
|
25
|
-
"@
|
|
26
|
-
"@modelcontextprotocol/
|
|
27
|
-
"ai": "^5.0.0",
|
|
28
|
-
"@skybridge/vite-plugin": "0.0.0-dev.df0cfdd"
|
|
25
|
+
"@modelcontextprotocol/client": "^2.0.0",
|
|
26
|
+
"@modelcontextprotocol/server": "^2.0.0"
|
|
29
27
|
},
|
|
30
28
|
"peerDependencies": {
|
|
29
|
+
"@skybridge/vite-plugin": "^2.0.0",
|
|
30
|
+
"ai": "^7.0.0",
|
|
31
31
|
"skybridge": "^2.0.0",
|
|
32
32
|
"vitest": "^4.1.0"
|
|
33
33
|
},
|
|
34
34
|
"devDependencies": {
|
|
35
35
|
"@types/node": "^24.13.3",
|
|
36
|
+
"ai": "^7.0.0",
|
|
36
37
|
"typescript": "^5.9.3",
|
|
37
38
|
"vitest": "^4.1.10",
|
|
38
|
-
"
|
|
39
|
+
"zod": "^4.4.3",
|
|
40
|
+
"@skybridge/vite-plugin": "0.0.0-dev.e3d3217",
|
|
41
|
+
"skybridge": "0.0.0-dev.e3d3217"
|
|
39
42
|
},
|
|
40
43
|
"engines": {
|
|
41
44
|
"node": ">=22.23.1"
|
|
42
45
|
},
|
|
43
|
-
"publishConfig": {
|
|
44
|
-
"tag": "beta"
|
|
45
|
-
},
|
|
46
46
|
"scripts": {
|
|
47
47
|
"build": "tsc -p tsconfig.json",
|
|
48
48
|
"format": "biome check --write --error-on-warnings",
|
package/dist/config.d.ts
DELETED
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
import type { EvalsOptions, ModelDescriptor } from "@skybridge/vite-plugin";
|
|
2
|
-
export type { EvalsOptions, ModelDescriptor, ModelFile, } from "@skybridge/vite-plugin";
|
|
3
|
-
/** The serializable half, the only part that crosses into the workers. */
|
|
4
|
-
export type ProvidedEvalsOptions = Omit<EvalsOptions, "model"> & {
|
|
5
|
-
model?: ModelDescriptor;
|
|
6
|
-
};
|
|
7
|
-
declare module "vitest" {
|
|
8
|
-
interface ProvidedContext {
|
|
9
|
-
skybridgeEvals: ProvidedEvalsOptions;
|
|
10
|
-
skybridgeEvalsUrl: string;
|
|
11
|
-
}
|
|
12
|
-
}
|
|
13
|
-
//# sourceMappingURL=config.d.ts.map
|
package/dist/config.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAE5E,YAAY,EACV,YAAY,EACZ,eAAe,EACf,SAAS,GACV,MAAM,wBAAwB,CAAC;AAEhC,0EAA0E;AAC1E,MAAM,MAAM,oBAAoB,GAAG,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,GAAG;IAC/D,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB,CAAC;AAEF,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,eAAe;QACvB,cAAc,EAAE,oBAAoB,CAAC;QACrC,iBAAiB,EAAE,MAAM,CAAC;KAC3B;CACF"}
|
package/dist/config.js
DELETED
package/dist/config.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":""}
|
package/dist/global-setup.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"global-setup.d.ts","sourceRoot":"","sources":["../src/global-setup.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAI/C,wBAA8B,KAAK,CAAC,OAAO,EAAE,WAAW,8CA4BvD"}
|
package/dist/global-setup.js
DELETED
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
import { startServer } from "./server-harness.js";
|
|
2
|
-
export default async function setup(project) {
|
|
3
|
-
const options = project.config.provide?.skybridgeEvals;
|
|
4
|
-
if (options === undefined) {
|
|
5
|
-
throw new Error("The evals plugin is not registered. Add `skybridge({ evals: {...} })` to the vitest config.");
|
|
6
|
-
}
|
|
7
|
-
if (options.server !== undefined) {
|
|
8
|
-
project.provide("skybridgeEvalsUrl", options.server);
|
|
9
|
-
return;
|
|
10
|
-
}
|
|
11
|
-
if (options.project === undefined) {
|
|
12
|
-
throw new Error("The evals plugin needs either `server` or `project`");
|
|
13
|
-
}
|
|
14
|
-
const server = await startServer(options.project);
|
|
15
|
-
project.provide("skybridgeEvalsUrl", server.url);
|
|
16
|
-
return async () => {
|
|
17
|
-
const output = server.output().trim();
|
|
18
|
-
await server.stop();
|
|
19
|
-
if (output !== "") {
|
|
20
|
-
console.log(`\n--- server output ---\n${output}\n---`);
|
|
21
|
-
}
|
|
22
|
-
};
|
|
23
|
-
}
|
|
24
|
-
//# sourceMappingURL=global-setup.js.map
|
package/dist/global-setup.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"global-setup.js","sourceRoot":"","sources":["../src/global-setup.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAElD,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,OAAoB;IACtD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,cAE3B,CAAC;IACd,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACjC,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACrD,OAAO;IACT,CAAC;IACD,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAClD,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IAEjD,OAAO,KAAK,IAAI,EAAE;QAChB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;YAClB,OAAO,CAAC,GAAG,CAAC,4BAA4B,MAAM,OAAO,CAAC,CAAC;QACzD,CAAC;IACH,CAAC,CAAC;AACJ,CAAC"}
|
package/dist/matchers.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../src/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D,OAAO,KAAK,EAAE,IAAI,EAAY,MAAM,WAAW,CAAC;AAEhD;;;;GAIG;AACH,MAAM,WAAW,YAAY,CAAC,GAAG;IAC/B,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GACnC,IAAI,CAAC;IACR,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GAClC,IAAI,CAAC;IACR,qBAAqB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;CACtE;AAED,MAAM,WAAW,aAAa,CAAC,GAAG,CAAE,SAAQ,YAAY,CAAC,GAAG,CAAC;IAC3D,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC;CACxB;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,YAAY;QACpB,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;KAChD;CACF"}
|
package/dist/matchers.js
DELETED
|
@@ -1,66 +0,0 @@
|
|
|
1
|
-
import { expect } from "vitest";
|
|
2
|
-
function attemptedCalls(chat, name) {
|
|
3
|
-
return chat.toolCalls.filter((call) => String(call.name) === name);
|
|
4
|
-
}
|
|
5
|
-
function acceptedCalls(chat, name) {
|
|
6
|
-
return attemptedCalls(chat, name).filter((call) => call.failed === undefined);
|
|
7
|
-
}
|
|
8
|
-
function observed(chat) {
|
|
9
|
-
if (chat.toolCalls.length === 0) {
|
|
10
|
-
return " (no tool was called)";
|
|
11
|
-
}
|
|
12
|
-
return chat.toolCalls
|
|
13
|
-
.map((call, index) => ` ${index + 1}. ${call.name} ${JSON.stringify(call.arguments)}${call.failed === undefined ? "" : ` (failed: ${call.failed})`}`)
|
|
14
|
-
.join("\n");
|
|
15
|
-
}
|
|
16
|
-
function definitions(chat) {
|
|
17
|
-
return chat.toolDefinitions
|
|
18
|
-
.map((tool) => ` - ${tool.name}: ${tool.description ?? ""}\n arguments: ${JSON.stringify(tool.inputSchema)}`)
|
|
19
|
-
.join("\n");
|
|
20
|
-
}
|
|
21
|
-
function report(chat, summary) {
|
|
22
|
-
return `${summary}
|
|
23
|
-
|
|
24
|
-
Calls the model actually made:
|
|
25
|
-
${observed(chat)}
|
|
26
|
-
|
|
27
|
-
Tool definitions the model was looking at:
|
|
28
|
-
${definitions(chat)}`;
|
|
29
|
-
}
|
|
30
|
-
expect.extend({
|
|
31
|
-
toHaveCalledToolOnce(received, name, args) {
|
|
32
|
-
const matching = acceptedCalls(received, name);
|
|
33
|
-
const argsMatch = args === undefined ||
|
|
34
|
-
(matching.length === 1 &&
|
|
35
|
-
this.equals(matching[0]?.arguments, expect.objectContaining(args)));
|
|
36
|
-
const pass = matching.length === 1 && argsMatch;
|
|
37
|
-
return {
|
|
38
|
-
pass,
|
|
39
|
-
message: () => report(received, pass
|
|
40
|
-
? `Expected "${name}" not to be called exactly once.`
|
|
41
|
-
: matching.length === 1
|
|
42
|
-
? `Expected "${name}" to be called once with ${JSON.stringify(args)}, but it was called with ${JSON.stringify(matching[0]?.arguments)}.`
|
|
43
|
-
: `Expected "${name}" to be called exactly once by a call that succeeded, but ${matching.length} such calls were made.`),
|
|
44
|
-
};
|
|
45
|
-
},
|
|
46
|
-
toHaveCalledToolWith(received, name, args) {
|
|
47
|
-
const pass = acceptedCalls(received, name).some((call) => this.equals(call.arguments, expect.objectContaining(args)));
|
|
48
|
-
return {
|
|
49
|
-
pass,
|
|
50
|
-
message: () => report(received, pass
|
|
51
|
-
? `Expected no call to "${name}" with ${JSON.stringify(args)}.`
|
|
52
|
-
: `Expected "${name}" to be called with ${JSON.stringify(args)}.`),
|
|
53
|
-
};
|
|
54
|
-
},
|
|
55
|
-
toNeverHaveCalledTool(received, name) {
|
|
56
|
-
const matching = attemptedCalls(received, name);
|
|
57
|
-
return {
|
|
58
|
-
pass: matching.length === 0,
|
|
59
|
-
message: () => report(received, matching.length === 0
|
|
60
|
-
? `Expected "${name}" to be called at least once.`
|
|
61
|
-
: `Expected "${name}" never to be called, but it was called ${matching.length} times.`),
|
|
62
|
-
};
|
|
63
|
-
},
|
|
64
|
-
});
|
|
65
|
-
expect.chat = ((chat) => expect(chat));
|
|
66
|
-
//# sourceMappingURL=matchers.js.map
|
package/dist/matchers.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../src/matchers.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AA8BhC,SAAS,cAAc,CACrB,IAAmB,EACnB,IAAY;IAEZ,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,aAAa,CAAC,IAAmB,EAAE,IAAY;IACtD,OAAO,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,QAAQ,CAAC,IAAmB;IACnC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,wBAAwB,CAAC;IAClC,CAAC;IACD,OAAO,IAAI,CAAC,SAAS;SAClB,GAAG,CACF,CAAC,IAAuB,EAAE,KAAa,EAAE,EAAE,CACzC,KAAK,KAAK,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,MAAM,GAAG,EAAE,CACnI;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAAC,IAAmB;IACtC,OAAO,IAAI,CAAC,eAAe;SACxB,GAAG,CACF,CAAC,IAAI,EAAE,EAAE,CACP,OAAO,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,WAAW,IAAI,EAAE,oBAAoB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CACpG;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,MAAM,CAAC,IAAmB,EAAE,OAAe;IAClD,OAAO,GAAG,OAAO;;;EAGjB,QAAQ,CAAC,IAAI,CAAC;;;EAGd,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACtB,CAAC;AAED,MAAM,CAAC,MAAM,CAAC;IACZ,oBAAoB,CAClB,QAAuB,EACvB,IAAY,EACZ,IAA8B;QAE9B,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,SAAS,GACb,IAAI,KAAK,SAAS;YAClB,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC;QAEhD,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,aAAa,IAAI,kCAAkC;gBACrD,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;oBACrB,CAAC,CAAC,aAAa,IAAI,4BAA4B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG;oBACxI,CAAC,CAAC,aAAa,IAAI,6DAA6D,QAAQ,CAAC,MAAM,wBAAwB,CAC5H;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAClB,QAAuB,EACvB,IAAY,EACZ,IAA6B;QAE7B,MAAM,IAAI,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACvD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAC3D,CAAC;QAEF,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,wBAAwB,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG;gBAC/D,CAAC,CAAC,aAAa,IAAI,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CACpE;SACJ,CAAC;IACJ,CAAC;IAED,qBAAqB,CAAC,QAAuB,EAAE,IAAY;QACzD,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAEhD,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC3B,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACnB,CAAC,CAAC,aAAa,IAAI,+BAA+B;gBAClD,CAAC,CAAC,aAAa,IAAI,2CAA2C,QAAQ,CAAC,MAAM,SAAS,CACzF;SACJ,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,IAAI,GAAG,CAAC,CAAM,IAAe,EAAE,EAAE,CACtC,MAAM,CAAC,IAAI,CAAC,CAAkC,CAAC"}
|
package/dist/model.d.ts
DELETED
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import type { LanguageModel } from "ai";
|
|
2
|
-
import type { ModelDescriptor } from "./config.js";
|
|
3
|
-
/**
|
|
4
|
-
* Registers a model instance from a setup file. The escape hatch for providers
|
|
5
|
-
* the descriptor cannot construct, such as a company LLM gateway.
|
|
6
|
-
*/
|
|
7
|
-
export declare function defineEvalModel(model: LanguageModel): void;
|
|
8
|
-
export declare function resolveModel(descriptor: ModelDescriptor | undefined): LanguageModel;
|
|
9
|
-
//# sourceMappingURL=model.d.ts.map
|
package/dist/model.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAInD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAE1D;AAED,wBAAgB,YAAY,CAC1B,UAAU,EAAE,eAAe,GAAG,SAAS,GACtC,aAAa,CAef"}
|
package/dist/model.js
DELETED
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
import { createAnthropic } from "@ai-sdk/anthropic";
|
|
2
|
-
let registered;
|
|
3
|
-
/**
|
|
4
|
-
* Registers a model instance from a setup file. The escape hatch for providers
|
|
5
|
-
* the descriptor cannot construct, such as a company LLM gateway.
|
|
6
|
-
*/
|
|
7
|
-
export function defineEvalModel(model) {
|
|
8
|
-
registered = model;
|
|
9
|
-
}
|
|
10
|
-
export function resolveModel(descriptor) {
|
|
11
|
-
if (registered !== undefined) {
|
|
12
|
-
return registered;
|
|
13
|
-
}
|
|
14
|
-
if (descriptor === undefined) {
|
|
15
|
-
throw new Error("No model configured. Pass `model` to the evals plugin, or call defineEvalModel() from a setup file.");
|
|
16
|
-
}
|
|
17
|
-
const name = descriptor.apiKeyEnv ?? "ANTHROPIC_API_KEY";
|
|
18
|
-
const apiKey = process.env[name];
|
|
19
|
-
if (apiKey === undefined || apiKey === "") {
|
|
20
|
-
throw new Error(`${name} is not set`);
|
|
21
|
-
}
|
|
22
|
-
return createAnthropic({ apiKey })(descriptor.name);
|
|
23
|
-
}
|
|
24
|
-
//# sourceMappingURL=model.js.map
|
package/dist/model.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"model.js","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAIpD,IAAI,UAAqC,CAAC;AAE1C;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,KAAoB;IAClD,UAAU,GAAG,KAAK,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,UAAuC;IAEvC,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CACb,qGAAqG,CACtG,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GAAG,UAAU,CAAC,SAAS,IAAI,mBAAmB,CAAC;IACzD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjC,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;QAC1C,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,aAAa,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;AACtD,CAAC"}
|
package/dist/plugin.d.ts
DELETED
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
import type { Plugin } from "vitest/config";
|
|
2
|
-
import type { EvalsOptions } from "./config.js";
|
|
3
|
-
/**
|
|
4
|
-
* Wires the eval runner into vitest. Usually reached through the `evals`
|
|
5
|
-
* option of the `skybridge()` Vite plugin rather than registered directly.
|
|
6
|
-
* Everything the runner needs is configured here, so a scenario file imports
|
|
7
|
-
* nothing but `start` and its own `AppType`.
|
|
8
|
-
*/
|
|
9
|
-
export declare function evals(options: EvalsOptions): Plugin;
|
|
10
|
-
//# sourceMappingURL=plugin.d.ts.map
|
package/dist/plugin.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,KAAK,EAAE,YAAY,EAAwB,MAAM,aAAa,CAAC;AAMtE;;;;;GAKG;AACH,wBAAgB,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM,CAuBnD"}
|
package/dist/plugin.js
DELETED
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
import { fileURLToPath } from "node:url";
|
|
2
|
-
function here(file) {
|
|
3
|
-
return fileURLToPath(new URL(file, import.meta.url));
|
|
4
|
-
}
|
|
5
|
-
/**
|
|
6
|
-
* Wires the eval runner into vitest. Usually reached through the `evals`
|
|
7
|
-
* option of the `skybridge()` Vite plugin rather than registered directly.
|
|
8
|
-
* Everything the runner needs is configured here, so a scenario file imports
|
|
9
|
-
* nothing but `start` and its own `AppType`.
|
|
10
|
-
*/
|
|
11
|
-
export function evals(options) {
|
|
12
|
-
const { model, ...rest } = options;
|
|
13
|
-
const provided = "file" in model ? rest : { ...rest, model };
|
|
14
|
-
const setupFiles = [here("./setup.js")];
|
|
15
|
-
if ("file" in model) {
|
|
16
|
-
setupFiles.push(model.file);
|
|
17
|
-
}
|
|
18
|
-
return {
|
|
19
|
-
name: "skybridge-evals",
|
|
20
|
-
config() {
|
|
21
|
-
return {
|
|
22
|
-
test: {
|
|
23
|
-
testTimeout: 120_000,
|
|
24
|
-
setupFiles,
|
|
25
|
-
globalSetup: [here("./global-setup.js")],
|
|
26
|
-
provide: { skybridgeEvals: provided },
|
|
27
|
-
},
|
|
28
|
-
};
|
|
29
|
-
},
|
|
30
|
-
};
|
|
31
|
-
}
|
|
32
|
-
//# sourceMappingURL=plugin.js.map
|
package/dist/plugin.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"plugin.js","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAIzC,SAAS,IAAI,CAAC,IAAY;IACxB,OAAO,aAAa,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AACvD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,KAAK,CAAC,OAAqB;IACzC,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;IACnC,MAAM,QAAQ,GACZ,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC;IAE9C,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;IACxC,IAAI,MAAM,IAAI,KAAK,EAAE,CAAC;QACpB,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,OAAO;QACL,IAAI,EAAE,iBAAiB;QACvB,MAAM;YACJ,OAAO;gBACL,IAAI,EAAE;oBACJ,WAAW,EAAE,OAAO;oBACpB,UAAU;oBACV,WAAW,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;oBACxC,OAAO,EAAE,EAAE,cAAc,EAAE,QAAQ,EAAE;iBACtC;aACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/dist/server-harness.d.ts
DELETED
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
export interface RunningServer {
|
|
2
|
-
url: string;
|
|
3
|
-
/** Everything the server wrote to stdout/stderr, for failure reports. */
|
|
4
|
-
output: () => string;
|
|
5
|
-
stop: () => Promise<void>;
|
|
6
|
-
}
|
|
7
|
-
/**
|
|
8
|
-
* Spawns the project's server with `__PORT=0` and an IPC channel, and waits
|
|
9
|
-
* for the `skybridge:listening` message carrying the port the OS picked.
|
|
10
|
-
* Binding and reporting both happen in the child, so there is no window in
|
|
11
|
-
* which another process can steal the port.
|
|
12
|
-
*/
|
|
13
|
-
export declare function startServer({ cwd, command, env, }: {
|
|
14
|
-
cwd: string;
|
|
15
|
-
command: string[];
|
|
16
|
-
env?: Record<string, string>;
|
|
17
|
-
}): Promise<RunningServer>;
|
|
18
|
-
//# sourceMappingURL=server-harness.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"server-harness.d.ts","sourceRoot":"","sources":["../src/server-harness.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,yEAAyE;IACzE,MAAM,EAAE,MAAM,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAiBD;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,EAChC,GAAG,EACH,OAAO,EACP,GAAQ,GACT,EAAE;IACD,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B,GAAG,OAAO,CAAC,aAAa,CAAC,CA4EzB"}
|
package/dist/server-harness.js
DELETED
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
import { spawn } from "node:child_process";
|
|
2
|
-
import { resolve } from "node:path";
|
|
3
|
-
function hasExited(child) {
|
|
4
|
-
return child.exitCode !== null || child.signalCode !== null;
|
|
5
|
-
}
|
|
6
|
-
function isListeningMessage(message) {
|
|
7
|
-
return (typeof message === "object" &&
|
|
8
|
-
message !== null &&
|
|
9
|
-
message.type === "skybridge:listening" &&
|
|
10
|
-
typeof message.port === "number");
|
|
11
|
-
}
|
|
12
|
-
/**
|
|
13
|
-
* Spawns the project's server with `__PORT=0` and an IPC channel, and waits
|
|
14
|
-
* for the `skybridge:listening` message carrying the port the OS picked.
|
|
15
|
-
* Binding and reporting both happen in the child, so there is no window in
|
|
16
|
-
* which another process can steal the port.
|
|
17
|
-
*/
|
|
18
|
-
export async function startServer({ cwd, command, env = {}, }) {
|
|
19
|
-
const timeoutMs = 60_000;
|
|
20
|
-
const [bin, ...args] = command;
|
|
21
|
-
if (bin === undefined) {
|
|
22
|
-
throw new Error("Empty server command");
|
|
23
|
-
}
|
|
24
|
-
const executable = bin.startsWith(".") ? resolve(cwd, bin) : bin;
|
|
25
|
-
const child = spawn(executable, args, {
|
|
26
|
-
cwd,
|
|
27
|
-
env: {
|
|
28
|
-
...process.env,
|
|
29
|
-
NODE_ENV: "production",
|
|
30
|
-
...env,
|
|
31
|
-
__PORT: "0",
|
|
32
|
-
},
|
|
33
|
-
stdio: ["ignore", "pipe", "pipe", "ipc"],
|
|
34
|
-
});
|
|
35
|
-
let captured = "";
|
|
36
|
-
const output = () => captured;
|
|
37
|
-
child.stdout?.on("data", (chunk) => {
|
|
38
|
-
captured += chunk;
|
|
39
|
-
});
|
|
40
|
-
child.stderr?.on("data", (chunk) => {
|
|
41
|
-
captured += chunk;
|
|
42
|
-
});
|
|
43
|
-
const stop = async () => {
|
|
44
|
-
if (hasExited(child)) {
|
|
45
|
-
return;
|
|
46
|
-
}
|
|
47
|
-
const exited = new Promise((resolveExit) => {
|
|
48
|
-
child.once("exit", () => resolveExit());
|
|
49
|
-
});
|
|
50
|
-
child.kill("SIGTERM");
|
|
51
|
-
const forced = setTimeout(() => child.kill("SIGKILL"), 3_000);
|
|
52
|
-
await exited;
|
|
53
|
-
clearTimeout(forced);
|
|
54
|
-
};
|
|
55
|
-
const port = await new Promise((resolvePort, reject) => {
|
|
56
|
-
const timer = setTimeout(() => {
|
|
57
|
-
reject(new Error(`Server did not report a port in ${timeoutMs}ms. \`command\` must start the server process directly (node, tsx): wrappers such as \`pnpm run\` do not forward the IPC channel the readiness signal uses. It must also run a skybridge version that sends skybridge:listening.\n${output()}`));
|
|
58
|
-
}, timeoutMs);
|
|
59
|
-
child.on("message", (message) => {
|
|
60
|
-
if (isListeningMessage(message)) {
|
|
61
|
-
clearTimeout(timer);
|
|
62
|
-
resolvePort(message.port);
|
|
63
|
-
}
|
|
64
|
-
});
|
|
65
|
-
child.on("error", (error) => {
|
|
66
|
-
clearTimeout(timer);
|
|
67
|
-
reject(new Error(`Failed to spawn ${command.join(" ")}: ${error.message}`));
|
|
68
|
-
});
|
|
69
|
-
child.on("exit", () => {
|
|
70
|
-
clearTimeout(timer);
|
|
71
|
-
reject(new Error(`Server exited with code ${child.exitCode} before reporting its port:\n${output()}`));
|
|
72
|
-
});
|
|
73
|
-
}).catch(async (error) => {
|
|
74
|
-
await stop();
|
|
75
|
-
throw error;
|
|
76
|
-
});
|
|
77
|
-
return { url: `http://127.0.0.1:${port}/mcp`, output, stop };
|
|
78
|
-
}
|
|
79
|
-
//# sourceMappingURL=server-harness.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"server-harness.js","sourceRoot":"","sources":["../src/server-harness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AASpC,SAAS,SAAS,CAAC,KAAmB;IACpC,OAAO,KAAK,CAAC,QAAQ,KAAK,IAAI,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI,CAAC;AAC9D,CAAC;AAED,SAAS,kBAAkB,CACzB,OAAgB;IAEhB,OAAO,CACL,OAAO,OAAO,KAAK,QAAQ;QAC3B,OAAO,KAAK,IAAI;QACf,OAA8B,CAAC,IAAI,KAAK,qBAAqB;QAC9D,OAAQ,OAA8B,CAAC,IAAI,KAAK,QAAQ,CACzD,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAChC,GAAG,EACH,OAAO,EACP,GAAG,GAAG,EAAE,GAKT;IACC,MAAM,SAAS,GAAG,MAAM,CAAC;IACzB,MAAM,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC;IAC/B,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAEjE,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,EAAE,IAAI,EAAE;QACpC,GAAG;QACH,GAAG,EAAE;YACH,GAAG,OAAO,CAAC,GAAG;YACd,QAAQ,EAAE,YAAY;YACtB,GAAG,GAAG;YACN,MAAM,EAAE,GAAG;SACZ;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC;KACzC,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,QAAQ,CAAC;IAC9B,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;QACjC,QAAQ,IAAI,KAAK,CAAC;IACpB,CAAC,CAAC,CAAC;IACH,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;QACjC,QAAQ,IAAI,KAAK,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;QACtB,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YACrB,OAAO;QACT,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,OAAO,CAAO,CAAC,WAAW,EAAE,EAAE;YAC/C,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,CAAC;QAC9D,MAAM,MAAM,CAAC;QACb,YAAY,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE;QAC7D,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,CACJ,IAAI,KAAK,CACP,mCAAmC,SAAS,qOAAqO,MAAM,EAAE,EAAE,CAC5R,CACF,CAAC;QACJ,CAAC,EAAE,SAAS,CAAC,CAAC;QACd,KAAK,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE;YAC9B,IAAI,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC;gBAChC,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC1B,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CACJ,IAAI,KAAK,CAAC,mBAAmB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CACpE,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YACpB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CACJ,IAAI,KAAK,CACP,2BAA2B,KAAK,CAAC,QAAQ,gCAAgC,MAAM,EAAE,EAAE,CACpF,CACF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAc,EAAE,EAAE;QAChC,MAAM,IAAI,EAAE,CAAC;QACb,MAAM,KAAK,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,GAAG,EAAE,oBAAoB,IAAI,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;AAC/D,CAAC"}
|
package/dist/setup.d.ts
DELETED
package/dist/setup.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"setup.d.ts","sourceRoot":"","sources":["../src/setup.ts"],"names":[],"mappings":"AAAA,OAAO,eAAe,CAAC"}
|
package/dist/setup.js
DELETED
package/dist/setup.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"setup.js","sourceRoot":"","sources":["../src/setup.ts"],"names":[],"mappings":"AAAA,OAAO,eAAe,CAAC"}
|