@skybridge/test 0.0.0-dev.7aada9d → 0.0.0-dev.7cd4271
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +126 -0
- package/dist/chat.d.ts +28 -30
- package/dist/chat.d.ts.map +1 -1
- package/dist/chat.js +150 -30
- package/dist/chat.js.map +1 -1
- package/dist/eval-middlewares.d.ts +13 -0
- package/dist/eval-middlewares.d.ts.map +1 -0
- package/dist/eval-middlewares.js +18 -0
- package/dist/eval-middlewares.js.map +1 -0
- package/dist/index.d.ts +3 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +0 -1
- package/dist/index.js.map +1 -1
- package/dist/input-files.d.ts +14 -0
- package/dist/input-files.d.ts.map +1 -0
- package/dist/input-files.js +126 -0
- package/dist/input-files.js.map +1 -0
- package/dist/matchers/index.d.ts +4 -0
- package/dist/matchers/index.d.ts.map +1 -0
- package/dist/matchers/index.js +2 -0
- package/dist/matchers/index.js.map +1 -0
- package/dist/matchers/matchers.d.ts +54 -0
- package/dist/matchers/matchers.d.ts.map +1 -0
- package/dist/matchers/matchers.js +258 -0
- package/dist/matchers/matchers.js.map +1 -0
- package/dist/matchers/types.d.ts +44 -0
- package/dist/matchers/types.d.ts.map +1 -0
- package/dist/matchers/types.js +2 -0
- package/dist/matchers/types.js.map +1 -0
- package/dist/middleware/list-tools.d.ts +4 -0
- package/dist/middleware/list-tools.d.ts.map +1 -0
- package/dist/middleware/list-tools.js +67 -0
- package/dist/middleware/list-tools.js.map +1 -0
- package/dist/middleware/resolve-files.d.ts +4 -0
- package/dist/middleware/resolve-files.d.ts.map +1 -0
- package/dist/middleware/resolve-files.js +13 -0
- package/dist/middleware/resolve-files.js.map +1 -0
- package/dist/middleware/stub-tool.d.ts +4 -0
- package/dist/middleware/stub-tool.d.ts.map +1 -0
- package/dist/middleware/stub-tool.js +16 -0
- package/dist/middleware/stub-tool.js.map +1 -0
- package/dist/session-registry.d.ts +53 -9
- package/dist/session-registry.d.ts.map +1 -1
- package/dist/session-registry.js +60 -20
- package/dist/session-registry.js.map +1 -1
- package/package.json +15 -13
- package/dist/config.d.ts +0 -40
- package/dist/config.d.ts.map +0 -1
- package/dist/config.js +0 -2
- package/dist/config.js.map +0 -1
- package/dist/global-setup.d.ts +0 -3
- package/dist/global-setup.d.ts.map +0 -1
- package/dist/global-setup.js +0 -39
- package/dist/global-setup.js.map +0 -1
- package/dist/matchers.d.ts +0 -21
- package/dist/matchers.d.ts.map +0 -1
- package/dist/matchers.js +0 -76
- package/dist/matchers.js.map +0 -1
- package/dist/model.d.ts +0 -9
- package/dist/model.d.ts.map +0 -1
- package/dist/model.js +0 -28
- package/dist/model.js.map +0 -1
- package/dist/plugin.d.ts +0 -10
- package/dist/plugin.d.ts.map +0 -1
- package/dist/plugin.js +0 -32
- package/dist/plugin.js.map +0 -1
- package/dist/server-harness.d.ts +0 -19
- package/dist/server-harness.d.ts.map +0 -1
- package/dist/server-harness.js +0 -78
- package/dist/server-harness.js.map +0 -1
- package/dist/setup.d.ts +0 -2
- package/dist/setup.d.ts.map +0 -1
- package/dist/setup.js +0 -7
- package/dist/setup.js.map +0 -1
- package/dist/telemetry.d.ts +0 -10
- package/dist/telemetry.d.ts.map +0 -1
- package/dist/telemetry.js +0 -77
- package/dist/telemetry.js.map +0 -1
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export function createResolveFilesMiddleware(files, fileParams, inputs) {
|
|
2
|
+
return async (request, _extra, next) => {
|
|
3
|
+
const params = request.params;
|
|
4
|
+
const args = await files.rewrite((params.arguments ?? {}), fileParams.get(params.name) ?? []);
|
|
5
|
+
params.arguments = args;
|
|
6
|
+
const callId = params._meta?.["skybridge/evalToolCallId"];
|
|
7
|
+
if (typeof callId === "string") {
|
|
8
|
+
inputs.set(callId, args);
|
|
9
|
+
}
|
|
10
|
+
return next();
|
|
11
|
+
};
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=resolve-files.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"resolve-files.js","sourceRoot":"","sources":["../../src/middleware/resolve-files.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,4BAA4B,CAC1C,KAAiB,EACjB,UAAiC,EACjC,MAA4C;IAE5C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE;QACrC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC9B,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,CAC9B,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAA4B,EACnD,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAc,CAAC,IAAI,EAAE,CAC5C,CAAC;QACF,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,MAAM,MAAM,GAAI,MAAM,CAAC,KAA6C,EAAE,CACpE,0BAA0B,CAC3B,CAAC;QACF,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/B,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC3B,CAAC;QACD,OAAO,IAAI,EAAE,CAAC;IAChB,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stub-tool.d.ts","sourceRoot":"","sources":["../../src/middleware/stub-tool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAExD,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,SAAS,GAAG,eAAe,CAiB1E"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export function createStubToolMiddleware(stubs) {
|
|
2
|
+
return async (request, _extra, next) => {
|
|
3
|
+
const result = await stubs[request.params.name]?.((request.params.arguments ?? {}));
|
|
4
|
+
return result === undefined
|
|
5
|
+
? next()
|
|
6
|
+
: {
|
|
7
|
+
content: [
|
|
8
|
+
{
|
|
9
|
+
type: "text",
|
|
10
|
+
text: typeof result === "string" ? result : JSON.stringify(result),
|
|
11
|
+
},
|
|
12
|
+
],
|
|
13
|
+
};
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
//# sourceMappingURL=stub-tool.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stub-tool.js","sourceRoot":"","sources":["../../src/middleware/stub-tool.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,wBAAwB,CAAC,KAAgB;IACvD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE;QACrC,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAc,CAAC,EAAE,CACzD,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAA4B,CAC5D,CAAC;QACF,OAAO,MAAM,KAAK,SAAS;YACzB,CAAC,CAAC,IAAI,EAAE;YACR,CAAC,CAAC;gBACE,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EACF,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;qBAC/D;iBACF;aACF,CAAC;IACR,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -1,16 +1,60 @@
|
|
|
1
|
+
import type { LanguageModel } from "ai";
|
|
2
|
+
import type { Skybridge, ToolDef, ToolInput, ToolNames } from "skybridge/server";
|
|
1
3
|
import { Chat } from "./chat.js";
|
|
4
|
+
/** The Skybridge surface used by an in-process eval session. */
|
|
5
|
+
export type EvalApp = Pick<Skybridge, "$types" | "createServerInstance" | "withMcpMiddleware">;
|
|
2
6
|
/**
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
* The identity an in-process session claims, handed to the app as the
|
|
8
|
+
* request's `authInfo`. Mirrors the SDK's `AuthInfo` structurally; `extra`
|
|
9
|
+
* carries whatever claims the app's verifier would have produced.
|
|
10
|
+
*/
|
|
11
|
+
export interface EvalIdentity {
|
|
12
|
+
token: string;
|
|
13
|
+
clientId: string;
|
|
14
|
+
scopes: string[];
|
|
15
|
+
expiresAt?: number;
|
|
16
|
+
resource?: URL;
|
|
17
|
+
extra?: Record<string, unknown>;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Results the runner answers with instead of calling the app, keyed by tool
|
|
21
|
+
* name and typed against the project's own registry. A stub may be async, and
|
|
22
|
+
* one that returns `undefined` falls through to the real server, so a scenario
|
|
23
|
+
* can pin one set of arguments and leave the rest live.
|
|
10
24
|
*/
|
|
11
|
-
export
|
|
25
|
+
export type ToolStubs<App = Skybridge<Record<string, ToolDef<Record<string, unknown>>>>> = {
|
|
26
|
+
[Name in ToolNames<App>]?: (args: ToolInput<App, Name>) => unknown;
|
|
27
|
+
};
|
|
28
|
+
interface StartOptions {
|
|
29
|
+
/** Any AI SDK model instance, built by the project. */
|
|
30
|
+
model: LanguageModel;
|
|
12
31
|
systemPrompt?: string;
|
|
13
32
|
temperature?: number;
|
|
33
|
+
maxSteps?: number;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Opens a fresh MCP session and conversation against the app under test. The
|
|
37
|
+
* app is served in-process by a handler built on `app.createServerInstance`
|
|
38
|
+
* for this conversation alone, and both are closed when the current test
|
|
39
|
+
* finishes, so scenarios never leak sessions and never write teardown, and
|
|
40
|
+
* tests that run concurrently cannot close each other's sessions.
|
|
41
|
+
*
|
|
42
|
+
* The assertions are inferred from the app value itself, so `expect.chat` gets
|
|
43
|
+
* the project's tool names and argument shapes with no type parameter.
|
|
44
|
+
*
|
|
45
|
+
* `stubs` answers a tool from the scenario instead of the app, so a case that
|
|
46
|
+
* depends on today's date or on a live catalogue still reads the same weeks
|
|
47
|
+
* later. A stubbed call bypasses the application pipeline, and shows up in
|
|
48
|
+
* `chat.toolCalls` like any other. See {@link ToolStubs}.
|
|
49
|
+
*
|
|
50
|
+
* `authInfo` claims an identity for the session: the app's per-tool scheme and
|
|
51
|
+
* scope enforcement runs against it for real, only token verification is
|
|
52
|
+
* skipped. Omit it to exercise the anonymous path, challenges included.
|
|
53
|
+
*/
|
|
54
|
+
export declare function start<App extends EvalApp>(options: StartOptions & {
|
|
55
|
+
app: App;
|
|
56
|
+
authInfo?: EvalIdentity;
|
|
57
|
+
stubs?: ToolStubs<App>;
|
|
14
58
|
}): Promise<Chat<App>>;
|
|
15
|
-
export
|
|
59
|
+
export {};
|
|
16
60
|
//# sourceMappingURL=session-registry.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxC,OAAO,KAAK,EACV,SAAS,EACT,OAAO,EACP,SAAS,EACT,SAAS,EACV,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,gEAAgE;AAChE,MAAM,MAAM,OAAO,GAAG,IAAI,CACxB,SAAS,EACT,QAAQ,GAAG,sBAAsB,GAAG,mBAAmB,CACxD,CAAC;AAEF;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,GAAG,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED;;;;;GAKG;AACH,MAAM,MAAM,SAAS,CACnB,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAC/D;KACD,IAAI,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,OAAO;CACnE,CAAC;AAEF,UAAU,YAAY;IACpB,uDAAuD;IACvD,KAAK,EAAE,aAAa,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAUD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,KAAK,CAAC,GAAG,SAAS,OAAO,EAC7C,OAAO,EAAE,YAAY,GAAG;IACtB,GAAG,EAAE,GAAG,CAAC;IACT,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,KAAK,CAAC,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC;CACxB,GACA,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CA2CpB"}
|
package/dist/session-registry.js
CHANGED
|
@@ -1,28 +1,68 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createMcpHandler } from "@modelcontextprotocol/server";
|
|
2
|
+
import { inject, onTestFinished } from "vitest";
|
|
2
3
|
import { Chat } from "./chat.js";
|
|
3
|
-
import {
|
|
4
|
-
|
|
4
|
+
import { EvalMiddlewares } from "./eval-middlewares.js";
|
|
5
|
+
function sharedDefaults() {
|
|
6
|
+
try {
|
|
7
|
+
return inject("skybridgeEvals");
|
|
8
|
+
}
|
|
9
|
+
catch {
|
|
10
|
+
return undefined;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
5
13
|
/**
|
|
6
|
-
* Opens a fresh MCP session and conversation against the
|
|
7
|
-
*
|
|
8
|
-
*
|
|
14
|
+
* Opens a fresh MCP session and conversation against the app under test. The
|
|
15
|
+
* app is served in-process by a handler built on `app.createServerInstance`
|
|
16
|
+
* for this conversation alone, and both are closed when the current test
|
|
17
|
+
* finishes, so scenarios never leak sessions and never write teardown, and
|
|
18
|
+
* tests that run concurrently cannot close each other's sessions.
|
|
19
|
+
*
|
|
20
|
+
* The assertions are inferred from the app value itself, so `expect.chat` gets
|
|
21
|
+
* the project's tool names and argument shapes with no type parameter.
|
|
22
|
+
*
|
|
23
|
+
* `stubs` answers a tool from the scenario instead of the app, so a case that
|
|
24
|
+
* depends on today's date or on a live catalogue still reads the same weeks
|
|
25
|
+
* later. A stubbed call bypasses the application pipeline, and shows up in
|
|
26
|
+
* `chat.toolCalls` like any other. See {@link ToolStubs}.
|
|
9
27
|
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
28
|
+
* `authInfo` claims an identity for the session: the app's per-tool scheme and
|
|
29
|
+
* scope enforcement runs against it for real, only token verification is
|
|
30
|
+
* skipped. Omit it to exercise the anonymous path, challenges included.
|
|
13
31
|
*/
|
|
14
|
-
export async function start(
|
|
15
|
-
const config =
|
|
16
|
-
const
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
32
|
+
export async function start(options) {
|
|
33
|
+
const config = sharedDefaults();
|
|
34
|
+
const { app, authInfo } = options;
|
|
35
|
+
const middleware = new EvalMiddlewares((options.stubs ?? {}));
|
|
36
|
+
app
|
|
37
|
+
.withMcpMiddleware({
|
|
38
|
+
filter: "tools/list",
|
|
39
|
+
handler: middleware.listTools,
|
|
40
|
+
})
|
|
41
|
+
.withMcpMiddleware({
|
|
42
|
+
filter: "tools/call",
|
|
43
|
+
handler: middleware.resolveFiles,
|
|
44
|
+
})
|
|
45
|
+
.withMcpMiddleware({
|
|
46
|
+
filter: "tools/call",
|
|
47
|
+
handler: middleware.stubTool,
|
|
48
|
+
});
|
|
49
|
+
const handler = createMcpHandler(() => app.createServerInstance());
|
|
50
|
+
let chat;
|
|
51
|
+
try {
|
|
52
|
+
chat = await Chat.open({
|
|
53
|
+
model: options.model,
|
|
54
|
+
temperature: options.temperature ?? config?.temperature,
|
|
55
|
+
systemPrompt: options.systemPrompt ?? config?.systemPrompt,
|
|
56
|
+
maxSteps: options.maxSteps ?? config?.maxSteps,
|
|
57
|
+
}, (url, init) => handler.fetch(new Request(url, init), authInfo === undefined ? undefined : { authInfo }), middleware, () => handler.close());
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
await Promise.allSettled([middleware.files.close(), handler.close()]);
|
|
61
|
+
throw error;
|
|
62
|
+
}
|
|
63
|
+
onTestFinished(async () => {
|
|
64
|
+
await chat.close();
|
|
21
65
|
});
|
|
22
|
-
opened.push(chat);
|
|
23
66
|
return chat;
|
|
24
67
|
}
|
|
25
|
-
export async function closeOpenedSessions() {
|
|
26
|
-
await Promise.all(opened.splice(0).map((chat) => chat.close()));
|
|
27
|
-
}
|
|
28
68
|
//# sourceMappingURL=session-registry.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAQhE,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AA0CxD,SAAS,cAAc;IACrB,IAAI,CAAC;QACH,OAAO,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,OAIC;IAED,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;IAChC,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAClC,MAAM,UAAU,GAAG,IAAI,eAAe,CAAC,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,CAAc,CAAC,CAAC;IAC3E,GAAG;SACA,iBAAiB,CAAC;QACjB,MAAM,EAAE,YAAY;QACpB,OAAO,EAAE,UAAU,CAAC,SAAS;KAC9B,CAAC;SACD,iBAAiB,CAAC;QACjB,MAAM,EAAE,YAAY;QACpB,OAAO,EAAE,UAAU,CAAC,YAAY;KACjC,CAAC;SACD,iBAAiB,CAAC;QACjB,MAAM,EAAE,YAAY;QACpB,OAAO,EAAE,UAAU,CAAC,QAAQ;KAC7B,CAAC,CAAC;IACL,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,oBAAoB,EAAE,CAAC,CAAC;IACnE,IAAI,IAAe,CAAC;IACpB,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CACpB;YACE,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,MAAM,EAAE,WAAW;YACvD,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,MAAM,EAAE,YAAY;YAC1D,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,MAAM,EAAE,QAAQ;SAC/C,EACD,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CACZ,OAAO,CAAC,KAAK,CACX,IAAI,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,EACtB,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAClD,EACH,UAAU,EACV,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,CACtB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACtE,MAAM,KAAK,CAAC;IACd,CAAC;IACD,cAAc,CAAC,KAAK,IAAI,EAAE;QACxB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skybridge/test",
|
|
3
|
-
"version": "0.0.0-dev.
|
|
4
|
-
"description": "Eval runner for Skybridge MCP
|
|
3
|
+
"version": "0.0.0-dev.7cd4271",
|
|
4
|
+
"description": "Eval runner for Skybridge MCP Apps: does a prompt reach the right tool with the right arguments",
|
|
5
5
|
"license": "ISC",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"repository": {
|
|
@@ -13,30 +13,32 @@
|
|
|
13
13
|
"types": "./dist/index.d.ts",
|
|
14
14
|
"default": "./dist/index.js"
|
|
15
15
|
},
|
|
16
|
-
"./
|
|
17
|
-
"types": "./dist/
|
|
18
|
-
"default": "./dist/
|
|
16
|
+
"./matchers": {
|
|
17
|
+
"types": "./dist/matchers/index.d.ts",
|
|
18
|
+
"default": "./dist/matchers/index.js"
|
|
19
19
|
}
|
|
20
20
|
},
|
|
21
21
|
"files": [
|
|
22
22
|
"dist"
|
|
23
23
|
],
|
|
24
24
|
"dependencies": {
|
|
25
|
-
"@
|
|
26
|
-
"@modelcontextprotocol/
|
|
27
|
-
"ai": "^5.0.0",
|
|
28
|
-
"ci-info": "^4.4.0",
|
|
29
|
-
"posthog-node": "^5.45.2"
|
|
25
|
+
"@modelcontextprotocol/client": "^2.0.0",
|
|
26
|
+
"@modelcontextprotocol/server": "^2.0.0"
|
|
30
27
|
},
|
|
31
28
|
"peerDependencies": {
|
|
32
|
-
"
|
|
33
|
-
"
|
|
29
|
+
"@skybridge/vite-plugin": "^2.0.0",
|
|
30
|
+
"ai": "^7.0.0",
|
|
31
|
+
"skybridge": "^2.0.0",
|
|
32
|
+
"vitest": "^4.1.0"
|
|
34
33
|
},
|
|
35
34
|
"devDependencies": {
|
|
36
35
|
"@types/node": "^24.13.3",
|
|
36
|
+
"ai": "^7.0.0",
|
|
37
37
|
"typescript": "^5.9.3",
|
|
38
38
|
"vitest": "^4.1.10",
|
|
39
|
-
"
|
|
39
|
+
"zod": "^4.4.3",
|
|
40
|
+
"@skybridge/vite-plugin": "0.0.0-dev.7cd4271",
|
|
41
|
+
"skybridge": "0.0.0-dev.7cd4271"
|
|
40
42
|
},
|
|
41
43
|
"engines": {
|
|
42
44
|
"node": ">=22.23.1"
|
package/dist/config.d.ts
DELETED
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
/** A provider we can construct ourselves, so no extra file is needed. */
|
|
2
|
-
export interface ModelDescriptor {
|
|
3
|
-
provider: "anthropic";
|
|
4
|
-
name: string;
|
|
5
|
-
/** Env var holding the key. Never the key itself. */
|
|
6
|
-
apiKeyEnv?: string;
|
|
7
|
-
}
|
|
8
|
-
/** Escape hatch: a module that calls `defineEvalModel` with its own instance. */
|
|
9
|
-
export interface ModelFile {
|
|
10
|
-
file: string;
|
|
11
|
-
}
|
|
12
|
-
export interface EvalsOptions {
|
|
13
|
-
model: ModelDescriptor | ModelFile;
|
|
14
|
-
temperature?: number;
|
|
15
|
-
systemPrompt?: string;
|
|
16
|
-
maxSteps?: number;
|
|
17
|
-
server?: string;
|
|
18
|
-
/**
|
|
19
|
-
* How the runner starts the project's server. `command` must start the
|
|
20
|
-
* server process directly (`node`, `tsx`), not through a wrapper such as
|
|
21
|
-
* `pnpm run`: readiness and the bound port arrive over the spawned
|
|
22
|
-
* process's IPC channel, which wrappers do not forward.
|
|
23
|
-
*/
|
|
24
|
-
project?: {
|
|
25
|
-
cwd: string;
|
|
26
|
-
command: string[];
|
|
27
|
-
env?: Record<string, string>;
|
|
28
|
-
};
|
|
29
|
-
}
|
|
30
|
-
/** The serializable half, the only part that crosses into the workers. */
|
|
31
|
-
export type ProvidedEvalsOptions = Omit<EvalsOptions, "model"> & {
|
|
32
|
-
model?: ModelDescriptor;
|
|
33
|
-
};
|
|
34
|
-
declare module "vitest" {
|
|
35
|
-
interface ProvidedContext {
|
|
36
|
-
skybridgeEvals: ProvidedEvalsOptions;
|
|
37
|
-
skybridgeEvalsUrl: string;
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
//# sourceMappingURL=config.d.ts.map
|
package/dist/config.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,WAAW,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,qDAAqD;IACrD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,iFAAiF;AACjF,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,eAAe,GAAG,SAAS,CAAC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;OAKG;IACH,OAAO,CAAC,EAAE;QACR,GAAG,EAAE,MAAM,CAAC;QACZ,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC9B,CAAC;CACH;AAED,0EAA0E;AAC1E,MAAM,MAAM,oBAAoB,GAAG,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,GAAG;IAC/D,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB,CAAC;AAEF,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,eAAe;QACvB,cAAc,EAAE,oBAAoB,CAAC;QACrC,iBAAiB,EAAE,MAAM,CAAC;KAC3B;CACF"}
|
package/dist/config.js
DELETED
package/dist/config.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":""}
|
package/dist/global-setup.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"global-setup.d.ts","sourceRoot":"","sources":["../src/global-setup.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAK/C,wBAA8B,KAAK,CAAC,OAAO,EAAE,WAAW,gCAwCvD"}
|
package/dist/global-setup.js
DELETED
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
import { startServer } from "./server-harness.js";
|
|
2
|
-
import { captureEvalRun } from "./telemetry.js";
|
|
3
|
-
export default async function setup(project) {
|
|
4
|
-
const options = project.config.provide
|
|
5
|
-
?.skybridgeEvals;
|
|
6
|
-
const startedAt = Date.now();
|
|
7
|
-
if (options.server !== undefined) {
|
|
8
|
-
project.provide("skybridgeEvalsUrl", options.server);
|
|
9
|
-
return async () => {
|
|
10
|
-
await captureEvalRun({
|
|
11
|
-
durationMs: Date.now() - startedAt,
|
|
12
|
-
serverStarted: true,
|
|
13
|
-
});
|
|
14
|
-
};
|
|
15
|
-
}
|
|
16
|
-
if (options.project === undefined) {
|
|
17
|
-
throw new Error("The evals plugin needs either `server` or `project`");
|
|
18
|
-
}
|
|
19
|
-
const server = await startServer(options.project).catch(async (error) => {
|
|
20
|
-
await captureEvalRun({
|
|
21
|
-
durationMs: Date.now() - startedAt,
|
|
22
|
-
serverStarted: false,
|
|
23
|
-
});
|
|
24
|
-
throw error;
|
|
25
|
-
});
|
|
26
|
-
project.provide("skybridgeEvalsUrl", server.url);
|
|
27
|
-
return async () => {
|
|
28
|
-
const output = server.output().trim();
|
|
29
|
-
await server.stop();
|
|
30
|
-
if (output !== "") {
|
|
31
|
-
console.log(`\n--- server output ---\n${output}\n---`);
|
|
32
|
-
}
|
|
33
|
-
await captureEvalRun({
|
|
34
|
-
durationMs: Date.now() - startedAt,
|
|
35
|
-
serverStarted: true,
|
|
36
|
-
});
|
|
37
|
-
};
|
|
38
|
-
}
|
|
39
|
-
//# sourceMappingURL=global-setup.js.map
|
package/dist/global-setup.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"global-setup.js","sourceRoot":"","sources":["../src/global-setup.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,OAAoB;IACtD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO;QACpC,EAAE,cAAsC,CAAC;IAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACjC,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACrD,OAAO,KAAK,IAAI,EAAE;YAChB,MAAM,cAAc,CAAC;gBACnB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,aAAa,EAAE,IAAI;aACpB,CAAC,CAAC;QACL,CAAC,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,CACrD,KAAK,EAAE,KAAc,EAAE,EAAE;QACvB,MAAM,cAAc,CAAC;YACnB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,aAAa,EAAE,KAAK;SACrB,CAAC,CAAC;QACH,MAAM,KAAK,CAAC;IACd,CAAC,CACF,CAAC;IACF,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IAEjD,OAAO,KAAK,IAAI,EAAE;QAChB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;YAClB,OAAO,CAAC,GAAG,CAAC,4BAA4B,MAAM,OAAO,CAAC,CAAC;QACzD,CAAC;QACD,MAAM,cAAc,CAAC;YACnB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,aAAa,EAAE,IAAI;SACpB,CAAC,CAAC;IACL,CAAC,CAAC;AACJ,CAAC"}
|
package/dist/matchers.d.ts
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import type { ToolInput, ToolNames } from "skybridge/server";
|
|
2
|
-
import type { Chat } from "./chat.js";
|
|
3
|
-
/**
|
|
4
|
-
* Assertions about the tool calls a conversation produced. Kept off vitest's
|
|
5
|
-
* global `Matchers` so they never appear on unrelated `expect()` calls, and so
|
|
6
|
-
* this stays the one place later tool-call helpers are added.
|
|
7
|
-
*/
|
|
8
|
-
export interface ChatMatchers<App> {
|
|
9
|
-
toHaveCalledToolOnce<Name extends ToolNames<App>>(name: Name, args?: Partial<ToolInput<App, Name>>): void;
|
|
10
|
-
toHaveCalledToolWith<Name extends ToolNames<App>>(name: Name, args: Partial<ToolInput<App, Name>>): void;
|
|
11
|
-
toNeverHaveCalledTool<Name extends ToolNames<App>>(name: Name): void;
|
|
12
|
-
}
|
|
13
|
-
export interface ChatAssertion<App> extends ChatMatchers<App> {
|
|
14
|
-
not: ChatMatchers<App>;
|
|
15
|
-
}
|
|
16
|
-
declare module "vitest" {
|
|
17
|
-
interface ExpectStatic {
|
|
18
|
-
chat<App>(chat: Chat<App>): ChatAssertion<App>;
|
|
19
|
-
}
|
|
20
|
-
}
|
|
21
|
-
//# sourceMappingURL=matchers.d.ts.map
|
package/dist/matchers.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../src/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D,OAAO,KAAK,EAAE,IAAI,EAAY,MAAM,WAAW,CAAC;AAEhD;;;;GAIG;AACH,MAAM,WAAW,YAAY,CAAC,GAAG;IAC/B,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GACnC,IAAI,CAAC;IACR,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GAClC,IAAI,CAAC;IACR,qBAAqB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;CACtE;AAED,MAAM,WAAW,aAAa,CAAC,GAAG,CAAE,SAAQ,YAAY,CAAC,GAAG,CAAC;IAC3D,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC;CACxB;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,YAAY;QACpB,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;KAChD;CACF"}
|
package/dist/matchers.js
DELETED
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
import { expect } from "vitest";
|
|
2
|
-
/**
|
|
3
|
-
* Every attempt at a tool, including calls the tool's schema rejected. Used by
|
|
4
|
-
* the negative matcher: an attempt the model should not have made is a failure
|
|
5
|
-
* whether or not its arguments parsed.
|
|
6
|
-
*/
|
|
7
|
-
function attemptsAt(chat, name) {
|
|
8
|
-
return chat.toolCalls.filter((call) => String(call.name) === name);
|
|
9
|
-
}
|
|
10
|
-
/**
|
|
11
|
-
* Only the calls that actually reached the server. Positive matchers use this,
|
|
12
|
-
* so a rejected call never satisfies an assertion that the tool ran; the
|
|
13
|
-
* rejected attempt still shows up in the failure report.
|
|
14
|
-
*/
|
|
15
|
-
function callsTo(chat, name) {
|
|
16
|
-
return attemptsAt(chat, name).filter((call) => call.invalid !== true);
|
|
17
|
-
}
|
|
18
|
-
function observed(chat) {
|
|
19
|
-
if (chat.toolCalls.length === 0) {
|
|
20
|
-
return " (no tool was called)";
|
|
21
|
-
}
|
|
22
|
-
return chat.toolCalls
|
|
23
|
-
.map((call, index) => ` ${index + 1}. ${String(call.name)} ${JSON.stringify(call.arguments)}${call.invalid === true ? " (arguments rejected by the tool schema)" : ""}`)
|
|
24
|
-
.join("\n");
|
|
25
|
-
}
|
|
26
|
-
function definitions(chat) {
|
|
27
|
-
return chat.toolDefinitions
|
|
28
|
-
.map((tool) => ` - ${tool.name}: ${tool.description ?? ""}\n arguments: ${JSON.stringify(tool.inputSchema)}`)
|
|
29
|
-
.join("\n");
|
|
30
|
-
}
|
|
31
|
-
function report(chat, summary) {
|
|
32
|
-
return `${summary}
|
|
33
|
-
|
|
34
|
-
Calls the model actually made:
|
|
35
|
-
${observed(chat)}
|
|
36
|
-
|
|
37
|
-
Tool definitions the model was looking at:
|
|
38
|
-
${definitions(chat)}`;
|
|
39
|
-
}
|
|
40
|
-
expect.extend({
|
|
41
|
-
toHaveCalledToolOnce(received, name, args) {
|
|
42
|
-
const matching = callsTo(received, name);
|
|
43
|
-
const argsMatch = args === undefined ||
|
|
44
|
-
(matching.length === 1 &&
|
|
45
|
-
this.equals(matching[0]?.arguments, expect.objectContaining(args)));
|
|
46
|
-
const pass = matching.length === 1 && argsMatch;
|
|
47
|
-
return {
|
|
48
|
-
pass,
|
|
49
|
-
message: () => report(received, pass
|
|
50
|
-
? `Expected "${name}" not to be called exactly once.`
|
|
51
|
-
: matching.length === 1
|
|
52
|
-
? `Expected "${name}" to be called once with ${JSON.stringify(args)}, but it was called with ${JSON.stringify(matching[0]?.arguments)}.`
|
|
53
|
-
: `Expected "${name}" to be called exactly once by a call the tool schema accepted, but ${matching.length} such calls were made.`),
|
|
54
|
-
};
|
|
55
|
-
},
|
|
56
|
-
toHaveCalledToolWith(received, name, args) {
|
|
57
|
-
const pass = callsTo(received, name).some((call) => this.equals(call.arguments, expect.objectContaining(args)));
|
|
58
|
-
return {
|
|
59
|
-
pass,
|
|
60
|
-
message: () => report(received, pass
|
|
61
|
-
? `Expected no call to "${name}" with ${JSON.stringify(args)}.`
|
|
62
|
-
: `Expected "${name}" to be called with ${JSON.stringify(args)}.`),
|
|
63
|
-
};
|
|
64
|
-
},
|
|
65
|
-
toNeverHaveCalledTool(received, name) {
|
|
66
|
-
const matching = attemptsAt(received, name);
|
|
67
|
-
return {
|
|
68
|
-
pass: matching.length === 0,
|
|
69
|
-
message: () => report(received, matching.length === 0
|
|
70
|
-
? `Expected "${name}" to be called at least once.`
|
|
71
|
-
: `Expected "${name}" never to be called, but it was called ${matching.length} times.`),
|
|
72
|
-
};
|
|
73
|
-
},
|
|
74
|
-
});
|
|
75
|
-
expect.chat = ((chat) => expect(chat));
|
|
76
|
-
//# sourceMappingURL=matchers.js.map
|
package/dist/matchers.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../src/matchers.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AA8BhC;;;;GAIG;AACH,SAAS,UAAU,CAAC,IAAmB,EAAE,IAAY;IACnD,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AACrE,CAAC;AAED;;;;GAIG;AACH,SAAS,OAAO,CAAC,IAAmB,EAAE,IAAY;IAChD,OAAO,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,QAAQ,CAAC,IAAmB;IACnC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,wBAAwB,CAAC;IAClC,CAAC;IACD,OAAO,IAAI,CAAC,SAAS;SAClB,GAAG,CACF,CAAC,IAAuB,EAAE,KAAa,EAAE,EAAE,CACzC,KAAK,KAAK,GAAG,CAAC,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,2CAA2C,CAAC,CAAC,CAAC,EAAE,EAAE,CACtJ;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAAC,IAAmB;IACtC,OAAO,IAAI,CAAC,eAAe;SACxB,GAAG,CACF,CAAC,IAAI,EAAE,EAAE,CACP,OAAO,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,WAAW,IAAI,EAAE,oBAAoB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CACpG;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,MAAM,CAAC,IAAmB,EAAE,OAAe;IAClD,OAAO,GAAG,OAAO;;;EAGjB,QAAQ,CAAC,IAAI,CAAC;;;EAGd,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACtB,CAAC;AAED,MAAM,CAAC,MAAM,CAAC;IACZ,oBAAoB,CAClB,QAAuB,EACvB,IAAY,EACZ,IAA8B;QAE9B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACzC,MAAM,SAAS,GACb,IAAI,KAAK,SAAS;YAClB,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC;QAEhD,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,aAAa,IAAI,kCAAkC;gBACrD,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;oBACrB,CAAC,CAAC,aAAa,IAAI,4BAA4B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG;oBACxI,CAAC,CAAC,aAAa,IAAI,uEAAuE,QAAQ,CAAC,MAAM,wBAAwB,CACtI;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAClB,QAAuB,EACvB,IAAY,EACZ,IAA6B;QAE7B,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACjD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAC3D,CAAC;QAEF,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,wBAAwB,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG;gBAC/D,CAAC,CAAC,aAAa,IAAI,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CACpE;SACJ,CAAC;IACJ,CAAC;IAED,qBAAqB,CAAC,QAAuB,EAAE,IAAY;QACzD,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAE5C,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC3B,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACnB,CAAC,CAAC,aAAa,IAAI,+BAA+B;gBAClD,CAAC,CAAC,aAAa,IAAI,2CAA2C,QAAQ,CAAC,MAAM,SAAS,CACzF;SACJ,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,IAAI,GAAG,CAAC,CAAM,IAAe,EAAE,EAAE,CACtC,MAAM,CAAC,IAAI,CAAC,CAAkC,CAAC"}
|
package/dist/model.d.ts
DELETED
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import type { LanguageModel } from "ai";
|
|
2
|
-
import type { ModelDescriptor } from "./config.js";
|
|
3
|
-
/**
|
|
4
|
-
* Registers a model instance from a setup file. The escape hatch for providers
|
|
5
|
-
* the descriptor cannot construct, such as a company LLM gateway.
|
|
6
|
-
*/
|
|
7
|
-
export declare function defineEvalModel(model: LanguageModel): void;
|
|
8
|
-
export declare function resolveModel(descriptor: ModelDescriptor | undefined): Promise<LanguageModel>;
|
|
9
|
-
//# sourceMappingURL=model.d.ts.map
|
package/dist/model.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAInD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAE1D;AAaD,wBAAsB,YAAY,CAChC,UAAU,EAAE,eAAe,GAAG,SAAS,GACtC,OAAO,CAAC,aAAa,CAAC,CAWxB"}
|
package/dist/model.js
DELETED
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
let registered;
|
|
2
|
-
/**
|
|
3
|
-
* Registers a model instance from a setup file. The escape hatch for providers
|
|
4
|
-
* the descriptor cannot construct, such as a company LLM gateway.
|
|
5
|
-
*/
|
|
6
|
-
export function defineEvalModel(model) {
|
|
7
|
-
registered = model;
|
|
8
|
-
}
|
|
9
|
-
const DEFAULT_KEY_ENV = { anthropic: "ANTHROPIC_API_KEY" };
|
|
10
|
-
function requireKey(descriptor) {
|
|
11
|
-
const name = descriptor.apiKeyEnv ?? DEFAULT_KEY_ENV[descriptor.provider];
|
|
12
|
-
const key = process.env[name];
|
|
13
|
-
if (key === undefined || key === "") {
|
|
14
|
-
throw new Error(`${name} is not set`);
|
|
15
|
-
}
|
|
16
|
-
return key;
|
|
17
|
-
}
|
|
18
|
-
export async function resolveModel(descriptor) {
|
|
19
|
-
if (registered !== undefined) {
|
|
20
|
-
return registered;
|
|
21
|
-
}
|
|
22
|
-
if (descriptor === undefined) {
|
|
23
|
-
throw new Error("No model configured. Pass `model` to the evals plugin, or call defineEvalModel() from a setup file.");
|
|
24
|
-
}
|
|
25
|
-
const { createAnthropic } = await import("@ai-sdk/anthropic");
|
|
26
|
-
return createAnthropic({ apiKey: requireKey(descriptor) })(descriptor.name);
|
|
27
|
-
}
|
|
28
|
-
//# sourceMappingURL=model.js.map
|
package/dist/model.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"model.js","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAGA,IAAI,UAAqC,CAAC;AAE1C;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,KAAoB;IAClD,UAAU,GAAG,KAAK,CAAC;AACrB,CAAC;AAED,MAAM,eAAe,GAAG,EAAE,SAAS,EAAE,mBAAmB,EAAW,CAAC;AAEpE,SAAS,UAAU,CAAC,UAA2B;IAC7C,MAAM,IAAI,GAAG,UAAU,CAAC,SAAS,IAAI,eAAe,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC1E,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,aAAa,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,UAAuC;IAEvC,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CACb,qGAAqG,CACtG,CAAC;IACJ,CAAC;IACD,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;IAC9D,OAAO,eAAe,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;AAC9E,CAAC"}
|
package/dist/plugin.d.ts
DELETED
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
import type { Plugin } from "vitest/config";
|
|
2
|
-
import type { EvalsOptions } from "./config.js";
|
|
3
|
-
/**
|
|
4
|
-
* Wires the eval runner into vitest. Usually reached through the `evals`
|
|
5
|
-
* option of the `skybridge()` Vite plugin rather than registered directly.
|
|
6
|
-
* Everything the runner needs is configured here, so a scenario file imports
|
|
7
|
-
* nothing but `start` and its own `AppType`.
|
|
8
|
-
*/
|
|
9
|
-
export declare function evals(options: EvalsOptions): Plugin;
|
|
10
|
-
//# sourceMappingURL=plugin.d.ts.map
|
package/dist/plugin.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,KAAK,EAAE,YAAY,EAAwB,MAAM,aAAa,CAAC;AAMtE;;;;;GAKG;AACH,wBAAgB,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM,CAuBnD"}
|
package/dist/plugin.js
DELETED
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
import { fileURLToPath } from "node:url";
|
|
2
|
-
function here(file) {
|
|
3
|
-
return fileURLToPath(new URL(file, import.meta.url));
|
|
4
|
-
}
|
|
5
|
-
/**
|
|
6
|
-
* Wires the eval runner into vitest. Usually reached through the `evals`
|
|
7
|
-
* option of the `skybridge()` Vite plugin rather than registered directly.
|
|
8
|
-
* Everything the runner needs is configured here, so a scenario file imports
|
|
9
|
-
* nothing but `start` and its own `AppType`.
|
|
10
|
-
*/
|
|
11
|
-
export function evals(options) {
|
|
12
|
-
const { model, ...rest } = options;
|
|
13
|
-
const provided = "file" in model ? rest : { ...rest, model };
|
|
14
|
-
const setupFiles = [here("./setup.js")];
|
|
15
|
-
if ("file" in model) {
|
|
16
|
-
setupFiles.push(model.file);
|
|
17
|
-
}
|
|
18
|
-
return {
|
|
19
|
-
name: "skybridge-evals",
|
|
20
|
-
config() {
|
|
21
|
-
return {
|
|
22
|
-
test: {
|
|
23
|
-
testTimeout: 120_000,
|
|
24
|
-
setupFiles,
|
|
25
|
-
globalSetup: [here("./global-setup.js")],
|
|
26
|
-
provide: { skybridgeEvals: provided },
|
|
27
|
-
},
|
|
28
|
-
};
|
|
29
|
-
},
|
|
30
|
-
};
|
|
31
|
-
}
|
|
32
|
-
//# sourceMappingURL=plugin.js.map
|
package/dist/plugin.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"plugin.js","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAIzC,SAAS,IAAI,CAAC,IAAY;IACxB,OAAO,aAAa,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AACvD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,KAAK,CAAC,OAAqB;IACzC,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;IACnC,MAAM,QAAQ,GACZ,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC;IAE9C,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;IACxC,IAAI,MAAM,IAAI,KAAK,EAAE,CAAC;QACpB,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,OAAO;QACL,IAAI,EAAE,iBAAiB;QACvB,MAAM;YACJ,OAAO;gBACL,IAAI,EAAE;oBACJ,WAAW,EAAE,OAAO;oBACpB,UAAU;oBACV,WAAW,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;oBACxC,OAAO,EAAE,EAAE,cAAc,EAAE,QAAQ,EAAE;iBACtC;aACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/dist/server-harness.d.ts
DELETED
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
export interface RunningServer {
|
|
2
|
-
url: string;
|
|
3
|
-
/** Everything the server wrote to stdout/stderr, for failure reports. */
|
|
4
|
-
output: () => string;
|
|
5
|
-
stop: () => Promise<void>;
|
|
6
|
-
}
|
|
7
|
-
/**
|
|
8
|
-
* Spawns the project's server with `__PORT=0` and an IPC channel, and waits
|
|
9
|
-
* for the `skybridge:listening` message carrying the port the OS picked.
|
|
10
|
-
* Binding and reporting both happen in the child, so there is no window in
|
|
11
|
-
* which another process can steal the port.
|
|
12
|
-
*/
|
|
13
|
-
export declare function startServer({ cwd, command, env, timeoutMs, }: {
|
|
14
|
-
cwd: string;
|
|
15
|
-
command: string[];
|
|
16
|
-
env?: Record<string, string>;
|
|
17
|
-
timeoutMs?: number;
|
|
18
|
-
}): Promise<RunningServer>;
|
|
19
|
-
//# sourceMappingURL=server-harness.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"server-harness.d.ts","sourceRoot":"","sources":["../src/server-harness.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,yEAAyE;IACzE,MAAM,EAAE,MAAM,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAiBD;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,EAChC,GAAG,EACH,OAAO,EACP,GAAQ,EACR,SAAkB,GACnB,EAAE;IACD,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,aAAa,CAAC,CA2EzB"}
|