@skybridge/test 0.0.0-dev.037008a → 0.0.0-dev.07b1700
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/chat.d.ts +5 -1
- package/dist/chat.d.ts.map +1 -1
- package/dist/chat.js +31 -6
- package/dist/chat.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/session-registry.d.ts +8 -12
- package/dist/session-registry.d.ts.map +1 -1
- package/dist/session-registry.js +10 -6
- package/dist/session-registry.js.map +1 -1
- package/package.json +7 -5
package/README.md
CHANGED
|
@@ -6,7 +6,7 @@ check.
|
|
|
6
6
|
|
|
7
7
|
Add the `evals` option to the Skybridge Vite plugin so `expect.chat` exists,
|
|
8
8
|
then write scenarios with vitest's own `it` and `expect`. Pass the app itself
|
|
9
|
-
to `start`: it is served in-process, dialing `app.
|
|
9
|
+
to `start`: it is served in-process, dialing an in-process handler built on `app.createServerInstance()`, and
|
|
10
10
|
the tool names and argument shapes come from the app value, so no type
|
|
11
11
|
parameter is needed.
|
|
12
12
|
|
package/dist/chat.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ToolCall } from "@skybridge/vite-plugin/evals";
|
|
1
|
+
import type { ToolCall, TranscriptEntry } from "@skybridge/vite-plugin/evals";
|
|
2
2
|
import { type LanguageModel } from "ai";
|
|
3
3
|
interface HostConfig {
|
|
4
4
|
model: LanguageModel;
|
|
@@ -22,12 +22,16 @@ export declare class Chat<App = unknown> {
|
|
|
22
22
|
readonly toolCalls: ToolCall<App>[];
|
|
23
23
|
/** What the assistant said, one entry per turn, intermediate ones included. */
|
|
24
24
|
readonly assistantTurns: string[];
|
|
25
|
+
/** Every turn and tool call in the order they happened, for the judge. */
|
|
26
|
+
readonly transcript: TranscriptEntry[];
|
|
25
27
|
private readonly messages;
|
|
26
28
|
private readonly failures;
|
|
27
29
|
private readonly client;
|
|
28
30
|
private readonly transport;
|
|
29
31
|
private readonly host;
|
|
30
32
|
private tools;
|
|
33
|
+
/** The model the conversation runs on, and the judge's default. */
|
|
34
|
+
get model(): LanguageModel;
|
|
31
35
|
private constructor();
|
|
32
36
|
static open<App>(host: HostConfig, fetchImpl: TransportFetch): Promise<Chat<App>>;
|
|
33
37
|
/** Takes a turn. Assert on the result with `expect.chat`. */
|
package/dist/chat.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;
|
|
1
|
+
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC9E,OAAO,EAIL,KAAK,aAAa,EAGnB,MAAM,IAAI,CAAC;AAEZ,UAAU,UAAU;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAsBD;;;GAGG;AACH,MAAM,MAAM,cAAc,GAAG,CAC3B,GAAG,EAAE,MAAM,GAAG,GAAG,EACjB,IAAI,CAAC,EAAE,WAAW,KACf,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEvB;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAC7B,4EAA4E;IAC5E,SAAiB,IAAI,EAAE,GAAG,CAAC;IAE3B,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAM;IAEzC,+EAA+E;IAC/E,QAAQ,CAAC,cAAc,EAAE,MAAM,EAAE,CAAM;IAEvC,0EAA0E;IAC1E,QAAQ,CAAC,UAAU,EAAE,eAAe,EAAE,CAAM;IAE5C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,KAAK,CAAc;IAE3B,mEAAmE;IACnE,IAAI,KAAK,IAAI,aAAa,CAEzB;IAED,OAAO;WAUM,IAAI,CAAC,GAAG,EACnB,IAAI,EAAE,UAAU,EAChB,SAAS,EAAE,cAAc,GACxB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAkBrB,6DAA6D;IACvD,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAyCnC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,OAAO;CAqChB"}
|
package/dist/chat.js
CHANGED
|
@@ -2,6 +2,17 @@ import { Client, StreamableHTTPClientTransport, } from "@modelcontextprotocol/cl
|
|
|
2
2
|
import { dynamicTool, generateText, jsonSchema, stepCountIs, } from "ai";
|
|
3
3
|
const DEFAULT_MAX_STEPS = 8;
|
|
4
4
|
const IN_PROCESS_URL = "http://in-process.skybridge.test/mcp";
|
|
5
|
+
const MAX_RESULT_CHARS = 2000;
|
|
6
|
+
function outputOf(results, toolCallId) {
|
|
7
|
+
const match = results.find((result) => result.toolCallId === toolCallId);
|
|
8
|
+
if (match === undefined) {
|
|
9
|
+
return "(no result)";
|
|
10
|
+
}
|
|
11
|
+
const rendered = JSON.stringify(match.output ?? null);
|
|
12
|
+
return rendered.length > MAX_RESULT_CHARS
|
|
13
|
+
? `${rendered.slice(0, MAX_RESULT_CHARS)}… (truncated)`
|
|
14
|
+
: rendered;
|
|
15
|
+
}
|
|
5
16
|
/**
|
|
6
17
|
* One conversation against one MCP session. A scenario asserts on
|
|
7
18
|
* {@link Chat.toolCalls}, the sequence of calls the model made.
|
|
@@ -11,12 +22,18 @@ export class Chat {
|
|
|
11
22
|
toolCalls = [];
|
|
12
23
|
/** What the assistant said, one entry per turn, intermediate ones included. */
|
|
13
24
|
assistantTurns = [];
|
|
25
|
+
/** Every turn and tool call in the order they happened, for the judge. */
|
|
26
|
+
transcript = [];
|
|
14
27
|
messages = [];
|
|
15
28
|
failures = new Map();
|
|
16
29
|
client;
|
|
17
30
|
transport;
|
|
18
31
|
host;
|
|
19
32
|
tools = [];
|
|
33
|
+
/** The model the conversation runs on, and the judge's default. */
|
|
34
|
+
get model() {
|
|
35
|
+
return this.host.model;
|
|
36
|
+
}
|
|
20
37
|
constructor(client, transport, host) {
|
|
21
38
|
this.client = client;
|
|
22
39
|
this.transport = transport;
|
|
@@ -40,6 +57,7 @@ export class Chat {
|
|
|
40
57
|
/** Takes a turn. Assert on the result with `expect.chat`. */
|
|
41
58
|
async send(prompt) {
|
|
42
59
|
this.messages.push({ role: "user", content: prompt });
|
|
60
|
+
this.transcript.push({ role: "user", text: prompt });
|
|
43
61
|
const result = await generateText({
|
|
44
62
|
model: this.host.model,
|
|
45
63
|
temperature: this.host.temperature ?? 0,
|
|
@@ -51,17 +69,24 @@ export class Chat {
|
|
|
51
69
|
stopWhen: stepCountIs(this.host.maxSteps ?? DEFAULT_MAX_STEPS),
|
|
52
70
|
});
|
|
53
71
|
for (const step of result.steps) {
|
|
72
|
+
if (step.text !== "") {
|
|
73
|
+
this.assistantTurns.push(step.text);
|
|
74
|
+
this.transcript.push({ role: "assistant", text: step.text });
|
|
75
|
+
}
|
|
54
76
|
for (const call of step.toolCalls) {
|
|
77
|
+
const failure = this.failureFor(call);
|
|
55
78
|
this.toolCalls.push({
|
|
56
79
|
name: call.toolName,
|
|
57
80
|
arguments: (call.input ?? {}),
|
|
58
|
-
...
|
|
81
|
+
...failure,
|
|
82
|
+
});
|
|
83
|
+
const outcome = failure.failed === undefined
|
|
84
|
+
? ` -> ${outputOf(step.toolResults, call.toolCallId)}`
|
|
85
|
+
: ` (failed: ${failure.failed})`;
|
|
86
|
+
this.transcript.push({
|
|
87
|
+
role: "tool",
|
|
88
|
+
text: `${call.toolName} ${JSON.stringify(call.input ?? {})}${outcome}`,
|
|
59
89
|
});
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
for (const step of result.steps) {
|
|
63
|
-
if (step.text !== "") {
|
|
64
|
-
this.assistantTurns.push(step.text);
|
|
65
90
|
}
|
|
66
91
|
}
|
|
67
92
|
this.messages.push(...result.response.messages);
|
package/dist/chat.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,6BAA6B,GAE9B,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,UAAU,EAGV,WAAW,GACZ,MAAM,IAAI,CAAC;AASZ,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,MAAM,cAAc,GAAG,sCAAsC,CAAC;
|
|
1
|
+
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,6BAA6B,GAE9B,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,UAAU,EAGV,WAAW,GACZ,MAAM,IAAI,CAAC;AASZ,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,MAAM,cAAc,GAAG,sCAAsC,CAAC;AAE9D,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,SAAS,QAAQ,CACf,OAAmD,EACnD,UAAkB;IAElB,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,KAAK,UAAU,CAAC,CAAC;IACzE,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,aAAa,CAAC;IACvB,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC;IACtD,OAAO,QAAQ,CAAC,MAAM,GAAG,gBAAgB;QACvC,CAAC,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,eAAe;QACvD,CAAC,CAAC,QAAQ,CAAC;AACf,CAAC;AAWD;;;GAGG;AACH,MAAM,OAAO,IAAI;IAIf,mEAAmE;IAC1D,SAAS,GAAoB,EAAE,CAAC;IAEzC,+EAA+E;IACtE,cAAc,GAAa,EAAE,CAAC;IAEvC,0EAA0E;IACjE,UAAU,GAAsB,EAAE,CAAC;IAE3B,QAAQ,GAAmB,EAAE,CAAC;IAC9B,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,MAAM,CAAS;IACf,SAAS,CAAgC;IACzC,IAAI,CAAa;IAC1B,KAAK,GAAW,EAAE,CAAC;IAE3B,mEAAmE;IACnE,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;IACzB,CAAC;IAED,YACE,MAAc,EACd,SAAwC,EACxC,IAAgB;QAEhB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACf,IAAgB,EAChB,SAAyB;QAEzB,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,6BAA6B,CACjD,IAAI,GAAG,CAAC,cAAc,CAAC,EACvB,EAAE,KAAK,EAAE,SAAS,EAAE,CACrB,CAAC;QACF,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACrB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QAErD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC;YACvC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;YACrB,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;SAC/D,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,IAAI,KAAK,EAAE,EAAE,CAAC;gBACrB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAC/D,CAAC;YACD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;gBACtC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAClB,IAAI,EAAE,IAAI,CAAC,QAAQ;oBACnB,SAAS,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA4B;oBACxD,GAAG,OAAO;iBACM,CAAC,CAAC;gBACpB,MAAM,OAAO,GACX,OAAO,CAAC,MAAM,KAAK,SAAS;oBAC1B,CAAC,CAAC,OAAO,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE;oBACtD,CAAC,CAAC,aAAa,OAAO,CAAC,MAAM,GAAG,CAAC;gBACrC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;oBACnB,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,GAAG,OAAO,EAAE;iBACvE,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAEO,UAAU,CAAC,IAA+C;QAGhE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,EAAE,MAAM,EAAE,2CAA2C,EAAE,CAAC;QACjE,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC5C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC;YACpE,OAAO;gBACL,UAAU,CAAC,IAAI;gBACf,WAAW,CAAC;oBACV,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,EAAE;oBACzC,WAAW,EAAE,UAAU,CACrB,UAA8C,CAC/C;oBACD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;wBACvC,IAAI,MAA+C,CAAC;wBACpD,IAAI,CAAC;4BACH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gCAClC,IAAI,EAAE,UAAU,CAAC,IAAI;gCACrB,SAAS,EAAE,CAAC,KAAK,IAAI,EAAE,CAA4B;6BACpD,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,sCAAsC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;4BACF,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;4BAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,+BAA+B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAChE,CAAC;wBACJ,CAAC;wBACD,OAAO,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC;oBAClC,CAAC;iBACF,CAAC;aACM,CAAC;QACb,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type { ChatAssertion, ChatMatchers, } from "@skybridge/vite-plugin/evals";
|
|
1
|
+
export type { ChatAssertion, ChatMatchers, JudgmentOptions, } from "@skybridge/vite-plugin/evals";
|
|
2
2
|
export type { Chat } from "./chat.js";
|
|
3
3
|
export type { EvalApp, EvalIdentity } from "./session-registry.js";
|
|
4
4
|
export { start } from "./session-registry.js";
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,aAAa,EACb,YAAY,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,aAAa,EACb,YAAY,EACZ,eAAe,GAChB,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,YAAY,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
|
|
@@ -1,20 +1,17 @@
|
|
|
1
|
+
import { type Server } from "@modelcontextprotocol/server";
|
|
1
2
|
import type { LanguageModel } from "ai";
|
|
2
3
|
import { Chat } from "./chat.js";
|
|
3
4
|
/**
|
|
4
5
|
* The minimum a Skybridge app has to expose to be served in-process: the
|
|
5
|
-
* `$types` marker the assertions infer tool names from, and the
|
|
6
|
-
* the session is
|
|
7
|
-
* no runtime dependency on `skybridge`.
|
|
6
|
+
* `$types` marker the assertions infer tool names from, and the per-request
|
|
7
|
+
* server builder the session's handler is built on. Structural on purpose, so
|
|
8
|
+
* this package needs no runtime dependency on `skybridge`.
|
|
8
9
|
*/
|
|
9
10
|
export interface EvalApp {
|
|
10
11
|
readonly $types: {
|
|
11
12
|
readonly tools: object;
|
|
12
13
|
};
|
|
13
|
-
|
|
14
|
-
fetch: (request: Request, options?: {
|
|
15
|
-
authInfo?: EvalIdentity;
|
|
16
|
-
}) => Promise<Response>;
|
|
17
|
-
};
|
|
14
|
+
createServerInstance(): Promise<Server>;
|
|
18
15
|
}
|
|
19
16
|
/**
|
|
20
17
|
* The identity an in-process session claims, handed to the app as the
|
|
@@ -38,14 +35,13 @@ interface StartOptions {
|
|
|
38
35
|
}
|
|
39
36
|
/**
|
|
40
37
|
* Opens a fresh MCP session and conversation against the app under test. The
|
|
41
|
-
* app is served in-process
|
|
38
|
+
* app is served in-process by a handler built on `app.createServerInstance`
|
|
39
|
+
* for this conversation alone, and both are closed when the current test
|
|
42
40
|
* finishes, so scenarios never leak sessions and never write teardown, and
|
|
43
41
|
* tests that run concurrently cannot close each other's sessions.
|
|
44
42
|
*
|
|
45
43
|
* The assertions are inferred from the app value itself, so `expect.chat` gets
|
|
46
|
-
* the project's tool names and argument shapes with no type parameter.
|
|
47
|
-
* session talks straight to `app.fetchHandler`, which never closes: the
|
|
48
|
-
* handler is memoized on the app and shared by every test.
|
|
44
|
+
* the project's tool names and argument shapes with no type parameter.
|
|
49
45
|
*
|
|
50
46
|
* `authInfo` claims an identity for the session: the app's per-tool scheme and
|
|
51
47
|
* scope enforcement runs against it for real, only token verification is
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAExC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC;;;;;GAKG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5C,
|
|
1
|
+
{"version":3,"file":"session-registry.d.ts","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAoB,KAAK,MAAM,EAAE,MAAM,8BAA8B,CAAC;AAC7E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAExC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC;;;;;GAKG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5C,oBAAoB,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;CACzC;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,GAAG,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED,UAAU,YAAY;IACpB,uDAAuD;IACvD,KAAK,EAAE,aAAa,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAUD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,KAAK,CAAC,GAAG,SAAS,OAAO,EAC7C,OAAO,EAAE,YAAY,GAAG;IAAE,GAAG,EAAE,GAAG,CAAC;IAAC,QAAQ,CAAC,EAAE,YAAY,CAAA;CAAE,GAC5D,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAwBpB"}
|
package/dist/session-registry.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createMcpHandler } from "@modelcontextprotocol/server";
|
|
1
2
|
import { inject, onTestFinished } from "vitest";
|
|
2
3
|
import { Chat } from "./chat.js";
|
|
3
4
|
function sharedDefaults() {
|
|
@@ -10,14 +11,13 @@ function sharedDefaults() {
|
|
|
10
11
|
}
|
|
11
12
|
/**
|
|
12
13
|
* Opens a fresh MCP session and conversation against the app under test. The
|
|
13
|
-
* app is served in-process
|
|
14
|
+
* app is served in-process by a handler built on `app.createServerInstance`
|
|
15
|
+
* for this conversation alone, and both are closed when the current test
|
|
14
16
|
* finishes, so scenarios never leak sessions and never write teardown, and
|
|
15
17
|
* tests that run concurrently cannot close each other's sessions.
|
|
16
18
|
*
|
|
17
19
|
* The assertions are inferred from the app value itself, so `expect.chat` gets
|
|
18
|
-
* the project's tool names and argument shapes with no type parameter.
|
|
19
|
-
* session talks straight to `app.fetchHandler`, which never closes: the
|
|
20
|
-
* handler is memoized on the app and shared by every test.
|
|
20
|
+
* the project's tool names and argument shapes with no type parameter.
|
|
21
21
|
*
|
|
22
22
|
* `authInfo` claims an identity for the session: the app's per-tool scheme and
|
|
23
23
|
* scope enforcement runs against it for real, only token verification is
|
|
@@ -26,13 +26,17 @@ function sharedDefaults() {
|
|
|
26
26
|
export async function start(options) {
|
|
27
27
|
const config = sharedDefaults();
|
|
28
28
|
const { app, authInfo } = options;
|
|
29
|
+
const handler = createMcpHandler(() => app.createServerInstance());
|
|
29
30
|
const chat = await Chat.open({
|
|
30
31
|
model: options.model,
|
|
31
32
|
temperature: options.temperature ?? config?.temperature,
|
|
32
33
|
systemPrompt: options.systemPrompt ?? config?.systemPrompt,
|
|
33
34
|
maxSteps: options.maxSteps ?? config?.maxSteps,
|
|
34
|
-
}, (url, init) =>
|
|
35
|
-
onTestFinished(() =>
|
|
35
|
+
}, (url, init) => handler.fetch(new Request(url, init), authInfo === undefined ? undefined : { authInfo }));
|
|
36
|
+
onTestFinished(async () => {
|
|
37
|
+
await chat.close();
|
|
38
|
+
await handler.close();
|
|
39
|
+
});
|
|
36
40
|
return chat;
|
|
37
41
|
}
|
|
38
42
|
//# sourceMappingURL=session-registry.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"session-registry.js","sourceRoot":"","sources":["../src/session-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAe,MAAM,8BAA8B,CAAC;AAE7E,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAmCjC,SAAS,cAAc;IACrB,IAAI,CAAC;QACH,OAAO,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,OAA6D;IAE7D,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;IAChC,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAClC,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,oBAAoB,EAAE,CAAC,CAAC;IAEnE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAC1B;QACE,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,MAAM,EAAE,WAAW;QACvD,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,MAAM,EAAE,YAAY;QAC1D,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,MAAM,EAAE,QAAQ;KAC/C,EACD,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CACZ,OAAO,CAAC,KAAK,CACX,IAAI,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,EACtB,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAClD,CACJ,CAAC;IAEF,cAAc,CAAC,KAAK,IAAI,EAAE;QACxB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skybridge/test",
|
|
3
|
-
"version": "0.0.0-dev.
|
|
3
|
+
"version": "0.0.0-dev.07b1700",
|
|
4
4
|
"description": "Eval runner for Skybridge MCP apps: does a prompt reach the right tool with the right arguments",
|
|
5
5
|
"license": "ISC",
|
|
6
6
|
"type": "module",
|
|
@@ -19,20 +19,22 @@
|
|
|
19
19
|
],
|
|
20
20
|
"dependencies": {
|
|
21
21
|
"@modelcontextprotocol/client": "^2.0.0",
|
|
22
|
-
"@
|
|
22
|
+
"@modelcontextprotocol/server": "^2.0.0"
|
|
23
23
|
},
|
|
24
24
|
"peerDependencies": {
|
|
25
|
-
"
|
|
25
|
+
"@skybridge/vite-plugin": "^2.0.0",
|
|
26
|
+
"ai": "^7.0.0",
|
|
26
27
|
"skybridge": "^2.0.0",
|
|
27
28
|
"vitest": "^4.1.0"
|
|
28
29
|
},
|
|
29
30
|
"devDependencies": {
|
|
30
31
|
"@types/node": "^24.13.3",
|
|
31
|
-
"ai": "^
|
|
32
|
+
"ai": "^7.0.0",
|
|
32
33
|
"typescript": "^5.9.3",
|
|
33
34
|
"vitest": "^4.1.10",
|
|
34
35
|
"zod": "^4.4.3",
|
|
35
|
-
"skybridge": "0.0.0-dev.
|
|
36
|
+
"@skybridge/vite-plugin": "0.0.0-dev.07b1700",
|
|
37
|
+
"skybridge": "0.0.0-dev.07b1700"
|
|
36
38
|
},
|
|
37
39
|
"engines": {
|
|
38
40
|
"node": ">=22.23.1"
|