@skybridge/test 0.0.0-dev.b6f5d60 → 0.0.0-dev.b9466d5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -8
- package/dist/chat.d.ts +6 -4
- package/dist/chat.d.ts.map +1 -1
- package/dist/chat.js +6 -11
- package/dist/chat.js.map +1 -1
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -61,13 +61,5 @@ skybridge({
|
|
|
61
61
|
`command` has to start the server process directly, since wrappers such as
|
|
62
62
|
`pnpm run` do not forward the channel the runner uses to learn the port.
|
|
63
63
|
|
|
64
|
-
Assertions on what the assistant said work the same way, though model prose
|
|
65
|
-
varies between runs, so keep them loose.
|
|
66
|
-
|
|
67
|
-
```ts
|
|
68
|
-
expect.chat(chat).toHaveSaid("in stock");
|
|
69
|
-
expect.chat(chat).not.toHaveSaid("refund");
|
|
70
|
-
```
|
|
71
|
-
|
|
72
64
|
See the [documentation](https://docs.skybridge.tech) for setup and the full
|
|
73
65
|
matcher list.
|
package/dist/chat.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ToolCall } from "@skybridge/vite-plugin/evals";
|
|
1
|
+
import type { ToolCall, ToolDefinition } from "@skybridge/vite-plugin/evals";
|
|
2
2
|
import { type LanguageModel } from "ai";
|
|
3
3
|
interface HostConfig {
|
|
4
4
|
model: LanguageModel;
|
|
@@ -6,11 +6,13 @@ interface HostConfig {
|
|
|
6
6
|
systemPrompt?: string;
|
|
7
7
|
maxSteps?: number;
|
|
8
8
|
}
|
|
9
|
+
/**
|
|
10
|
+
* One conversation against one MCP session. A scenario asserts on
|
|
11
|
+
* {@link Chat.toolCalls}, the sequence of calls the model made.
|
|
12
|
+
*/
|
|
9
13
|
export declare class Chat<App = unknown> {
|
|
10
14
|
/** Every call the model made, in order, as it crossed the wire. */
|
|
11
15
|
readonly toolCalls: ToolCall<App>[];
|
|
12
|
-
/** What the assistant said, one entry per turn, intermediate ones included. */
|
|
13
|
-
readonly assistantTurns: string[];
|
|
14
16
|
private readonly messages;
|
|
15
17
|
private readonly failures;
|
|
16
18
|
private readonly client;
|
|
@@ -20,7 +22,7 @@ export declare class Chat<App = unknown> {
|
|
|
20
22
|
private constructor();
|
|
21
23
|
static open<App>(serverUrl: string, host: HostConfig): Promise<Chat<App>>;
|
|
22
24
|
/** The tool definitions the model was looking at, for failure reports. */
|
|
23
|
-
|
|
25
|
+
get toolDefinitions(): ToolDefinition[];
|
|
24
26
|
/** Takes a turn and returns the model's answer. */
|
|
25
27
|
send(prompt: string): Promise<string>;
|
|
26
28
|
close(): Promise<void>;
|
package/dist/chat.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;
|
|
1
|
+
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC7E,OAAO,EAIL,KAAK,aAAa,EAGnB,MAAM,IAAI,CAAC;AAEZ,UAAU,UAAU;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAID;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAC7B,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAM;IAEzC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,KAAK,CAAwB;IAErC,OAAO;WAUM,IAAI,CAAC,GAAG,EACnB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,UAAU,GACf,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAerB,0EAA0E;IAC1E,IAAI,eAAe,IAAI,cAAc,EAAE,CAEtC;IAED,mDAAmD;IAC7C,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IA4BrC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,OAAO;CAmChB"}
|
package/dist/chat.js
CHANGED
|
@@ -2,11 +2,13 @@ import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
|
2
2
|
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
|
|
3
3
|
import { dynamicTool, generateText, jsonSchema, stepCountIs, } from "ai";
|
|
4
4
|
const DEFAULT_MAX_STEPS = 8;
|
|
5
|
+
/**
|
|
6
|
+
* One conversation against one MCP session. A scenario asserts on
|
|
7
|
+
* {@link Chat.toolCalls}, the sequence of calls the model made.
|
|
8
|
+
*/
|
|
5
9
|
export class Chat {
|
|
6
10
|
/** Every call the model made, in order, as it crossed the wire. */
|
|
7
11
|
toolCalls = [];
|
|
8
|
-
/** What the assistant said, one entry per turn, intermediate ones included. */
|
|
9
|
-
assistantTurns = [];
|
|
10
12
|
messages = [];
|
|
11
13
|
failures = new Map();
|
|
12
14
|
client;
|
|
@@ -34,10 +36,8 @@ export class Chat {
|
|
|
34
36
|
return chat;
|
|
35
37
|
}
|
|
36
38
|
/** The tool definitions the model was looking at, for failure reports. */
|
|
37
|
-
|
|
38
|
-
return this.tools
|
|
39
|
-
.map((tool) => ` - ${tool.name}: ${tool.description ?? ""}\n arguments: ${JSON.stringify(tool.inputSchema)}`)
|
|
40
|
-
.join("\n");
|
|
39
|
+
get toolDefinitions() {
|
|
40
|
+
return this.tools;
|
|
41
41
|
}
|
|
42
42
|
/** Takes a turn and returns the model's answer. */
|
|
43
43
|
async send(prompt) {
|
|
@@ -61,11 +61,6 @@ export class Chat {
|
|
|
61
61
|
});
|
|
62
62
|
}
|
|
63
63
|
}
|
|
64
|
-
for (const step of result.steps) {
|
|
65
|
-
if (step.text !== "") {
|
|
66
|
-
this.assistantTurns.push(step.text);
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
64
|
this.messages.push(...result.response.messages);
|
|
70
65
|
return result.text;
|
|
71
66
|
}
|
package/dist/chat.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AAEnG,OAAO,EACL,WAAW,EACX,YAAY,EACZ,UAAU,EAGV,WAAW,GACZ,MAAM,IAAI,CAAC;AASZ,MAAM,iBAAiB,GAAG,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AAEnG,OAAO,EACL,WAAW,EACX,YAAY,EACZ,UAAU,EAGV,WAAW,GACZ,MAAM,IAAI,CAAC;AASZ,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B;;;GAGG;AACH,MAAM,OAAO,IAAI;IACf,mEAAmE;IAC1D,SAAS,GAAoB,EAAE,CAAC;IAExB,QAAQ,GAAmB,EAAE,CAAC;IAC9B,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,MAAM,CAAS;IACf,SAAS,CAAgC;IACzC,IAAI,CAAa;IAC1B,KAAK,GAAqB,EAAE,CAAC;IAErC,YACE,MAAc,EACd,SAAwC,EACxC,IAAgB;QAEhB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACf,SAAiB,EACjB,IAAgB;QAEhB,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,6BAA6B,CAAC,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;QACxE,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACrB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,0EAA0E;IAC1E,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,mDAAmD;IACnD,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAEtD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC;YACvC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;YACrB,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;SAC/D,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAClB,IAAI,EAAE,IAAI,CAAC,QAAQ;oBACnB,SAAS,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA4B;oBACxD,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;iBACR,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChD,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAEO,UAAU,CAAC,IAA+C;QAGhE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,EAAE,MAAM,EAAE,2CAA2C,EAAE,CAAC;QACjE,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC5C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC;YACpE,OAAO;gBACL,UAAU,CAAC,IAAI;gBACf,WAAW,CAAC;oBACV,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,EAAE;oBACzC,WAAW,EAAE,UAAU,CAAC,UAAU,CAAC;oBACnC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;wBACvC,IAAI,MAA+C,CAAC;wBACpD,IAAI,CAAC;4BACH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gCAClC,IAAI,EAAE,UAAU,CAAC,IAAI;gCACrB,SAAS,EAAE,CAAC,KAAK,IAAI,EAAE,CAA4B;6BACpD,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,sCAAsC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;4BACF,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;4BAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,+BAA+B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAChE,CAAC;wBACJ,CAAC;wBACD,OAAO,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC;oBAClC,CAAC;iBACF,CAAC;aACM,CAAC;QACb,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skybridge/test",
|
|
3
|
-
"version": "0.0.0-dev.
|
|
3
|
+
"version": "0.0.0-dev.b9466d5",
|
|
4
4
|
"description": "Eval runner for Skybridge MCP apps: does a prompt reach the right tool with the right arguments",
|
|
5
5
|
"license": "ISC",
|
|
6
6
|
"type": "module",
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
],
|
|
20
20
|
"dependencies": {
|
|
21
21
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
22
|
-
"@skybridge/vite-plugin": "0.0.0-dev.
|
|
22
|
+
"@skybridge/vite-plugin": "0.0.0-dev.b9466d5"
|
|
23
23
|
},
|
|
24
24
|
"peerDependencies": {
|
|
25
25
|
"ai": "^5.0.0",
|
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
"ai": "^5.0.0",
|
|
32
32
|
"typescript": "^5.9.3",
|
|
33
33
|
"vitest": "^4.1.10",
|
|
34
|
-
"skybridge": "0.0.0-dev.
|
|
34
|
+
"skybridge": "0.0.0-dev.b9466d5"
|
|
35
35
|
},
|
|
36
36
|
"engines": {
|
|
37
37
|
"node": ">=22.23.1"
|