@skybridge/test 0.0.0-dev.fb985cd → 0.0.0-dev.fca7959
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chat.d.ts +5 -1
- package/dist/chat.d.ts.map +1 -1
- package/dist/chat.js +17 -6
- package/dist/chat.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/package.json +3 -3
package/dist/chat.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ToolCall } from "@skybridge/vite-plugin/evals";
|
|
1
|
+
import type { ToolCall, TranscriptEntry } from "@skybridge/vite-plugin/evals";
|
|
2
2
|
import { type LanguageModel } from "ai";
|
|
3
3
|
interface HostConfig {
|
|
4
4
|
model: LanguageModel;
|
|
@@ -22,12 +22,16 @@ export declare class Chat<App = unknown> {
|
|
|
22
22
|
readonly toolCalls: ToolCall<App>[];
|
|
23
23
|
/** What the assistant said, one entry per turn, intermediate ones included. */
|
|
24
24
|
readonly assistantTurns: string[];
|
|
25
|
+
/** Every turn and tool call in the order they happened, for the judge. */
|
|
26
|
+
readonly transcript: TranscriptEntry[];
|
|
25
27
|
private readonly messages;
|
|
26
28
|
private readonly failures;
|
|
27
29
|
private readonly client;
|
|
28
30
|
private readonly transport;
|
|
29
31
|
private readonly host;
|
|
30
32
|
private tools;
|
|
33
|
+
/** The model the conversation runs on, and the judge's default. */
|
|
34
|
+
get model(): LanguageModel;
|
|
31
35
|
private constructor();
|
|
32
36
|
static open<App>(host: HostConfig, fetchImpl: TransportFetch): Promise<Chat<App>>;
|
|
33
37
|
/** Takes a turn. Assert on the result with `expect.chat`. */
|
package/dist/chat.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;
|
|
1
|
+
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC9E,OAAO,EAIL,KAAK,aAAa,EAGnB,MAAM,IAAI,CAAC;AAEZ,UAAU,UAAU;IAClB,KAAK,EAAE,aAAa,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAMD;;;GAGG;AACH,MAAM,MAAM,cAAc,GAAG,CAC3B,GAAG,EAAE,MAAM,GAAG,GAAG,EACjB,IAAI,CAAC,EAAE,WAAW,KACf,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEvB;;;GAGG;AACH,qBAAa,IAAI,CAAC,GAAG,GAAG,OAAO;IAC7B,4EAA4E;IAC5E,SAAiB,IAAI,EAAE,GAAG,CAAC;IAE3B,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAM;IAEzC,+EAA+E;IAC/E,QAAQ,CAAC,cAAc,EAAE,MAAM,EAAE,CAAM;IAEvC,0EAA0E;IAC1E,QAAQ,CAAC,UAAU,EAAE,eAAe,EAAE,CAAM;IAE5C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgC;IAC1D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,KAAK,CAAc;IAE3B,mEAAmE;IACnE,IAAI,KAAK,IAAI,aAAa,CAEzB;IAED,OAAO;WAUM,IAAI,CAAC,GAAG,EACnB,IAAI,EAAE,UAAU,EAChB,SAAS,EAAE,cAAc,GACxB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAkBrB,6DAA6D;IACvD,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAqCnC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,OAAO;CAqChB"}
|
package/dist/chat.js
CHANGED
|
@@ -11,12 +11,18 @@ export class Chat {
|
|
|
11
11
|
toolCalls = [];
|
|
12
12
|
/** What the assistant said, one entry per turn, intermediate ones included. */
|
|
13
13
|
assistantTurns = [];
|
|
14
|
+
/** Every turn and tool call in the order they happened, for the judge. */
|
|
15
|
+
transcript = [];
|
|
14
16
|
messages = [];
|
|
15
17
|
failures = new Map();
|
|
16
18
|
client;
|
|
17
19
|
transport;
|
|
18
20
|
host;
|
|
19
21
|
tools = [];
|
|
22
|
+
/** The model the conversation runs on, and the judge's default. */
|
|
23
|
+
get model() {
|
|
24
|
+
return this.host.model;
|
|
25
|
+
}
|
|
20
26
|
constructor(client, transport, host) {
|
|
21
27
|
this.client = client;
|
|
22
28
|
this.transport = transport;
|
|
@@ -40,6 +46,7 @@ export class Chat {
|
|
|
40
46
|
/** Takes a turn. Assert on the result with `expect.chat`. */
|
|
41
47
|
async send(prompt) {
|
|
42
48
|
this.messages.push({ role: "user", content: prompt });
|
|
49
|
+
this.transcript.push({ role: "user", text: prompt });
|
|
43
50
|
const result = await generateText({
|
|
44
51
|
model: this.host.model,
|
|
45
52
|
temperature: this.host.temperature ?? 0,
|
|
@@ -51,17 +58,21 @@ export class Chat {
|
|
|
51
58
|
stopWhen: stepCountIs(this.host.maxSteps ?? DEFAULT_MAX_STEPS),
|
|
52
59
|
});
|
|
53
60
|
for (const step of result.steps) {
|
|
61
|
+
if (step.text !== "") {
|
|
62
|
+
this.assistantTurns.push(step.text);
|
|
63
|
+
this.transcript.push({ role: "assistant", text: step.text });
|
|
64
|
+
}
|
|
54
65
|
for (const call of step.toolCalls) {
|
|
66
|
+
const failure = this.failureFor(call);
|
|
55
67
|
this.toolCalls.push({
|
|
56
68
|
name: call.toolName,
|
|
57
69
|
arguments: (call.input ?? {}),
|
|
58
|
-
...
|
|
70
|
+
...failure,
|
|
71
|
+
});
|
|
72
|
+
this.transcript.push({
|
|
73
|
+
role: "tool",
|
|
74
|
+
text: `${call.toolName} ${JSON.stringify(call.input ?? {})}${failure.failed === undefined ? "" : ` (failed: ${failure.failed})`}`,
|
|
59
75
|
});
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
for (const step of result.steps) {
|
|
63
|
-
if (step.text !== "") {
|
|
64
|
-
this.assistantTurns.push(step.text);
|
|
65
76
|
}
|
|
66
77
|
}
|
|
67
78
|
this.messages.push(...result.response.messages);
|
package/dist/chat.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,6BAA6B,GAE9B,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,UAAU,EAGV,WAAW,GACZ,MAAM,IAAI,CAAC;AASZ,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,MAAM,cAAc,GAAG,sCAAsC,CAAC;AAW9D;;;GAGG;AACH,MAAM,OAAO,IAAI;IAIf,mEAAmE;IAC1D,SAAS,GAAoB,EAAE,CAAC;IAEzC,+EAA+E;IACtE,cAAc,GAAa,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,6BAA6B,GAE9B,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,UAAU,EAGV,WAAW,GACZ,MAAM,IAAI,CAAC;AASZ,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,MAAM,cAAc,GAAG,sCAAsC,CAAC;AAW9D;;;GAGG;AACH,MAAM,OAAO,IAAI;IAIf,mEAAmE;IAC1D,SAAS,GAAoB,EAAE,CAAC;IAEzC,+EAA+E;IACtE,cAAc,GAAa,EAAE,CAAC;IAEvC,0EAA0E;IACjE,UAAU,GAAsB,EAAE,CAAC;IAE3B,QAAQ,GAAmB,EAAE,CAAC;IAC9B,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,MAAM,CAAS;IACf,SAAS,CAAgC;IACzC,IAAI,CAAa;IAC1B,KAAK,GAAW,EAAE,CAAC;IAE3B,mEAAmE;IACnE,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;IACzB,CAAC;IAED,YACE,MAAc,EACd,SAAwC,EACxC,IAAgB;QAEhB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACf,IAAgB,EAChB,SAAyB;QAEzB,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,IAAI,6BAA6B,CACjD,IAAI,GAAG,CAAC,cAAc,CAAC,EACvB,EAAE,KAAK,EAAE,SAAS,EAAE,CACrB,CAAC;QACF,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACrB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QAErD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC;YACvC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE;YACrB,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;SAC/D,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,IAAI,KAAK,EAAE,EAAE,CAAC;gBACrB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAC/D,CAAC;YACD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;gBACtC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAClB,IAAI,EAAE,IAAI,CAAC,QAAQ;oBACnB,SAAS,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAA4B;oBACxD,GAAG,OAAO;iBACM,CAAC,CAAC;gBACpB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;oBACnB,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,OAAO,CAAC,MAAM,GAAG,EAAE;iBAClI,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAEO,UAAU,CAAC,IAA+C;QAGhE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,EAAE,MAAM,EAAE,2CAA2C,EAAE,CAAC;QACjE,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,OAAO;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAC5C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,WAAW,CAAC;YACpE,OAAO;gBACL,UAAU,CAAC,IAAI;gBACf,WAAW,CAAC;oBACV,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,EAAE;oBACzC,WAAW,EAAE,UAAU,CACrB,UAA8C,CAC/C;oBACD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;wBACvC,IAAI,MAA+C,CAAC;wBACpD,IAAI,CAAC;4BACH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gCAClC,IAAI,EAAE,UAAU,CAAC,IAAI;gCACrB,SAAS,EAAE,CAAC,KAAK,IAAI,EAAE,CAA4B;6BACpD,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,sCAAsC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC/F,CAAC;4BACF,MAAM,KAAK,CAAC;wBACd,CAAC;wBACD,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;4BAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,UAAU,EACV,+BAA+B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAChE,CAAC;wBACJ,CAAC;wBACD,OAAO,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC;oBAClC,CAAC;iBACF,CAAC;aACM,CAAC;QACb,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type { ChatAssertion, ChatMatchers, } from "@skybridge/vite-plugin/evals";
|
|
1
|
+
export type { ChatAssertion, ChatMatchers, JudgmentOptions, } from "@skybridge/vite-plugin/evals";
|
|
2
2
|
export type { Chat } from "./chat.js";
|
|
3
3
|
export type { EvalApp, EvalIdentity } from "./session-registry.js";
|
|
4
4
|
export { start } from "./session-registry.js";
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,aAAa,EACb,YAAY,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,aAAa,EACb,YAAY,EACZ,eAAe,GAChB,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACtC,YAAY,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skybridge/test",
|
|
3
|
-
"version": "0.0.0-dev.
|
|
3
|
+
"version": "0.0.0-dev.fca7959",
|
|
4
4
|
"description": "Eval runner for Skybridge MCP apps: does a prompt reach the right tool with the right arguments",
|
|
5
5
|
"license": "ISC",
|
|
6
6
|
"type": "module",
|
|
@@ -33,8 +33,8 @@
|
|
|
33
33
|
"typescript": "^5.9.3",
|
|
34
34
|
"vitest": "^4.1.10",
|
|
35
35
|
"zod": "^4.4.3",
|
|
36
|
-
"@skybridge/vite-plugin": "0.0.0-dev.
|
|
37
|
-
"skybridge": "0.0.0-dev.
|
|
36
|
+
"@skybridge/vite-plugin": "0.0.0-dev.fca7959",
|
|
37
|
+
"skybridge": "0.0.0-dev.fca7959"
|
|
38
38
|
},
|
|
39
39
|
"engines": {
|
|
40
40
|
"node": ">=22.23.1"
|