@skybridge/vite-plugin 0.0.0-dev.b6f5d60 → 0.0.0-dev.b9466d5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/evals/index.d.ts +1 -1
- package/dist/evals/index.d.ts.map +1 -1
- package/dist/evals/matchers.d.ts +0 -7
- package/dist/evals/matchers.d.ts.map +1 -1
- package/dist/evals/matchers.js +4 -28
- package/dist/evals/matchers.js.map +1 -1
- package/dist/evals/types.d.ts +6 -8
- package/dist/evals/types.d.ts.map +1 -1
- package/package.json +2 -2
package/dist/evals/index.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/evals/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AACjE,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/evals/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AACjE,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC"}
|
package/dist/evals/matchers.d.ts
CHANGED
|
@@ -10,13 +10,6 @@ export interface ChatMatchers<App> {
|
|
|
10
10
|
toHaveCalledToolWith<Name extends ToolNames<App>>(name: Name, args: Partial<ToolInput<App, Name>>): void;
|
|
11
11
|
toNeverHaveCalledTool<Name extends ToolNames<App>>(name: Name): void;
|
|
12
12
|
toHaveFailedToolCall<Name extends ToolNames<App>>(name: Name): void;
|
|
13
|
-
/**
|
|
14
|
-
* Whether any assistant turn matches. A string matches case-insensitively as
|
|
15
|
-
* a substring, with runs of whitespace collapsed; a RegExp is used as
|
|
16
|
-
* written. Model prose varies between runs, so prefer asserting on tool
|
|
17
|
-
* calls where you can, and keep these assertions loose.
|
|
18
|
-
*/
|
|
19
|
-
toHaveSaid(text: string | RegExp): void;
|
|
20
13
|
}
|
|
21
14
|
export interface ChatAssertion<App> extends ChatMatchers<App> {
|
|
22
15
|
not: ChatMatchers<App>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../../src/evals/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D,OAAO,KAAK,EAAE,QAAQ,EAAY,MAAM,YAAY,CAAC;AAErD;;;;GAIG;AACH,MAAM,WAAW,YAAY,CAAC,GAAG;IAC/B,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GACnC,IAAI,CAAC;IACR,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GAClC,IAAI,CAAC;IACR,qBAAqB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;IACrE,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../../src/evals/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7D,OAAO,KAAK,EAAE,QAAQ,EAAY,MAAM,YAAY,CAAC;AAErD;;;;GAIG;AACH,MAAM,WAAW,YAAY,CAAC,GAAG;IAC/B,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GACnC,IAAI,CAAC;IACR,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAC9C,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GAClC,IAAI,CAAC;IACR,qBAAqB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;IACrE,oBAAoB,CAAC,IAAI,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;CACrE;AAED,MAAM,WAAW,aAAa,CAAC,GAAG,CAAE,SAAQ,YAAY,CAAC,GAAG,CAAC;IAC3D,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC;CACxB;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,YAAY;QACpB,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;KACpD;CACF"}
|
package/dist/evals/matchers.js
CHANGED
|
@@ -13,20 +13,9 @@ function observed(chat) {
|
|
|
13
13
|
.map((call, index) => ` ${index + 1}. ${call.name} ${JSON.stringify(call.arguments)}${call.failed === undefined ? "" : ` (failed: ${call.failed})`}`)
|
|
14
14
|
.join("\n");
|
|
15
15
|
}
|
|
16
|
-
function
|
|
17
|
-
return
|
|
18
|
-
}
|
|
19
|
-
function said(chat, text) {
|
|
20
|
-
return chat.assistantTurns.some((turn) => typeof text === "string"
|
|
21
|
-
? collapse(turn).toLowerCase().includes(collapse(text).toLowerCase())
|
|
22
|
-
: text.test(turn));
|
|
23
|
-
}
|
|
24
|
-
function spoken(chat) {
|
|
25
|
-
if (chat.assistantTurns.length === 0) {
|
|
26
|
-
return " (the assistant said nothing)";
|
|
27
|
-
}
|
|
28
|
-
return chat.assistantTurns
|
|
29
|
-
.map((turn, index) => ` ${index + 1}. ${collapse(turn)}`)
|
|
16
|
+
function definitions(chat) {
|
|
17
|
+
return chat.toolDefinitions
|
|
18
|
+
.map((tool) => ` - ${tool.name}: ${tool.description ?? ""}\n arguments: ${JSON.stringify(tool.inputSchema)}`)
|
|
30
19
|
.join("\n");
|
|
31
20
|
}
|
|
32
21
|
function report(chat, summary) {
|
|
@@ -36,7 +25,7 @@ Calls the model actually made:
|
|
|
36
25
|
${observed(chat)}
|
|
37
26
|
|
|
38
27
|
Tool definitions the model was looking at:
|
|
39
|
-
${chat
|
|
28
|
+
${definitions(chat)}`;
|
|
40
29
|
}
|
|
41
30
|
expect.extend({
|
|
42
31
|
toHaveCalledToolOnce(received, name, args) {
|
|
@@ -75,19 +64,6 @@ expect.extend({
|
|
|
75
64
|
: `Expected "${name}" to fail, but all ${attempted.length} calls succeeded.`),
|
|
76
65
|
};
|
|
77
66
|
},
|
|
78
|
-
toHaveSaid(received, text) {
|
|
79
|
-
const pass = said(received, text);
|
|
80
|
-
const wanted = typeof text === "string" ? `"${text}"` : String(text);
|
|
81
|
-
return {
|
|
82
|
-
pass,
|
|
83
|
-
message: () => `${pass
|
|
84
|
-
? `Expected the assistant not to say ${wanted}.`
|
|
85
|
-
: `Expected the assistant to say ${wanted}.`}
|
|
86
|
-
|
|
87
|
-
What the assistant actually said:
|
|
88
|
-
${spoken(received)}`,
|
|
89
|
-
};
|
|
90
|
-
},
|
|
91
67
|
toNeverHaveCalledTool(received, name) {
|
|
92
68
|
const matching = attemptedCalls(received, name);
|
|
93
69
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../../src/evals/matchers.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../../src/evals/matchers.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AA+BhC,SAAS,cAAc,CACrB,IAAuB,EACvB,IAAY;IAEZ,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,aAAa,CACpB,IAAuB,EACvB,IAAY;IAEZ,OAAO,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,QAAQ,CAAC,IAAuB;IACvC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,wBAAwB,CAAC;IAClC,CAAC;IACD,OAAO,IAAI,CAAC,SAAS;SAClB,GAAG,CACF,CAAC,IAAuB,EAAE,KAAa,EAAE,EAAE,CACzC,KAAK,KAAK,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,MAAM,GAAG,EAAE,CACnI;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAAC,IAAuB;IAC1C,OAAO,IAAI,CAAC,eAAe;SACxB,GAAG,CACF,CAAC,IAAI,EAAE,EAAE,CACP,OAAO,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,WAAW,IAAI,EAAE,oBAAoB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CACpG;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,MAAM,CAAC,IAAuB,EAAE,OAAe;IACtD,OAAO,GAAG,OAAO;;;EAGjB,QAAQ,CAAC,IAAI,CAAC;;;EAGd,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACtB,CAAC;AAED,MAAM,CAAC,MAAM,CAAC;IACZ,oBAAoB,CAClB,QAA2B,EAC3B,IAAY,EACZ,IAA8B;QAE9B,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,SAAS,GACb,IAAI,KAAK,SAAS;YAClB,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC;QAEhD,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,aAAa,IAAI,kCAAkC;gBACrD,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;oBACrB,CAAC,CAAC,aAAa,IAAI,4BAA4B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG;oBACxI,CAAC,CAAC,aAAa,IAAI,6DAA6D,QAAQ,CAAC,MAAM,wBAAwB,CAC5H;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAClB,QAA2B,EAC3B,IAAY,EACZ,IAA6B;QAE7B,MAAM,IAAI,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CACvD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAC3D,CAAC;QAEF,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,IAAI;gBACF,CAAC,CAAC,wBAAwB,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG;gBAC/D,CAAC,CAAC,aAAa,IAAI,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CACpE;SACJ,CAAC;IACJ,CAAC;IAED,oBAAoB,CAAC,QAA2B,EAAE,IAAY;QAC5D,MAAM,SAAS,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;QAErE,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;YACvB,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,MAAM,CAAC,MAAM,GAAG,CAAC;gBACf,CAAC,CAAC,2BAA2B,IAAI,eAAe;gBAChD,CAAC,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;oBACtB,CAAC,CAAC,aAAa,IAAI,mDAAmD;oBACtE,CAAC,CAAC,aAAa,IAAI,sBAAsB,SAAS,CAAC,MAAM,mBAAmB,CACjF;SACJ,CAAC;IACJ,CAAC;IAED,qBAAqB,CAAC,QAA2B,EAAE,IAAY;QAC7D,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAEhD,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC3B,OAAO,EAAE,GAAG,EAAE,CACZ,MAAM,CACJ,QAAQ,EACR,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACnB,CAAC,CAAC,aAAa,IAAI,+BAA+B;gBAClD,CAAC,CAAC,aAAa,IAAI,2CAA2C,QAAQ,CAAC,MAAM,SAAS,CACzF;SACJ,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,IAAI,GAAG,CAAM,IAAmB,EAAsB,EAAE,CAC7D,MAAM,CAAC,IAAI,CAAkC,CAAC"}
|
package/dist/evals/types.d.ts
CHANGED
|
@@ -12,17 +12,15 @@ export type ToolCall<App> = {
|
|
|
12
12
|
failed?: string;
|
|
13
13
|
};
|
|
14
14
|
}[ToolNames<App>];
|
|
15
|
+
export interface ToolDefinition {
|
|
16
|
+
name: string;
|
|
17
|
+
description?: string;
|
|
18
|
+
inputSchema: Record<string, unknown>;
|
|
19
|
+
}
|
|
15
20
|
/** The part of a conversation the matchers assert on. */
|
|
16
21
|
export interface ChatLike<App> {
|
|
17
22
|
readonly toolCalls: ToolCall<App>[];
|
|
18
|
-
readonly
|
|
19
|
-
/**
|
|
20
|
-
* The tool definitions the model was looking at, rendered for failure
|
|
21
|
-
* reports. Deliberately a string: descriptions and schemas are prompt text
|
|
22
|
-
* that changes freely, so they are worth reading when an eval fails and not
|
|
23
|
-
* worth asserting on.
|
|
24
|
-
*/
|
|
25
|
-
describeTools(): string;
|
|
23
|
+
readonly toolDefinitions: ToolDefinition[];
|
|
26
24
|
}
|
|
27
25
|
declare module "vitest" {
|
|
28
26
|
interface ProvidedContext {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/evals/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEjD;;;;GAIG;AACH,MAAM,MAAM,QAAQ,CAAC,GAAG,IAAI;KACzB,IAAI,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG;QACxB,IAAI,EAAE,IAAI,CAAC;QACX,SAAS,EAAE,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAChC,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB;CACF,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;AAElB,yDAAyD;AACzD,MAAM,WAAW,QAAQ,CAAC,GAAG;IAC3B,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;IACpC,QAAQ,CAAC,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/evals/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEjD;;;;GAIG;AACH,MAAM,MAAM,QAAQ,CAAC,GAAG,IAAI;KACzB,IAAI,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG;QACxB,IAAI,EAAE,IAAI,CAAC;QACX,SAAS,EAAE,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAChC,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB;CACF,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;AAElB,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAED,yDAAyD;AACzD,MAAM,WAAW,QAAQ,CAAC,GAAG;IAC3B,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;IACpC,QAAQ,CAAC,eAAe,EAAE,cAAc,EAAE,CAAC;CAC5C;AAED,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,eAAe;QACvB,cAAc,EAAE,YAAY,GAAG,SAAS,CAAC;QACzC,iBAAiB,EAAE,MAAM,GAAG,SAAS,CAAC;KACvC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skybridge/vite-plugin",
|
|
3
|
-
"version": "0.0.0-dev.
|
|
3
|
+
"version": "0.0.0-dev.b9466d5",
|
|
4
4
|
"description": "The Skybridge Vite plugin: view discovery, data-llm transforms, and eval wiring",
|
|
5
5
|
"license": "ISC",
|
|
6
6
|
"type": "module",
|
|
@@ -35,7 +35,7 @@
|
|
|
35
35
|
"typescript": "^5.9.3",
|
|
36
36
|
"vite": "^8.1.5",
|
|
37
37
|
"vitest": "^4.1.10",
|
|
38
|
-
"skybridge": "0.0.0-dev.
|
|
38
|
+
"skybridge": "0.0.0-dev.b9466d5"
|
|
39
39
|
},
|
|
40
40
|
"engines": {
|
|
41
41
|
"node": ">=22.23.1"
|