experimental-ash 0.3.0-alpha.35 → 0.3.0-alpha.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/evals/define-eval-suite.js +3 -0
- package/dist/src/evals/define-eval-suite.js.map +1 -1
- package/dist/src/evals/runner/execute-suite.d.ts.map +1 -1
- package/dist/src/evals/runner/execute-suite.js +5 -3
- package/dist/src/evals/runner/execute-suite.js.map +1 -1
- package/dist/src/evals/scorers/autoevals-client.d.ts +15 -0
- package/dist/src/evals/scorers/autoevals-client.d.ts.map +1 -0
- package/dist/src/evals/scorers/autoevals-client.js +88 -0
- package/dist/src/evals/scorers/autoevals-client.js.map +1 -0
- package/dist/src/evals/scorers/autoevals.d.ts +44 -0
- package/dist/src/evals/scorers/autoevals.d.ts.map +1 -0
- package/dist/src/evals/scorers/autoevals.js +75 -0
- package/dist/src/evals/scorers/autoevals.js.map +1 -0
- package/dist/src/evals/scorers/text.d.ts +7 -0
- package/dist/src/evals/scorers/text.d.ts.map +1 -1
- package/dist/src/evals/scorers/text.js +15 -0
- package/dist/src/evals/scorers/text.js.map +1 -1
- package/dist/src/evals/scores/index.d.ts +21 -1
- package/dist/src/evals/scores/index.d.ts.map +1 -1
- package/dist/src/evals/scores/index.js +20 -1
- package/dist/src/evals/scores/index.js.map +1 -1
- package/dist/src/evals/types.d.ts +18 -0
- package/dist/src/evals/types.d.ts.map +1 -1
- package/dist/src/internal/application/package.js +1 -1
- package/dist/src/internal/workflow-bundle/builder.d.ts.map +1 -1
- package/dist/src/internal/workflow-bundle/builder.js +16 -0
- package/dist/src/internal/workflow-bundle/builder.js.map +1 -1
- package/docs/public/evals.md +66 -9
- package/docs/public/typescript-api.md +1 -1
- package/package.json +2 -1
|
@@ -18,6 +18,9 @@ function validateSuiteInput(input) {
|
|
|
18
18
|
if (!input.name || typeof input.name !== "string") {
|
|
19
19
|
throw new Error("Eval suite requires a non-empty string `name`.");
|
|
20
20
|
}
|
|
21
|
+
if (input.model === undefined) {
|
|
22
|
+
throw new Error("Eval suite requires a `model`.");
|
|
23
|
+
}
|
|
21
24
|
if (input.load !== undefined && input.cases !== undefined) {
|
|
22
25
|
throw new Error("Eval suite cannot specify both `load` and `cases`. Use one or the other.");
|
|
23
26
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"define-eval-suite.js","sourceRoot":"","sources":["../../../src/evals/define-eval-suite.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,KAAwB;IACtD,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAE1B,iEAAiE;IACjE,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAI,KAAK,CAAC,KAAgC,CAAC,CAAC,CAAC,CAAC;IAEjG,OAAO;QACL,GAAG,KAAK;QACR,IAAI;QACJ,IAAI,EAAE,cAAc;KACrB,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAwB;IAClD,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,OAAO,KAAK,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IAClE,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,0EAA0E,CAAC,CAAC;IAC9F,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;IACtF,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,EAAE,MAAM,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,EAAE,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CACb,oFAAoF,CACrF,CAAC;IACJ,CAAC;IAED,IACE,KAAK,CAAC,cAAc,KAAK,SAAS;QAClC,CAAC,KAAK,CAAC,cAAc,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,EACrE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC7E,CAAC;IAED,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;QAChG,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;IAClF,CAAC;AACH,CAAC"}
|
|
1
|
+
{"version":3,"file":"define-eval-suite.js","sourceRoot":"","sources":["../../../src/evals/define-eval-suite.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,KAAwB;IACtD,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAE1B,iEAAiE;IACjE,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAI,KAAK,CAAC,KAAgC,CAAC,CAAC,CAAC,CAAC;IAEjG,OAAO;QACL,GAAG,KAAK;QACR,IAAI;QACJ,IAAI,EAAE,cAAc;KACrB,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAwB;IAClD,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,OAAO,KAAK,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IAClE,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,0EAA0E,CAAC,CAAC;IAC9F,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;IACtF,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,EAAE,MAAM,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,EAAE,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CACb,oFAAoF,CACrF,CAAC;IACJ,CAAC;IAED,IACE,KAAK,CAAC,cAAc,KAAK,SAAS;QAClC,CAAC,KAAK,CAAC,cAAc,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,EACrE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC7E,CAAC;IAED,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;QAChG,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;IAClF,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"execute-suite.d.ts","sourceRoot":"","sources":["../../../../src/evals/runner/execute-suite.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"execute-suite.d.ts","sourceRoot":"","sources":["../../../../src/evals/runner/execute-suite.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,KAAK,EAKV,YAAY,EACZ,kBAAkB,EAClB,aAAa,EAEd,MAAM,iBAAiB,CAAC;AAGzB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kCAAkC,CAAC;AAIrE;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,SAAS,EAAE,SAAS,YAAY,EAAE,CAAC;IAC5C,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB;;;;OAIG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;;;GAGG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAgE5F"}
|
|
@@ -85,7 +85,7 @@ async function executeSingleCase(suite, testCase, client) {
|
|
|
85
85
|
},
|
|
86
86
|
};
|
|
87
87
|
}
|
|
88
|
-
const scores = await runScorers(suite
|
|
88
|
+
const scores = await runScorers(suite, testCase, result);
|
|
89
89
|
return {
|
|
90
90
|
case: testCase,
|
|
91
91
|
result,
|
|
@@ -93,7 +93,7 @@ async function executeSingleCase(suite, testCase, client) {
|
|
|
93
93
|
error,
|
|
94
94
|
};
|
|
95
95
|
}
|
|
96
|
-
async function runScorers(
|
|
96
|
+
async function runScorers(suite, testCase, result) {
|
|
97
97
|
const input = typeof testCase.input === "string" ? testCase.input : JSON.stringify(testCase.input);
|
|
98
98
|
const output = String(result.output ?? "");
|
|
99
99
|
const expected = testCase.expected !== undefined ? String(testCase.expected) : undefined;
|
|
@@ -101,11 +101,13 @@ async function runScorers(scorers, testCase, result) {
|
|
|
101
101
|
input,
|
|
102
102
|
output,
|
|
103
103
|
expected,
|
|
104
|
+
model: suite.model,
|
|
105
|
+
modelOptions: suite.modelOptions,
|
|
104
106
|
case: testCase,
|
|
105
107
|
result,
|
|
106
108
|
};
|
|
107
109
|
const results = [];
|
|
108
|
-
for (const scorer of
|
|
110
|
+
for (const scorer of suite.scores) {
|
|
109
111
|
const scorerResult = await scorer(args);
|
|
110
112
|
if (scorerResult !== null) {
|
|
111
113
|
results.push(scorerResult);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"execute-suite.js","sourceRoot":"","sources":["../../../../src/evals/runner/execute-suite.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"execute-suite.js","sourceRoot":"","sources":["../../../../src/evals/runner/execute-suite.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAWnD,OAAO,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AACtF,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAG5D,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAkBlC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC7D,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC9D,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAE3C,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACjC,MAAM,cAAc,GAAG,KAAK,CAAC,cAAc,IAAI,uBAAuB,CAAC;IACvE,MAAM,WAAW,GAAwB,EAAE,CAAC;IAE5C,yCAAyC;IACzC,MAAM,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IAC3B,MAAM,SAAS,GAAG,IAAI,GAAG,EAAiB,CAAC;IAE3C,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QAChD,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,GAAG,cAAc,EAAE,CAAC;YAC7D,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;YACjC,IAAI,QAAQ,KAAK,SAAS;gBAAE,MAAM;YAElC,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE;gBACvB,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACpE,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAE7B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;oBACjC,MAAM,QAAQ,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;gBAC5C,CAAC;YACH,CAAC,CAAC,EAAE,CAAC;YAEL,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE;gBAChC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC5B,CAAC,CAAC,CAAC;YACH,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;QAED,IAAI,SAAS,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAE7C,MAAM,WAAW,GAAuB;QACtC,KAAK,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE;QACzC,MAAM;QACN,KAAK,EAAE,WAAW;QAClB,SAAS;QACT,WAAW;QACX,MAAM,EAAE,WAAW,CAAC,MAAM,CACxB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,CAC9E,CAAC,MAAM;QACR,OAAO,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM;KACjE,CAAC;IAEF,kBAAkB;IAClB,MAAM,WAAW,GAAG,wBAAwB,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;IAChE,MAAM,cAAc,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;IAE/C,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,KAAmB,EACnB,QAAqB,EACrB,MAAc;IAEd,IAAI,MAAyB,CAAC;IAC9B,IAAI,KAAyB,CAAC;IAE9B,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,WAAW,CAAC;YACzB,MAAM;YACN,QAAQ;YACR,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,SAAS,EAAE,KAAK,CAAC,SAAS;SAC3B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,KAAK,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,GAAG;YACP,MAAM,EAAE,IAAI;YACZ,YAAY,EAAE,IAAI;YAClB,MAAM,EAAE,QAAQ;YAChB,MAAM,EAAE,EAAE;YACV,OAAO,EAAE;gBACP,SAAS,EAAE,EAAE;gBACb,aAAa,EAAE,CAAC;gBAChB,aAAa,EAAE,EAAE;gBACjB,iBAAiB,EAAE,CAAC;gBACpB,YAAY,EAAE,CAAC;gBACf,mBAAmB,EAAE,CAAC;aACvB;SACF,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IAEzD,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,MAAM;QACN,MAAM;QACN,KAAK;KACN,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,KAAmB,EACnB,QAAqB,EACrB,MAAyB;IAEzB,MAAM,KAAK,GACT,OAAO,QAAQ,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACvF,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEzF,MAAM,IAAI,GAAsB;QAC9B,KAAK;QACL,MAAM;QACN,QAAQ;QACR,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,IAAI,EAAE,QAAQ;QACd,MAAM;KACP,CAAC;IAEF,MAAM,OAAO,GAA0B,EAAE,CAAC;IAE1C,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;GAIG;AACH,SAAS,cAAc,CACrB,MAA2B,EAC3B,UAA6C;IAE7C,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IACvC,MAAM,SAAS,GAAG,UAAU,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjD,OAAO,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC;AACnC,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { generateText, type LanguageModel } from "ai";
|
|
2
|
+
import { Factuality } from "autoevals";
|
|
3
|
+
/**
|
|
4
|
+
* The OpenAI-shaped client surface autoevals expects. Extracted from the
|
|
5
|
+
* library so we don't take a direct dependency on `openai` types.
|
|
6
|
+
*/
|
|
7
|
+
export type AutoevalsClient = NonNullable<Parameters<typeof Factuality>[0]["client"]>;
|
|
8
|
+
type ProviderOptions = Parameters<typeof generateText>[0]["providerOptions"];
|
|
9
|
+
export interface AutoevalsClientConfig {
|
|
10
|
+
readonly languageModel: LanguageModel;
|
|
11
|
+
readonly providerOptions?: ProviderOptions;
|
|
12
|
+
}
|
|
13
|
+
export declare function createAutoevalsClient(config: AutoevalsClientConfig): AutoevalsClient;
|
|
14
|
+
export {};
|
|
15
|
+
//# sourceMappingURL=autoevals-client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"autoevals-client.d.ts","sourceRoot":"","sources":["../../../../src/evals/scorers/autoevals-client.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,YAAY,EAEZ,KAAK,aAAa,EAInB,MAAM,IAAI,CAAC;AACZ,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC;;;GAGG;AACH,MAAM,MAAM,eAAe,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;AAEtF,KAAK,eAAe,GAAG,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;AAE7E,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,aAAa,EAAE,aAAa,CAAC;IACtC,QAAQ,CAAC,eAAe,CAAC,EAAE,eAAe,CAAC;CAC5C;AAgCD,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,qBAAqB,GAAG,eAAe,CASpF"}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { generateText, jsonSchema, } from "ai";
|
|
2
|
+
import { Factuality } from "autoevals";
|
|
3
|
+
export function createAutoevalsClient(config) {
|
|
4
|
+
const adapter = {
|
|
5
|
+
chat: {
|
|
6
|
+
completions: {
|
|
7
|
+
create: (params) => createChatCompletion(params, config),
|
|
8
|
+
},
|
|
9
|
+
},
|
|
10
|
+
};
|
|
11
|
+
return adapter;
|
|
12
|
+
}
|
|
13
|
+
async function createChatCompletion(params, config) {
|
|
14
|
+
const tools = convertTools(params.tools);
|
|
15
|
+
const result = await generateText({
|
|
16
|
+
model: config.languageModel,
|
|
17
|
+
messages: convertMessages(params.messages ?? []),
|
|
18
|
+
tools: Object.keys(tools).length > 0 ? tools : undefined,
|
|
19
|
+
toolChoice: convertToolChoice(params.tool_choice),
|
|
20
|
+
providerOptions: config.providerOptions,
|
|
21
|
+
});
|
|
22
|
+
const toolCalls = result.toolCalls.map((call) => ({
|
|
23
|
+
id: call.toolCallId,
|
|
24
|
+
type: "function",
|
|
25
|
+
function: {
|
|
26
|
+
name: call.toolName,
|
|
27
|
+
arguments: JSON.stringify(call.input ?? {}),
|
|
28
|
+
},
|
|
29
|
+
}));
|
|
30
|
+
return {
|
|
31
|
+
choices: [
|
|
32
|
+
{
|
|
33
|
+
index: 0,
|
|
34
|
+
finish_reason: toolCalls.length > 0 ? "tool_calls" : "stop",
|
|
35
|
+
message: {
|
|
36
|
+
role: "assistant",
|
|
37
|
+
content: result.text || null,
|
|
38
|
+
tool_calls: toolCalls.length > 0 ? toolCalls : undefined,
|
|
39
|
+
},
|
|
40
|
+
},
|
|
41
|
+
],
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
function convertMessages(messages) {
|
|
45
|
+
return messages.map((message) => {
|
|
46
|
+
const content = contentToText(message.content);
|
|
47
|
+
switch (message.role) {
|
|
48
|
+
case "assistant":
|
|
49
|
+
return { role: "assistant", content };
|
|
50
|
+
case "developer":
|
|
51
|
+
case "system":
|
|
52
|
+
return { role: "system", content };
|
|
53
|
+
case "user":
|
|
54
|
+
default:
|
|
55
|
+
return { role: "user", content };
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
function contentToText(content) {
|
|
60
|
+
if (content == null)
|
|
61
|
+
return "";
|
|
62
|
+
if (typeof content === "string")
|
|
63
|
+
return content;
|
|
64
|
+
return content
|
|
65
|
+
.map((part) => part.text ?? "")
|
|
66
|
+
.filter(Boolean)
|
|
67
|
+
.join("\n");
|
|
68
|
+
}
|
|
69
|
+
function convertTools(tools) {
|
|
70
|
+
const result = {};
|
|
71
|
+
for (const item of tools ?? []) {
|
|
72
|
+
if (item.type !== "function" || item.function?.name === undefined)
|
|
73
|
+
continue;
|
|
74
|
+
result[item.function.name] = {
|
|
75
|
+
description: item.function.description,
|
|
76
|
+
inputSchema: jsonSchema(item.function.parameters ?? {}),
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
return result;
|
|
80
|
+
}
|
|
81
|
+
function convertToolChoice(choice) {
|
|
82
|
+
if (choice === undefined)
|
|
83
|
+
return undefined;
|
|
84
|
+
if (typeof choice === "string")
|
|
85
|
+
return choice;
|
|
86
|
+
return { type: "tool", toolName: choice.function.name };
|
|
87
|
+
}
|
|
88
|
+
//# sourceMappingURL=autoevals-client.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"autoevals-client.js","sourceRoot":"","sources":["../../../../src/evals/scorers/autoevals-client.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,YAAY,EACZ,UAAU,GAKX,MAAM,IAAI,CAAC;AACZ,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AA6CvC,MAAM,UAAU,qBAAqB,CAAC,MAA6B;IACjE,MAAM,OAAO,GAAY;QACvB,IAAI,EAAE;YACJ,WAAW,EAAE;gBACX,MAAM,EAAE,CAAC,MAAkB,EAAE,EAAE,CAAC,oBAAoB,CAAC,MAAM,EAAE,MAAM,CAAC;aACrE;SACF;KACF,CAAC;IACF,OAAO,OAA0B,CAAC;AACpC,CAAC;AAED,KAAK,UAAU,oBAAoB,CACjC,MAAkB,EAClB,MAA6B;IAE7B,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;QAChC,KAAK,EAAE,MAAM,CAAC,aAAa;QAC3B,QAAQ,EAAE,eAAe,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC;QAChD,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;QACxD,UAAU,EAAE,iBAAiB,CAAC,MAAM,CAAC,WAAW,CAAC;QACjD,eAAe,EAAE,MAAM,CAAC,eAAe;KACxC,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAChD,EAAE,EAAE,IAAI,CAAC,UAAU;QACnB,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE;YACR,IAAI,EAAE,IAAI,CAAC,QAAQ;YACnB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;SAC5C;KACF,CAAC,CAAC,CAAC;IAEJ,OAAO;QACL,OAAO,EAAE;YACP;gBACE,KAAK,EAAE,CAAC;gBACR,aAAa,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM;gBAC3D,OAAO,EAAE;oBACP,IAAI,EAAE,WAAW;oBACjB,OAAO,EAAE,MAAM,CAAC,IAAI,IAAI,IAAI;oBAC5B,UAAU,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;iBACzD;aACF;SACF;KACF,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,QAAgC;IACvD,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QAC9B,MAAM,OAAO,GAAG,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/C,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;YACrB,KAAK,WAAW;gBACd,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC;YACxC,KAAK,WAAW,CAAC;YACjB,KAAK,QAAQ;gBACX,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;YACrC,KAAK,MAAM,CAAC;YACZ;gBACE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;QACrC,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,aAAa,CAAC,OAA+B;IACpD,IAAI,OAAO,IAAI,IAAI;QAAE,OAAO,EAAE,CAAC;IAC/B,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC;IAChD,OAAO,OAAO;SACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;SAC9B,MAAM,CAAC,OAAO,CAAC;SACf,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,KAAsC;IAC1D,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,IAAI,EAAE,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,IAAI,KAAK,UAAU,IAAI,IAAI,CAAC,QAAQ,EAAE,IAAI,KAAK,SAAS;YAAE,SAAS;QAC5E,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG;YAC3B,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,WAAW;YACtC,WAAW,EAAE,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC;SACxD,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,iBAAiB,CAAC,MAAkC;IAC3D,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC3C,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC;IAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;AAC1D,CAAC"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { type LanguageModel } from "ai";
|
|
2
|
+
import type { AgentModelOptionsDefinition } from "#shared/agent-definition.js";
|
|
3
|
+
import type { AshEvalScorer } from "#evals/types.js";
|
|
4
|
+
/**
|
|
5
|
+
* Optional per-scorer overrides for Autoevals language scorers. When omitted,
|
|
6
|
+
* the suite's `model` and `modelOptions` are used.
|
|
7
|
+
*/
|
|
8
|
+
export interface AutoevalsModelScorerOptions {
|
|
9
|
+
readonly model?: LanguageModel;
|
|
10
|
+
readonly modelOptions?: AgentModelOptionsDefinition;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Options for {@link closedQA}. Extends {@link AutoevalsModelScorerOptions}
|
|
14
|
+
* with the required free-form `criteria` the judge applies.
|
|
15
|
+
*/
|
|
16
|
+
export interface AutoevalsClosedQAOptions extends AutoevalsModelScorerOptions {
|
|
17
|
+
/**
|
|
18
|
+
* Natural-language criterion the judge model evaluates the output against.
|
|
19
|
+
*
|
|
20
|
+
* @example "The answer must mention the refund policy."
|
|
21
|
+
*/
|
|
22
|
+
readonly criteria: string;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Test whether an output is factual compared to the expected answer.
|
|
26
|
+
*/
|
|
27
|
+
export declare function factuality(options?: AutoevalsModelScorerOptions): AshEvalScorer;
|
|
28
|
+
/**
|
|
29
|
+
* Test whether an output is a better summary of the input than the expected value.
|
|
30
|
+
*/
|
|
31
|
+
export declare function summary(options?: AutoevalsModelScorerOptions): AshEvalScorer;
|
|
32
|
+
/**
|
|
33
|
+
* Test whether an output SQL query is semantically equivalent to the expected query.
|
|
34
|
+
*/
|
|
35
|
+
export declare function sql(options?: AutoevalsModelScorerOptions): AshEvalScorer;
|
|
36
|
+
/**
|
|
37
|
+
* Judge whether an output meets a free-form natural-language `criteria`.
|
|
38
|
+
*
|
|
39
|
+
* Use this when "correct" can't be expressed as exact/fuzzy match against an
|
|
40
|
+
* `expected` value — for example, "must cite a source" or "must apologize
|
|
41
|
+
* before refusing." Per-case `expected` is ignored.
|
|
42
|
+
*/
|
|
43
|
+
export declare function closedQA(options: AutoevalsClosedQAOptions): AshEvalScorer;
|
|
44
|
+
//# sourceMappingURL=autoevals.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"autoevals.d.ts","sourceRoot":"","sources":["../../../../src/evals/scorers/autoevals.ts"],"names":[],"mappings":"AAMA,OAAO,EAAgB,KAAK,aAAa,EAAE,MAAM,IAAI,CAAC;AAEtD,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,6BAA6B,CAAC;AAG/E,OAAO,KAAK,EAAE,aAAa,EAA0C,MAAM,iBAAiB,CAAC;AAI7F;;;GAGG;AACH,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,CAAC,KAAK,CAAC,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,YAAY,CAAC,EAAE,2BAA2B,CAAC;CACrD;AAED;;;GAGG;AACH,MAAM,WAAW,wBAAyB,SAAQ,2BAA2B;IAC3E;;;;OAIG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAID;;GAEG;AACH,wBAAgB,UAAU,CAAC,OAAO,GAAE,2BAAgC,GAAG,aAAa,CAEnF;AAED;;GAEG;AACH,wBAAgB,OAAO,CAAC,OAAO,GAAE,2BAAgC,GAAG,aAAa,CAEhF;AAED;;GAEG;AACH,wBAAgB,GAAG,CAAC,OAAO,GAAE,2BAAgC,GAAG,aAAa,CAE5E;AAED;;;;;;GAMG;AACH,wBAAgB,QAAQ,CAAC,OAAO,EAAE,wBAAwB,GAAG,aAAa,CAezE"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { ClosedQA as AutoevalsClosedQA, Factuality as AutoevalsFactuality, Sql as AutoevalsSql, Summary as AutoevalsSummary, } from "autoevals";
|
|
2
|
+
import { generateText } from "ai";
|
|
3
|
+
import { formatLanguageModelGatewayId } from "#internal/runtime-model.js";
|
|
4
|
+
import { createAutoevalsClient } from "#evals/scorers/autoevals-client.js";
|
|
5
|
+
/**
|
|
6
|
+
* Test whether an output is factual compared to the expected answer.
|
|
7
|
+
*/
|
|
8
|
+
export function factuality(options = {}) {
|
|
9
|
+
return languageScorer(AutoevalsFactuality, options);
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Test whether an output is a better summary of the input than the expected value.
|
|
13
|
+
*/
|
|
14
|
+
export function summary(options = {}) {
|
|
15
|
+
return languageScorer(AutoevalsSummary, options);
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Test whether an output SQL query is semantically equivalent to the expected query.
|
|
19
|
+
*/
|
|
20
|
+
export function sql(options = {}) {
|
|
21
|
+
return languageScorer(AutoevalsSql, options);
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Judge whether an output meets a free-form natural-language `criteria`.
|
|
25
|
+
*
|
|
26
|
+
* Use this when "correct" can't be expressed as exact/fuzzy match against an
|
|
27
|
+
* `expected` value — for example, "must cite a source" or "must apologize
|
|
28
|
+
* before refusing." Per-case `expected` is ignored.
|
|
29
|
+
*/
|
|
30
|
+
export function closedQA(options) {
|
|
31
|
+
return async (args) => {
|
|
32
|
+
const model = options.model ?? args.model;
|
|
33
|
+
const score = await AutoevalsClosedQA({
|
|
34
|
+
input: args.input,
|
|
35
|
+
output: args.output,
|
|
36
|
+
criteria: options.criteria,
|
|
37
|
+
model: formatLanguageModelGatewayId(model),
|
|
38
|
+
client: createAutoevalsClient({
|
|
39
|
+
languageModel: model,
|
|
40
|
+
providerOptions: mergeProviderOptions(args, options.modelOptions),
|
|
41
|
+
}),
|
|
42
|
+
});
|
|
43
|
+
return toScorerResult(score);
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
function languageScorer(scorer, options) {
|
|
47
|
+
return async (args) => {
|
|
48
|
+
const model = options.model ?? args.model;
|
|
49
|
+
const score = await scorer({
|
|
50
|
+
input: args.input,
|
|
51
|
+
output: args.output,
|
|
52
|
+
expected: args.expected,
|
|
53
|
+
model: formatLanguageModelGatewayId(model),
|
|
54
|
+
client: createAutoevalsClient({
|
|
55
|
+
languageModel: model,
|
|
56
|
+
providerOptions: mergeProviderOptions(args, options.modelOptions),
|
|
57
|
+
}),
|
|
58
|
+
});
|
|
59
|
+
return toScorerResult(score);
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
function mergeProviderOptions(args, override) {
|
|
63
|
+
const suite = args.modelOptions?.providerOptions;
|
|
64
|
+
const scorer = override?.providerOptions;
|
|
65
|
+
if (suite === undefined && scorer === undefined)
|
|
66
|
+
return undefined;
|
|
67
|
+
// Ash's deeply-readonly `JsonObject` is structurally identical to the AI
|
|
68
|
+
// SDK's `JSONObject`. The mutability brand is the only difference, so
|
|
69
|
+
// cross the boundary once here.
|
|
70
|
+
return { ...suite, ...scorer };
|
|
71
|
+
}
|
|
72
|
+
function toScorerResult(score) {
|
|
73
|
+
return { name: score.name, score: score.score, metadata: score.metadata };
|
|
74
|
+
}
|
|
75
|
+
//# sourceMappingURL=autoevals.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"autoevals.js","sourceRoot":"","sources":["../../../../src/evals/scorers/autoevals.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,iBAAiB,EAC7B,UAAU,IAAI,mBAAmB,EACjC,GAAG,IAAI,YAAY,EACnB,OAAO,IAAI,gBAAgB,GAC5B,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,YAAY,EAAsB,MAAM,IAAI,CAAC;AAGtD,OAAO,EAAE,4BAA4B,EAAE,MAAM,4BAA4B,CAAC;AAC1E,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AA6B3E;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,OAAO,GAAgC,EAAE;IAClE,OAAO,cAAc,CAAC,mBAAmB,EAAE,OAAO,CAAC,CAAC;AACtD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,OAAO,CAAC,OAAO,GAAgC,EAAE;IAC/D,OAAO,cAAc,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC;AACnD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,GAAG,CAAC,OAAO,GAAgC,EAAE;IAC3D,OAAO,cAAc,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;AAC/C,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CAAC,OAAiC;IACxD,OAAO,KAAK,EAAE,IAAI,EAAE,EAAE;QACpB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC;YACpC,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,KAAK,EAAE,4BAA4B,CAAC,KAAK,CAAC;YAC1C,MAAM,EAAE,qBAAqB,CAAC;gBAC5B,aAAa,EAAE,KAAK;gBACpB,eAAe,EAAE,oBAAoB,CAAC,IAAI,EAAE,OAAO,CAAC,YAAY,CAAC;aAClE,CAAC;SACH,CAAC,CAAC;QACH,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CACrB,MAAsB,EACtB,OAAoC;IAEpC,OAAO,KAAK,EAAE,IAAI,EAAE,EAAE;QACpB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,4BAA4B,CAAC,KAAK,CAAC;YAC1C,MAAM,EAAE,qBAAqB,CAAC;gBAC5B,aAAa,EAAE,KAAK;gBACpB,eAAe,EAAE,oBAAoB,CAAC,IAAI,EAAE,OAAO,CAAC,YAAY,CAAC;aAClE,CAAC;SACH,CAAC,CAAC;QACH,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAC3B,IAAuB,EACvB,QAAiD;IAEjD,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC;IACjD,MAAM,MAAM,GAAG,QAAQ,EAAE,eAAe,CAAC;IACzC,IAAI,KAAK,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAClE,yEAAyE;IACzE,sEAAsE;IACtE,gCAAgC;IAChC,OAAO,EAAE,GAAG,KAAK,EAAE,GAAG,MAAM,EAAqB,CAAC;AACpD,CAAC;AAED,SAAS,cAAc,CAAC,KAIvB;IACC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;AAC5E,CAAC"}
|
|
@@ -9,4 +9,11 @@ export declare function exact(): AshEvalScorer;
|
|
|
9
9
|
* 0 otherwise. Returns `null` when expected is absent.
|
|
10
10
|
*/
|
|
11
11
|
export declare function includes(): AshEvalScorer;
|
|
12
|
+
/**
|
|
13
|
+
* Scores by normalized character-level Levenshtein similarity
|
|
14
|
+
* (1 = identical, 0 = entirely different). Useful for fuzzy text
|
|
15
|
+
* comparisons where exact match is too strict and an LLM judge or
|
|
16
|
+
* embedding model is overkill. Returns `null` when expected is absent.
|
|
17
|
+
*/
|
|
18
|
+
export declare function levenshtein(): AshEvalScorer;
|
|
12
19
|
//# sourceMappingURL=text.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"text.d.ts","sourceRoot":"","sources":["../../../../src/evals/scorers/text.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"text.d.ts","sourceRoot":"","sources":["../../../../src/evals/scorers/text.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAErD;;;GAGG;AACH,wBAAgB,KAAK,IAAI,aAAa,CAMrC;AAED;;;GAGG;AACH,wBAAgB,QAAQ,IAAI,aAAa,CAMxC;AAED;;;;;GAKG;AACH,wBAAgB,WAAW,IAAI,aAAa,CAM3C"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Levenshtein as AutoevalsLevenshtein } from "autoevals";
|
|
1
2
|
/**
|
|
2
3
|
* Scores 1 when `output` exactly equals `expected` (string comparison),
|
|
3
4
|
* 0 otherwise. Returns `null` when expected is absent.
|
|
@@ -22,4 +23,18 @@ export function includes() {
|
|
|
22
23
|
return { name: "text.includes", score };
|
|
23
24
|
};
|
|
24
25
|
}
|
|
26
|
+
/**
|
|
27
|
+
* Scores by normalized character-level Levenshtein similarity
|
|
28
|
+
* (1 = identical, 0 = entirely different). Useful for fuzzy text
|
|
29
|
+
* comparisons where exact match is too strict and an LLM judge or
|
|
30
|
+
* embedding model is overkill. Returns `null` when expected is absent.
|
|
31
|
+
*/
|
|
32
|
+
export function levenshtein() {
|
|
33
|
+
return async ({ output, expected }) => {
|
|
34
|
+
if (expected === undefined)
|
|
35
|
+
return null;
|
|
36
|
+
const score = await AutoevalsLevenshtein({ output, expected });
|
|
37
|
+
return { name: "text.levenshtein", score: score.score };
|
|
38
|
+
};
|
|
39
|
+
}
|
|
25
40
|
//# sourceMappingURL=text.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"text.js","sourceRoot":"","sources":["../../../../src/evals/scorers/text.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"text.js","sourceRoot":"","sources":["../../../../src/evals/scorers/text.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,IAAI,oBAAoB,EAAE,MAAM,WAAW,CAAC;AAIhE;;;GAGG;AACH,MAAM,UAAU,KAAK;IACnB,OAAO,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE;QAC9B,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC;QACxC,MAAM,KAAK,GAAG,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1C,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;IACvC,CAAC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,QAAQ;IACtB,OAAO,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE;QAC9B,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC;QACxC,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAChD,OAAO,EAAE,IAAI,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC;IAC1C,CAAC,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,WAAW;IACzB,OAAO,KAAK,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE;QACpC,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC;QACxC,MAAM,KAAK,GAAG,MAAM,oBAAoB,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC/D,OAAO,EAAE,IAAI,EAAE,kBAAkB,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;IAC1D,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -2,8 +2,10 @@ import * as JsonScorers from "#evals/scorers/json.js";
|
|
|
2
2
|
import * as RunScorers from "#evals/scorers/run.js";
|
|
3
3
|
import * as SqlScorers from "#evals/scorers/sql.js";
|
|
4
4
|
import * as TextScorers from "#evals/scorers/text.js";
|
|
5
|
+
import * as AutoevalsScorers from "#evals/scorers/autoevals.js";
|
|
6
|
+
export type { AutoevalsClosedQAOptions, AutoevalsModelScorerOptions, } from "#evals/scorers/autoevals.js";
|
|
5
7
|
/**
|
|
6
|
-
* Text scorers for exact and
|
|
8
|
+
* Text scorers for exact, substring, and fuzzy character-level matching.
|
|
7
9
|
*
|
|
8
10
|
* @example
|
|
9
11
|
* ```ts
|
|
@@ -11,6 +13,7 @@ import * as TextScorers from "#evals/scorers/text.js";
|
|
|
11
13
|
*
|
|
12
14
|
* Text.exact()
|
|
13
15
|
* Text.includes()
|
|
16
|
+
* Text.levenshtein()
|
|
14
17
|
* ```
|
|
15
18
|
*/
|
|
16
19
|
export declare const Text: typeof TextScorers;
|
|
@@ -50,4 +53,21 @@ export declare const Sql: typeof SqlScorers;
|
|
|
50
53
|
* ```
|
|
51
54
|
*/
|
|
52
55
|
export declare const Run: typeof RunScorers;
|
|
56
|
+
/**
|
|
57
|
+
* Ash-owned wrappers around Braintrust Autoevals scorers.
|
|
58
|
+
*
|
|
59
|
+
* String model IDs route through the Vercel AI Gateway, while AI SDK model
|
|
60
|
+
* instances are used directly.
|
|
61
|
+
*
|
|
62
|
+
* @example
|
|
63
|
+
* ```ts
|
|
64
|
+
* import { Autoevals } from "experimental-ash/evals/scores";
|
|
65
|
+
*
|
|
66
|
+
* Autoevals.factuality()
|
|
67
|
+
* Autoevals.summary()
|
|
68
|
+
* Autoevals.sql()
|
|
69
|
+
* Autoevals.closedQA({ criteria: "Must cite a source." })
|
|
70
|
+
* ```
|
|
71
|
+
*/
|
|
72
|
+
export declare const Autoevals: typeof AutoevalsScorers;
|
|
53
73
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/evals/scores/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,wBAAwB,CAAC;AACtD,OAAO,KAAK,UAAU,MAAM,uBAAuB,CAAC;AACpD,OAAO,KAAK,UAAU,MAAM,uBAAuB,CAAC;AACpD,OAAO,KAAK,WAAW,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/evals/scores/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,wBAAwB,CAAC;AACtD,OAAO,KAAK,UAAU,MAAM,uBAAuB,CAAC;AACpD,OAAO,KAAK,UAAU,MAAM,uBAAuB,CAAC;AACpD,OAAO,KAAK,WAAW,MAAM,wBAAwB,CAAC;AACtD,OAAO,KAAK,gBAAgB,MAAM,6BAA6B,CAAC;AAEhE,YAAY,EACV,wBAAwB,EACxB,2BAA2B,GAC5B,MAAM,6BAA6B,CAAC;AAErC;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,IAAI,oBAAc,CAAC;AAEhC;;;;;;;;;GASG;AACH,eAAO,MAAM,IAAI,oBAAc,CAAC;AAEhC;;;;;;;;;GASG;AACH,eAAO,MAAM,GAAG,mBAAa,CAAC;AAE9B;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,GAAG,mBAAa,CAAC;AAE9B;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,SAAS,yBAAmB,CAAC"}
|
|
@@ -2,8 +2,9 @@ import * as JsonScorers from "#evals/scorers/json.js";
|
|
|
2
2
|
import * as RunScorers from "#evals/scorers/run.js";
|
|
3
3
|
import * as SqlScorers from "#evals/scorers/sql.js";
|
|
4
4
|
import * as TextScorers from "#evals/scorers/text.js";
|
|
5
|
+
import * as AutoevalsScorers from "#evals/scorers/autoevals.js";
|
|
5
6
|
/**
|
|
6
|
-
* Text scorers for exact and
|
|
7
|
+
* Text scorers for exact, substring, and fuzzy character-level matching.
|
|
7
8
|
*
|
|
8
9
|
* @example
|
|
9
10
|
* ```ts
|
|
@@ -11,6 +12,7 @@ import * as TextScorers from "#evals/scorers/text.js";
|
|
|
11
12
|
*
|
|
12
13
|
* Text.exact()
|
|
13
14
|
* Text.includes()
|
|
15
|
+
* Text.levenshtein()
|
|
14
16
|
* ```
|
|
15
17
|
*/
|
|
16
18
|
export const Text = TextScorers;
|
|
@@ -50,4 +52,21 @@ export const Sql = SqlScorers;
|
|
|
50
52
|
* ```
|
|
51
53
|
*/
|
|
52
54
|
export const Run = RunScorers;
|
|
55
|
+
/**
|
|
56
|
+
* Ash-owned wrappers around Braintrust Autoevals scorers.
|
|
57
|
+
*
|
|
58
|
+
* String model IDs route through the Vercel AI Gateway, while AI SDK model
|
|
59
|
+
* instances are used directly.
|
|
60
|
+
*
|
|
61
|
+
* @example
|
|
62
|
+
* ```ts
|
|
63
|
+
* import { Autoevals } from "experimental-ash/evals/scores";
|
|
64
|
+
*
|
|
65
|
+
* Autoevals.factuality()
|
|
66
|
+
* Autoevals.summary()
|
|
67
|
+
* Autoevals.sql()
|
|
68
|
+
* Autoevals.closedQA({ criteria: "Must cite a source." })
|
|
69
|
+
* ```
|
|
70
|
+
*/
|
|
71
|
+
export const Autoevals = AutoevalsScorers;
|
|
53
72
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/evals/scores/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,wBAAwB,CAAC;AACtD,OAAO,KAAK,UAAU,MAAM,uBAAuB,CAAC;AACpD,OAAO,KAAK,UAAU,MAAM,uBAAuB,CAAC;AACpD,OAAO,KAAK,WAAW,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/evals/scores/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,wBAAwB,CAAC;AACtD,OAAO,KAAK,UAAU,MAAM,uBAAuB,CAAC;AACpD,OAAO,KAAK,UAAU,MAAM,uBAAuB,CAAC;AACpD,OAAO,KAAK,WAAW,MAAM,wBAAwB,CAAC;AACtD,OAAO,KAAK,gBAAgB,MAAM,6BAA6B,CAAC;AAOhE;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,IAAI,GAAG,WAAW,CAAC;AAEhC;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,IAAI,GAAG,WAAW,CAAC;AAEhC;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,GAAG,GAAG,UAAU,CAAC;AAE9B;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,GAAG,GAAG,UAAU,CAAC;AAE9B;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,gBAAgB,CAAC"}
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
import type { LanguageModel } from "ai";
|
|
1
2
|
import type { HandleMessageStreamEvent, RuntimeIdentity } from "#protocol/message.js";
|
|
3
|
+
import type { AgentModelOptionsDefinition } from "#shared/agent-definition.js";
|
|
2
4
|
import type { EvalReporter } from "#evals/runner/reporters/types.js";
|
|
3
5
|
/**
|
|
4
6
|
* One normalized eval case. Suites produce these from raw data sources.
|
|
@@ -67,6 +69,10 @@ export interface AshEvalScorerArgs {
|
|
|
67
69
|
readonly output: string;
|
|
68
70
|
/** The expected value, coerced to string. Autoevals-compatible. */
|
|
69
71
|
readonly expected?: string | undefined;
|
|
72
|
+
/** Suite-level scorer model. This does not change the target Ash agent model. */
|
|
73
|
+
readonly model: LanguageModel;
|
|
74
|
+
/** Suite-level provider options for model-backed scorers. */
|
|
75
|
+
readonly modelOptions?: AgentModelOptionsDefinition;
|
|
70
76
|
/** Full eval case. */
|
|
71
77
|
readonly case: AshEvalCase;
|
|
72
78
|
/** Full task result with events and derived facts. */
|
|
@@ -116,6 +122,18 @@ interface AshEvalSuiteBase {
|
|
|
116
122
|
readonly description?: string;
|
|
117
123
|
readonly task?: AshEvalTask;
|
|
118
124
|
readonly scores: readonly AshEvalScorer[];
|
|
125
|
+
/**
|
|
126
|
+
* Required model used by model-backed scorers in this suite.
|
|
127
|
+
*
|
|
128
|
+
* String model IDs route through the Vercel AI Gateway, while provider
|
|
129
|
+
* model instances are used directly. This model is only for scoring and
|
|
130
|
+
* never changes the Ash agent under test.
|
|
131
|
+
*/
|
|
132
|
+
readonly model: LanguageModel;
|
|
133
|
+
/**
|
|
134
|
+
* Provider-specific options passed to model-backed scorers.
|
|
135
|
+
*/
|
|
136
|
+
readonly modelOptions?: AgentModelOptionsDefinition;
|
|
119
137
|
/**
|
|
120
138
|
* Maximum number of cases the runner executes at once.
|
|
121
139
|
* Defaults to 8 when omitted.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/evals/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,wBAAwB,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACtF,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kCAAkC,CAAC;AAErE;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjD,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACvD;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,SAAS,CAAC;IACpD,QAAQ,CAAC,MAAM,EAAE,SAAS,wBAAwB,EAAE,CAAC;IACrD,QAAQ,CAAC,OAAO,EAAE,mBAAmB,CAAC;IACtC;;;OAGG;IACH,QAAQ,CAAC,eAAe,CAAC,EAAE,eAAe,CAAC;CAC5C;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,6DAA6D;IAC7D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,8EAA8E;IAC9E,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,kEAAkE;IAClE,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACvD;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,iBAAiB;IAChC,oEAAoE;IACpE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,gEAAgE;IAChE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,mEAAmE;IACnE,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,sBAAsB;IACtB,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,sDAAsD;IACtD,QAAQ,CAAC,MAAM,EAAE,iBAAiB,CAAC;CACpC;AAED;;;;GAIG;AACH,MAAM,MAAM,aAAa,GAAG,CAC1B,IAAI,EAAE,iBAAiB,KACpB,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;AAEtE;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B;;;OAGG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,WAAW,KAAK,MAAM,CAAC;IAEpD;;;OAGG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,WAAW,KAAK,MAAM,EAAE,CAAC;IAExD;;;OAGG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,iBAAiB,KAAK,OAAO,CAAC;CAC/D;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,OAAO,GAAG,QAAQ,CAAC;IAClC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAMD;;GAEG;AACH,UAAU,gBAAgB;IACxB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,IAAI,CAAC,EAAE,WAAW,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,SAAS,aAAa,EAAE,CAAC;IAC1C;;;OAGG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACtD,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IAC7C;;;;;;;;;;;;OAYG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CACxD;AAED;;;;;;GAMG;AACH,MAAM,WAAW,iBAAkB,SAAQ,gBAAgB;IACzD,+DAA+D;IAC/D,IAAI,CAAC,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAChC,2DAA2D;IAC3D,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;CACzC;AAED;;;GAGG;AACH,MAAM,WAAW,YAAa,SAAQ,gBAAgB;IACpD,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;IAC9B,iFAAiF;IACjF,IAAI,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,iBAAiB,CAAC;IACnC,QAAQ,CAAC,MAAM,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAChD,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,KAAK,EAAE;QAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/D,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,KAAK,EAAE,SAAS,iBAAiB,EAAE,CAAC;IAC7C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,6EAA6E;IAC7E,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/evals/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAExC,OAAO,KAAK,EAAE,wBAAwB,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACtF,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,6BAA6B,CAAC;AAC/E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kCAAkC,CAAC;AAErE;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjD,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACvD;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,SAAS,CAAC;IACpD,QAAQ,CAAC,MAAM,EAAE,SAAS,wBAAwB,EAAE,CAAC;IACrD,QAAQ,CAAC,OAAO,EAAE,mBAAmB,CAAC;IACtC;;;OAGG;IACH,QAAQ,CAAC,eAAe,CAAC,EAAE,eAAe,CAAC;CAC5C;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,6DAA6D;IAC7D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,8EAA8E;IAC9E,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,kEAAkE;IAClE,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACvD;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,iBAAiB;IAChC,oEAAoE;IACpE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,gEAAgE;IAChE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,mEAAmE;IACnE,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,iFAAiF;IACjF,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;IAC9B,6DAA6D;IAC7D,QAAQ,CAAC,YAAY,CAAC,EAAE,2BAA2B,CAAC;IACpD,sBAAsB;IACtB,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,sDAAsD;IACtD,QAAQ,CAAC,MAAM,EAAE,iBAAiB,CAAC;CACpC;AAED;;;;GAIG;AACH,MAAM,MAAM,aAAa,GAAG,CAC1B,IAAI,EAAE,iBAAiB,KACpB,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;AAEtE;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B;;;OAGG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,WAAW,KAAK,MAAM,CAAC;IAEpD;;;OAGG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,WAAW,KAAK,MAAM,EAAE,CAAC;IAExD;;;OAGG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,iBAAiB,KAAK,OAAO,CAAC;CAC/D;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,OAAO,GAAG,QAAQ,CAAC;IAClC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAMD;;GAEG;AACH,UAAU,gBAAgB;IACxB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,IAAI,CAAC,EAAE,WAAW,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,SAAS,aAAa,EAAE,CAAC;IAC1C;;;;;;OAMG;IACH,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;IAC9B;;OAEG;IACH,QAAQ,CAAC,YAAY,CAAC,EAAE,2BAA2B,CAAC;IACpD;;;OAGG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACtD,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IAC7C;;;;;;;;;;;;OAYG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CACxD;AAED;;;;;;GAMG;AACH,MAAM,WAAW,iBAAkB,SAAQ,gBAAgB;IACzD,+DAA+D;IAC/D,IAAI,CAAC,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAChC,2DAA2D;IAC3D,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;CACzC;AAED;;;GAGG;AACH,MAAM,WAAW,YAAa,SAAQ,gBAAgB;IACpD,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;IAC9B,iFAAiF;IACjF,IAAI,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,iBAAiB,CAAC;IACnC,QAAQ,CAAC,MAAM,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAChD,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,KAAK,EAAE;QAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/D,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,QAAQ,CAAC,KAAK,EAAE,SAAS,iBAAiB,EAAE,CAAC;IAC7C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,6EAA6E;IAC7E,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B"}
|
|
@@ -6,7 +6,7 @@ import { ASH_PACKAGE_NAME } from "#package-name.js";
|
|
|
6
6
|
let cachedPackageInfo;
|
|
7
7
|
// The package build stamps the published version into `dist` so bundled
|
|
8
8
|
// deployments can still report package metadata without resolving package.json.
|
|
9
|
-
const BUNDLED_FALLBACK_PACKAGE_VERSION = "0.3.0-alpha.
|
|
9
|
+
const BUNDLED_FALLBACK_PACKAGE_VERSION = "0.3.0-alpha.36";
|
|
10
10
|
const BUNDLED_FALLBACK_PACKAGE_VERSION_PLACEHOLDER = "__ASH_PACKAGE_VERSION_PLACEHOLDER__";
|
|
11
11
|
const WORKFLOW_MODULE_ALIASES = {
|
|
12
12
|
"workflow/api": "@workflow/core/runtime",
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"builder.d.ts","sourceRoot":"","sources":["../../../../src/internal/workflow-bundle/builder.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,WAAW,EAA8C,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"builder.d.ts","sourceRoot":"","sources":["../../../../src/internal/workflow-bundle/builder.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,WAAW,EAA8C,MAAM,oBAAoB,CAAC;AA+B7F;;GAEG;AACH,qBAAa,qBAAsB,SAAQ,WAAW;;IAIpD,YAAY,OAAO,EAAE,4BAA4B,EAoBhD;IAED,cAAuB,wBAAwB,IAAI,OAAO,CAEzD;IAEK,KAAK,CACT,OAAO,GAAE;QAAE,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAAC,oBAAoB,CAAC,EAAE,MAAM,CAAA;KAAO,GACzE,OAAO,CAAC,IAAI,CAAC,CAQf;IA8FD;;;OAGG;IACG,iBAAiB,CAAC,OAAO,EAAE;QAC/B,kBAAkB,EAAE,MAAM,CAAC;QAC3B,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,kBAAkB,EAAE,MAAM,CAAC;KAC5B,GAAG,OAAO,CAAC,IAAI,CAAC,CAqHhB;CA2DF;AA0JD,UAAU,4BAA4B;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,8BAA8B,EAAE,MAAM,CAAC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,OAAO,CAAC;CAChB"}
|
|
@@ -7,6 +7,16 @@ import { prepareAshVersionedCacheDirectory, writeAshVersionedCacheMetadata, } fr
|
|
|
7
7
|
import { normalizeEsmImportSpecifier } from "#internal/application/import-specifier.js";
|
|
8
8
|
import { writeNitroStepEntrypoint } from "#internal/workflow-bundle/nitro-step-entry.js";
|
|
9
9
|
import { copyNitroFunctionDirectory, createWorkflowFunctionEnvironment, retargetNitroFunctionDirectoryToWorkflowRoute, WORKFLOW_BUILDER_DEFERRED_PACKAGES, WORKFLOW_STEP_EXTERNAL_PACKAGES, } from "#internal/workflow-bundle/vercel-workflow-output.js";
|
|
10
|
+
/**
|
|
11
|
+
* Process-level lock keyed on `outDir` so concurrent `WorkflowBundleBuilder`
|
|
12
|
+
* instances writing to the same workflow cache directory serialize their
|
|
13
|
+
* `build()` calls. Without this, parallel Vercel surface builds (`flow` and
|
|
14
|
+
* `step`) race on the post-`createWorkflowsBundle` rewrite passes and rolldown
|
|
15
|
+
* can read `workflows.mjs` while it still contains the giant template-literal
|
|
16
|
+
* form emitted by `@workflow/builders`, surfacing as an "Unterminated string"
|
|
17
|
+
* parse error.
|
|
18
|
+
*/
|
|
19
|
+
const workflowBundleBuildLocks = new Map();
|
|
10
20
|
/**
|
|
11
21
|
* Builds the package-owned workflow bundles used by the Nitro host.
|
|
12
22
|
*/
|
|
@@ -37,6 +47,12 @@ export class WorkflowBundleBuilder extends BaseBuilder {
|
|
|
37
47
|
return false;
|
|
38
48
|
}
|
|
39
49
|
async build(options = {}) {
|
|
50
|
+
const previous = workflowBundleBuildLocks.get(this.#outDir) ?? Promise.resolve();
|
|
51
|
+
const next = previous.then(() => this.#performBuild(options));
|
|
52
|
+
workflowBundleBuildLocks.set(this.#outDir, next.catch(() => { }));
|
|
53
|
+
await next;
|
|
54
|
+
}
|
|
55
|
+
async #performBuild(options) {
|
|
40
56
|
await prepareAshVersionedCacheDirectory(this.#outDir);
|
|
41
57
|
const inputFiles = await this.#getBuildInputFiles();
|
|
42
58
|
if (inputFiles.length === 0) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"builder.js","sourceRoot":"","sources":["../../../../src/internal/workflow-bundle/builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EAAE,WAAW,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAE7F,OAAO,EACL,iCAAiC,EACjC,yBAAyB,GAC1B,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EACL,iCAAiC,EACjC,8BAA8B,GAC/B,MAAM,yCAAyC,CAAC;AACjD,OAAO,EAAE,2BAA2B,EAAE,MAAM,2CAA2C,CAAC;AACxF,OAAO,EAAE,wBAAwB,EAAE,MAAM,+CAA+C,CAAC;AACzF,OAAO,EACL,0BAA0B,EAC1B,iCAAiC,EACjC,6CAA6C,EAC7C,kCAAkC,EAClC,+BAA+B,GAChC,MAAM,qDAAqD,CAAC;AAE7D;;GAEG;AACH,MAAM,OAAO,qBAAsB,SAAQ,WAAW;IAC3C,+BAA+B,CAAS;IACxC,OAAO,CAAS;IAEzB,YAAY,OAAqC;QAC/C,KAAK,CAAC;YACJ,WAAW,EAAE,YAAY;YACzB,IAAI,EAAE,CAAC,iCAAiC,CAAC,eAAe,CAAC,CAAC;YAC1D,gBAAgB,EAAE,CAAC,GAAG,+BAA+B,EAAE,GAAG,kCAAkC,CAAC;YAC7F,+DAA+D;YAC/D,wEAAwE;YACxE,2BAA2B;YAC3B,WAAW,EAAE,OAAO,CAAC,OAAO;YAC5B,0BAA0B,EAAE,IAAI;YAChC,iCAAiC,EAAE,IAAI;YACvC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,UAAU,EAAE,OAAO,CAAC,OAAO;YAC3B,eAAe,EAAE,EAAE;YACnB,mBAAmB,EAAE,EAAE;YACvB,iBAAiB,EAAE,EAAE;SACtB,CAAC,CAAC;QAEH,IAAI,CAAC,+BAA+B,GAAG,OAAO,CAAC,8BAA8B,CAAC;QAC9E,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;IAChC,CAAC;IAED,IAAuB,wBAAwB;QAC7C,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,KAAK,CACT,OAAO,GAAiE,EAAE;QAE1E,MAAM,iCAAiC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEtD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAEpD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CACb,sDAAsD,iCAAiC,CAAC,eAAe,CAAC,IAAI,CAC7G,CAAC;QACJ,CAAC;QAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAEnD,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QAE7F,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;QAC7D,MAAM,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC;YACvE,iBAAiB;YACjB,uEAAuE;YACvE,wEAAwE;YACxE,qDAAqD;YACrD,wBAAwB,EAAE,KAAK;YAC/B,OAAO,EAAE,gBAAgB;YACzB,iBAAiB,EAAE,KAAK;YACxB,MAAM,EAAE,KAAK;YACb,UAAU;YACV,YAAY;SACb,CAAC,CAAC;QACH,MAAM,6BAA6B,CAAC,gBAAgB,CAAC,CAAC;QACtD,MAAM,0BAA0B,CAAC,gBAAgB,CAAC,CAAC;QAEnD,MAAM,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC;QAE1D,IAAI,oBAAoB,KAAK,SAAS,IAAI,oBAAoB,KAAK,gBAAgB,EAAE,CAAC;YACpF,MAAM,KAAK,CAAC,OAAO,CAAC,oBAAoB,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAChE,MAAM,yBAAyB,CAAC,gBAAgB,EAAE,oBAAoB,CAAC,CAAC;QAC1E,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACrD,MAAM,aAAa,GAAG,MAAM,wBAAwB,CAAC;YACnD,YAAY,EAAE,yBAAyB,CAAC,4BAA4B,CAAC;YACrE,iBAAiB;YACjB,OAAO,EAAE,YAAY;YACrB,yBAAyB,EAAE,IAAI;YAC/B,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU;YAC9D,WAAW,EAAE,yBAAyB,CAAC,kBAAkB,CAAC;YAC1D,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;SACnC,CAAC,CAAC;QACH,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;QAElD,IAAI,gBAAgB,KAAK,SAAS,IAAI,gBAAgB,KAAK,YAAY,EAAE,CAAC;YACxE,MAAM,wBAAwB,CAAC;gBAC7B,YAAY,EAAE,yBAAyB,CAAC,4BAA4B,CAAC;gBACrE,iBAAiB;gBACjB,OAAO,EAAE,gBAAgB;gBACzB,yBAAyB,EAAE,IAAI;gBAC/B,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU;gBAC9D,WAAW,EAAE,yBAAyB,CAAC,kBAAkB,CAAC;gBAC1D,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;aACnC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,IAAI,CAAC,cAAc,CAAC;YACxB,kBAAkB,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC;YACvD,WAAW,EAAE,IAAI,CAAC,OAAO;YACzB,QAAQ,EAAE;gBACR,KAAK,EAAE;oBACL,GAAG,aAAa,CAAC,KAAK;oBACtB,GAAG,iBAAiB,CAAC,KAAK;iBAC3B;gBACD,SAAS,EAAE;oBACT,GAAG,aAAa,CAAC,SAAS;oBAC1B,GAAG,iBAAiB,CAAC,SAAS;iBAC/B;gBACD,OAAO,EAAE;oBACP,GAAG,aAAa,CAAC,OAAO;oBACxB,GAAG,iBAAiB,CAAC,OAAO;iBAC7B;aACF;SACF,CAAC,CAAC;QACH,MAAM,8BAA8B,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEnD,sEAAsE;QACtE,yEAAyE;QACzE,2EAA2E;QAC3E,WAAW;IACb,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,iBAAiB,CAAC,OAKvB;QACC,uEAAuE;QACvE,wEAAwE;QACxE,2EAA2E;QAC3E,uDAAuD;QACvD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAEnB,MAAM,0BAA0B,GAAG,IAAI,CACrC,IAAI,CAAC,OAAO,EACZ,qBAAqB,EACrB,WAAW,EACX,aAAa,EACb,UAAU,EACV,IAAI,CACL,CAAC;QACF,MAAM,qBAAqB,GAAG,IAAI,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;QAC5E,MAAM,qBAAqB,GAAG,IAAI,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;QAC5E,MAAM,oBAAoB,GAAG,IAAI,CAC/B,OAAO,CAAC,SAAS,EACjB,WAAW,EACX,aAAa,EACb,UAAU,EACV,IAAI,CACL,CAAC;QACF,MAAM,0BAA0B,GAAG,IAAI,CACrC,OAAO,CAAC,kBAAkB,EAC1B,WAAW,EACX,eAAe,CAChB,CAAC;QACF,MAAM,oBAAoB,GAAG,IAAI,CAC/B,OAAO,CAAC,kBAAkB,EAC1B,WAAW,EACX,aAAa,EACb,UAAU,EACV,IAAI,EACJ,WAAW,CACZ,CAAC;QACF,MAAM,0BAA0B,GAAG,IAAI,CACrC,OAAO,CAAC,kBAAkB,EAC1B,WAAW,EACX,eAAe,CAChB,CAAC;QACF,MAAM,oBAAoB,GAAG,IAAI,CAC/B,OAAO,CAAC,kBAAkB,EAC1B,WAAW,EACX,aAAa,EACb,UAAU,EACV,IAAI,EACJ,WAAW,CACZ,CAAC;QACF,MAAM,eAAe,GAAG,IAAI,CAAC,oBAAoB,EAAE,WAAW,CAAC,CAAC;QAChE,MAAM,eAAe,GAAG,IAAI,CAAC,oBAAoB,EAAE,WAAW,CAAC,CAAC;QAChE,MAAM,uBAAuB,GAAG,IAAI,CAAC,oBAAoB,EAAE,SAAS,EAAE,cAAc,CAAC,CAAC;QAEtF,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,0BAA0B,CAAC;gBACzB,YAAY,EAAE,0BAA0B;gBACxC,UAAU,EAAE,oBAAoB;gBAChC,UAAU,EAAE,qBAAqB;aAClC,CAAC;YACF,0BAA0B,CAAC;gBACzB,YAAY,EAAE,0BAA0B;gBACxC,UAAU,EAAE,oBAAoB;gBAChC,UAAU,EAAE,qBAAqB;aAClC,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,IAAI,CAAC,0BAA0B,CAAC,qBAAqB,EAAE;gBACrD,oBAAoB,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,sBAAsB,CAAC,CAAC;gBAC1D,WAAW,EAAE,KAAK;gBAClB,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI;gBAChC,gBAAgB,EAAE,KAAK;aACxB,CAAC;YACF,IAAI,CAAC,0BAA0B,CAAC,qBAAqB,EAAE;gBACrD,oBAAoB,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,kBAAkB,CAAC,CAAC;gBACtD,WAAW,EAAE,KAAK;gBAClB,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI;gBAChC,gBAAgB,EAAE,KAAK;gBACvB,yBAAyB,EAAE,IAAI;aAChC,CAAC;YACF,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;SAC3F,CAAC,CAAC;QACH,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,6CAA6C,CAAC;gBAC5C,qBAAqB,EAAE,qBAAqB;gBAC5C,iBAAiB,EAAE,+BAA+B;aACnD,CAAC;YACF,6CAA6C,CAAC;gBAC5C,qBAAqB,EAAE,qBAAqB;gBAC5C,iBAAiB,EAAE,+BAA+B;aACnD,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,EAAE,CAAC,eAAe,EAAE;gBAClB,KAAK,EAAE,IAAI;gBACX,SAAS,EAAE,IAAI;aAChB,CAAC;YACF,EAAE,CAAC,eAAe,EAAE;gBAClB,KAAK,EAAE,IAAI;gBACX,SAAS,EAAE,IAAI;aAChB,CAAC;YACF,EAAE,CAAC,uBAAuB,EAAE;gBAC1B,KAAK,EAAE,IAAI;gBACX,SAAS,EAAE,IAAI;aAChB,CAAC;SACH,CAAC,CAAC;QACH,MAAM,KAAK,CAAC,oBAAoB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACvD,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,EAAE,CAAC,qBAAqB,EAAE,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;YAC/D,EAAE,CAAC,qBAAqB,EAAE,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;YAC/D,EAAE,CACA,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,EACjD,IAAI,CAAC,oBAAoB,EAAE,eAAe,CAAC,CAC5C;SACF,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC9C,OAAO,CAAC,GAAG,UAAU,EAAE,IAAI,CAAC,+BAA+B,CAAC,CAAC;IAC/D,CAAC;IAED,KAAK,CAAC,0BAA0B,CAC9B,aAAqB,EACrB,KAMC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;QAC1D,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC;QACpE,MAAM,UAAU,GAA4B;YAC1C,GAAG,UAAU;SACd,CAAC;QACF,UAAU,CAAC,WAAW,GAAG,iCAAiC,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;QAEnF,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC3B,UAAU,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;QACrC,CAAC;QAED,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACpC,UAAU,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC;QAC7C,CAAC;QAED,IAAI,KAAK,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;YACzC,UAAU,CAAC,gBAAgB,GAAG,KAAK,CAAC,gBAAgB,CAAC;QACvD,CAAC;QAED,IAAI,KAAK,CAAC,yBAAyB,KAAK,SAAS,EAAE,CAAC;YAClD,UAAU,CAAC,yBAAyB,GAAG,KAAK,CAAC,yBAAyB,CAAC;QACzE,CAAC;QAED,IAAI,KAAK,CAAC,oBAAoB,KAAK,SAAS,EAAE,CAAC;YAC7C,UAAU,CAAC,oBAAoB,GAAG,CAAC,GAAG,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,SAAS,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IAC1E,CAAC;IAED,KAAK,CAAC,yBAAyB,CAAC,UAAkB;QAChD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;YAC9D,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAClD,OAAO,MAAiC,CAAC;YAC3C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;CACF;AAED,KAAK,UAAU,6BAA6B,CAAC,QAAgB;IAC3D,MAAM,MAAM,GAAG,MAAM,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAErD,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QACpB,OAAO;IACT,CAAC;IAED,IAAI,UAAU,GAAG,MAAM,CAAC;IAExB,KAAK,MAAM,SAAS,IAAI;QACtB,UAAU;QACV,cAAc;QACd,4BAA4B;QAC5B,2BAA2B;QAC3B,kBAAkB;KACnB,EAAE,CAAC;QACF,MAAM,iBAAiB,GAAG,4BAA4B,CAAC,yBAAyB,CAAC,SAAS,CAAC,CAAC,CAAC;QAC7F,UAAU,GAAG,6BAA6B,CAAC,UAAU,EAAE,SAAS,EAAE,iBAAiB,CAAC,CAAC;IACvF,CAAC;IAED,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;QAC1B,MAAM,SAAS,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACxC,CAAC;AACH,CAAC;AAED,KAAK,UAAU,0BAA0B,CAAC,QAAgB;IACxD,MAAM,MAAM,GAAG,MAAM,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAErD,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QACpB,OAAO;IACT,CAAC;IAED,MAAM,iBAAiB,GAAG,uBAAuB,CAAC;IAClD,MAAM,iBAAiB,GAAG,4DAA4D,CAAC;IACvF,MAAM,eAAe,GAAG,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAC1D,MAAM,aAAa,GAAG,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC;IAE5D,IAAI,eAAe,KAAK,CAAC,CAAC,IAAI,aAAa,KAAK,CAAC,CAAC,IAAI,aAAa,IAAI,eAAe,EAAE,CAAC;QACvF,OAAO;IACT,CAAC;IAED,MAAM,UAAU,GAAG,eAAe,GAAG,iBAAiB,CAAC,MAAM,CAAC;IAC9D,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;IAE3D,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5C,OAAO;IACT,CAAC;IAED,MAAM,YAAY,GAAG,iCAAiC,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAC7E,MAAM,UAAU,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,GAAG,yBAAyB,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC,KAAK,CACxG,aAAa,CACd,EAAE,CAAC;IAEJ,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;QAC1B,MAAM,SAAS,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACxC,CAAC;AACH,CAAC;AAED,SAAS,yBAAyB,CAAC,YAAoB;IACrD,MAAM,mBAAmB,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACjF,MAAM,MAAM,GAAG,mBAAmB,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEhE,OAAO,eAAe,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,uCAAuC,CAAC;AACtF,CAAC;AAED,SAAS,iCAAiC,CAAC,UAAkB,EAAE,QAAgB;IAC7E,MAAM,iBAAiB,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC;IAE5C,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CAAC,gDAAgD,QAAQ,qBAAqB,CAAC,CAAC;IACjG,CAAC;IAED,MAAM,QAAQ,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAChD,IAAI,KAAK,GAAG,EAAE,CAAC;IAEf,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACxD,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QAE7B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,KAAK,IAAI,IAAI,CAAC;YACd,SAAS;QACX,CAAC;QAED,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAExC,IAAI,WAAW,KAAK,IAAI,IAAI,WAAW,KAAK,GAAG,IAAI,WAAW,KAAK,GAAG,EAAE,CAAC;YACvE,KAAK,IAAI,WAAW,CAAC;YACrB,KAAK,IAAI,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,KAAK,IAAI,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,6BAA6B,CAAC,MAAc,EAAE,IAAY,EAAE,EAAU;IAC7E,OAAO,MAAM;SACV,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;SACpD,UAAU,CAAC,IAAI,IAAI,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,4BAA4B,CAAC,IAAY;IAChD,OAAO,2BAA2B,CAAC,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,QAAgB;IACnD,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzE,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,uBAAuB,CAAC,QAAgB;IACrD,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzE,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;;;GASG;AACH,KAAK,UAAU,yBAAyB,CAAC,UAAkB,EAAE,UAAkB;IAC7E,MAAM,cAAc,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,CAAC;IAClD,MAAM,gBAAgB,GAAG,MAAM,uBAAuB,CAAC,UAAU,CAAC,CAAC;IAEnE,IAAI,gBAAgB,KAAK,IAAI,IAAI,gBAAgB,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC;QACzE,OAAO;IACT,CAAC;IAED,MAAM,SAAS,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;AAC9C,CAAC"}
|
|
1
|
+
{"version":3,"file":"builder.js","sourceRoot":"","sources":["../../../../src/internal/workflow-bundle/builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EAAE,WAAW,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAE7F,OAAO,EACL,iCAAiC,EACjC,yBAAyB,GAC1B,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EACL,iCAAiC,EACjC,8BAA8B,GAC/B,MAAM,yCAAyC,CAAC;AACjD,OAAO,EAAE,2BAA2B,EAAE,MAAM,2CAA2C,CAAC;AACxF,OAAO,EAAE,wBAAwB,EAAE,MAAM,+CAA+C,CAAC;AACzF,OAAO,EACL,0BAA0B,EAC1B,iCAAiC,EACjC,6CAA6C,EAC7C,kCAAkC,EAClC,+BAA+B,GAChC,MAAM,qDAAqD,CAAC;AAE7D;;;;;;;;GAQG;AACH,MAAM,wBAAwB,GAAG,IAAI,GAAG,EAAyB,CAAC;AAElE;;GAEG;AACH,MAAM,OAAO,qBAAsB,SAAQ,WAAW;IAC3C,+BAA+B,CAAS;IACxC,OAAO,CAAS;IAEzB,YAAY,OAAqC;QAC/C,KAAK,CAAC;YACJ,WAAW,EAAE,YAAY;YACzB,IAAI,EAAE,CAAC,iCAAiC,CAAC,eAAe,CAAC,CAAC;YAC1D,gBAAgB,EAAE,CAAC,GAAG,+BAA+B,EAAE,GAAG,kCAAkC,CAAC;YAC7F,+DAA+D;YAC/D,wEAAwE;YACxE,2BAA2B;YAC3B,WAAW,EAAE,OAAO,CAAC,OAAO;YAC5B,0BAA0B,EAAE,IAAI;YAChC,iCAAiC,EAAE,IAAI;YACvC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,UAAU,EAAE,OAAO,CAAC,OAAO;YAC3B,eAAe,EAAE,EAAE;YACnB,mBAAmB,EAAE,EAAE;YACvB,iBAAiB,EAAE,EAAE;SACtB,CAAC,CAAC;QAEH,IAAI,CAAC,+BAA+B,GAAG,OAAO,CAAC,8BAA8B,CAAC;QAC9E,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;IAChC,CAAC;IAED,IAAuB,wBAAwB;QAC7C,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,KAAK,CACT,OAAO,GAAiE,EAAE;QAE1E,MAAM,QAAQ,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACjF,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC;QAC9D,wBAAwB,CAAC,GAAG,CAC1B,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CACrB,CAAC;QACF,MAAM,IAAI,CAAC;IACb,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,OAGnB;QACC,MAAM,iCAAiC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEtD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAEpD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CACb,sDAAsD,iCAAiC,CAAC,eAAe,CAAC,IAAI,CAC7G,CAAC;QACJ,CAAC;QAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAEnD,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QAE7F,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;QAC7D,MAAM,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC;YACvE,iBAAiB;YACjB,uEAAuE;YACvE,wEAAwE;YACxE,qDAAqD;YACrD,wBAAwB,EAAE,KAAK;YAC/B,OAAO,EAAE,gBAAgB;YACzB,iBAAiB,EAAE,KAAK;YACxB,MAAM,EAAE,KAAK;YACb,UAAU;YACV,YAAY;SACb,CAAC,CAAC;QACH,MAAM,6BAA6B,CAAC,gBAAgB,CAAC,CAAC;QACtD,MAAM,0BAA0B,CAAC,gBAAgB,CAAC,CAAC;QAEnD,MAAM,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC;QAE1D,IAAI,oBAAoB,KAAK,SAAS,IAAI,oBAAoB,KAAK,gBAAgB,EAAE,CAAC;YACpF,MAAM,KAAK,CAAC,OAAO,CAAC,oBAAoB,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAChE,MAAM,yBAAyB,CAAC,gBAAgB,EAAE,oBAAoB,CAAC,CAAC;QAC1E,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACrD,MAAM,aAAa,GAAG,MAAM,wBAAwB,CAAC;YACnD,YAAY,EAAE,yBAAyB,CAAC,4BAA4B,CAAC;YACrE,iBAAiB;YACjB,OAAO,EAAE,YAAY;YACrB,yBAAyB,EAAE,IAAI;YAC/B,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU;YAC9D,WAAW,EAAE,yBAAyB,CAAC,kBAAkB,CAAC;YAC1D,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;SACnC,CAAC,CAAC;QACH,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;QAElD,IAAI,gBAAgB,KAAK,SAAS,IAAI,gBAAgB,KAAK,YAAY,EAAE,CAAC;YACxE,MAAM,wBAAwB,CAAC;gBAC7B,YAAY,EAAE,yBAAyB,CAAC,4BAA4B,CAAC;gBACrE,iBAAiB;gBACjB,OAAO,EAAE,gBAAgB;gBACzB,yBAAyB,EAAE,IAAI;gBAC/B,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU;gBAC9D,WAAW,EAAE,yBAAyB,CAAC,kBAAkB,CAAC;gBAC1D,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;aACnC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,IAAI,CAAC,cAAc,CAAC;YACxB,kBAAkB,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC;YACvD,WAAW,EAAE,IAAI,CAAC,OAAO;YACzB,QAAQ,EAAE;gBACR,KAAK,EAAE;oBACL,GAAG,aAAa,CAAC,KAAK;oBACtB,GAAG,iBAAiB,CAAC,KAAK;iBAC3B;gBACD,SAAS,EAAE;oBACT,GAAG,aAAa,CAAC,SAAS;oBAC1B,GAAG,iBAAiB,CAAC,SAAS;iBAC/B;gBACD,OAAO,EAAE;oBACP,GAAG,aAAa,CAAC,OAAO;oBACxB,GAAG,iBAAiB,CAAC,OAAO;iBAC7B;aACF;SACF,CAAC,CAAC;QACH,MAAM,8BAA8B,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEnD,sEAAsE;QACtE,yEAAyE;QACzE,2EAA2E;QAC3E,WAAW;IACb,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,iBAAiB,CAAC,OAKvB;QACC,uEAAuE;QACvE,wEAAwE;QACxE,2EAA2E;QAC3E,uDAAuD;QACvD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAEnB,MAAM,0BAA0B,GAAG,IAAI,CACrC,IAAI,CAAC,OAAO,EACZ,qBAAqB,EACrB,WAAW,EACX,aAAa,EACb,UAAU,EACV,IAAI,CACL,CAAC;QACF,MAAM,qBAAqB,GAAG,IAAI,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;QAC5E,MAAM,qBAAqB,GAAG,IAAI,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;QAC5E,MAAM,oBAAoB,GAAG,IAAI,CAC/B,OAAO,CAAC,SAAS,EACjB,WAAW,EACX,aAAa,EACb,UAAU,EACV,IAAI,CACL,CAAC;QACF,MAAM,0BAA0B,GAAG,IAAI,CACrC,OAAO,CAAC,kBAAkB,EAC1B,WAAW,EACX,eAAe,CAChB,CAAC;QACF,MAAM,oBAAoB,GAAG,IAAI,CAC/B,OAAO,CAAC,kBAAkB,EAC1B,WAAW,EACX,aAAa,EACb,UAAU,EACV,IAAI,EACJ,WAAW,CACZ,CAAC;QACF,MAAM,0BAA0B,GAAG,IAAI,CACrC,OAAO,CAAC,kBAAkB,EAC1B,WAAW,EACX,eAAe,CAChB,CAAC;QACF,MAAM,oBAAoB,GAAG,IAAI,CAC/B,OAAO,CAAC,kBAAkB,EAC1B,WAAW,EACX,aAAa,EACb,UAAU,EACV,IAAI,EACJ,WAAW,CACZ,CAAC;QACF,MAAM,eAAe,GAAG,IAAI,CAAC,oBAAoB,EAAE,WAAW,CAAC,CAAC;QAChE,MAAM,eAAe,GAAG,IAAI,CAAC,oBAAoB,EAAE,WAAW,CAAC,CAAC;QAChE,MAAM,uBAAuB,GAAG,IAAI,CAAC,oBAAoB,EAAE,SAAS,EAAE,cAAc,CAAC,CAAC;QAEtF,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,0BAA0B,CAAC;gBACzB,YAAY,EAAE,0BAA0B;gBACxC,UAAU,EAAE,oBAAoB;gBAChC,UAAU,EAAE,qBAAqB;aAClC,CAAC;YACF,0BAA0B,CAAC;gBACzB,YAAY,EAAE,0BAA0B;gBACxC,UAAU,EAAE,oBAAoB;gBAChC,UAAU,EAAE,qBAAqB;aAClC,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,IAAI,CAAC,0BAA0B,CAAC,qBAAqB,EAAE;gBACrD,oBAAoB,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,sBAAsB,CAAC,CAAC;gBAC1D,WAAW,EAAE,KAAK;gBAClB,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI;gBAChC,gBAAgB,EAAE,KAAK;aACxB,CAAC;YACF,IAAI,CAAC,0BAA0B,CAAC,qBAAqB,EAAE;gBACrD,oBAAoB,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,kBAAkB,CAAC,CAAC;gBACtD,WAAW,EAAE,KAAK;gBAClB,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI;gBAChC,gBAAgB,EAAE,KAAK;gBACvB,yBAAyB,EAAE,IAAI;aAChC,CAAC;YACF,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;SAC3F,CAAC,CAAC;QACH,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,6CAA6C,CAAC;gBAC5C,qBAAqB,EAAE,qBAAqB;gBAC5C,iBAAiB,EAAE,+BAA+B;aACnD,CAAC;YACF,6CAA6C,CAAC;gBAC5C,qBAAqB,EAAE,qBAAqB;gBAC5C,iBAAiB,EAAE,+BAA+B;aACnD,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,EAAE,CAAC,eAAe,EAAE;gBAClB,KAAK,EAAE,IAAI;gBACX,SAAS,EAAE,IAAI;aAChB,CAAC;YACF,EAAE,CAAC,eAAe,EAAE;gBAClB,KAAK,EAAE,IAAI;gBACX,SAAS,EAAE,IAAI;aAChB,CAAC;YACF,EAAE,CAAC,uBAAuB,EAAE;gBAC1B,KAAK,EAAE,IAAI;gBACX,SAAS,EAAE,IAAI;aAChB,CAAC;SACH,CAAC,CAAC;QACH,MAAM,KAAK,CAAC,oBAAoB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACvD,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,EAAE,CAAC,qBAAqB,EAAE,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;YAC/D,EAAE,CAAC,qBAAqB,EAAE,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;YAC/D,EAAE,CACA,IAAI,CAAC,0BAA0B,EAAE,eAAe,CAAC,EACjD,IAAI,CAAC,oBAAoB,EAAE,eAAe,CAAC,CAC5C;SACF,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC9C,OAAO,CAAC,GAAG,UAAU,EAAE,IAAI,CAAC,+BAA+B,CAAC,CAAC;IAC/D,CAAC;IAED,KAAK,CAAC,0BAA0B,CAC9B,aAAqB,EACrB,KAMC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;QAC1D,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC;QACpE,MAAM,UAAU,GAA4B;YAC1C,GAAG,UAAU;SACd,CAAC;QACF,UAAU,CAAC,WAAW,GAAG,iCAAiC,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;QAEnF,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC3B,UAAU,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;QACrC,CAAC;QAED,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACpC,UAAU,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC;QAC7C,CAAC;QAED,IAAI,KAAK,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;YACzC,UAAU,CAAC,gBAAgB,GAAG,KAAK,CAAC,gBAAgB,CAAC;QACvD,CAAC;QAED,IAAI,KAAK,CAAC,yBAAyB,KAAK,SAAS,EAAE,CAAC;YAClD,UAAU,CAAC,yBAAyB,GAAG,KAAK,CAAC,yBAAyB,CAAC;QACzE,CAAC;QAED,IAAI,KAAK,CAAC,oBAAoB,KAAK,SAAS,EAAE,CAAC;YAC7C,UAAU,CAAC,oBAAoB,GAAG,CAAC,GAAG,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,SAAS,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IAC1E,CAAC;IAED,KAAK,CAAC,yBAAyB,CAAC,UAAkB;QAChD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;YAC9D,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAClD,OAAO,MAAiC,CAAC;YAC3C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;CACF;AAED,KAAK,UAAU,6BAA6B,CAAC,QAAgB;IAC3D,MAAM,MAAM,GAAG,MAAM,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAErD,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QACpB,OAAO;IACT,CAAC;IAED,IAAI,UAAU,GAAG,MAAM,CAAC;IAExB,KAAK,MAAM,SAAS,IAAI;QACtB,UAAU;QACV,cAAc;QACd,4BAA4B;QAC5B,2BAA2B;QAC3B,kBAAkB;KACnB,EAAE,CAAC;QACF,MAAM,iBAAiB,GAAG,4BAA4B,CAAC,yBAAyB,CAAC,SAAS,CAAC,CAAC,CAAC;QAC7F,UAAU,GAAG,6BAA6B,CAAC,UAAU,EAAE,SAAS,EAAE,iBAAiB,CAAC,CAAC;IACvF,CAAC;IAED,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;QAC1B,MAAM,SAAS,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACxC,CAAC;AACH,CAAC;AAED,KAAK,UAAU,0BAA0B,CAAC,QAAgB;IACxD,MAAM,MAAM,GAAG,MAAM,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAErD,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QACpB,OAAO;IACT,CAAC;IAED,MAAM,iBAAiB,GAAG,uBAAuB,CAAC;IAClD,MAAM,iBAAiB,GAAG,4DAA4D,CAAC;IACvF,MAAM,eAAe,GAAG,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAC1D,MAAM,aAAa,GAAG,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC;IAE5D,IAAI,eAAe,KAAK,CAAC,CAAC,IAAI,aAAa,KAAK,CAAC,CAAC,IAAI,aAAa,IAAI,eAAe,EAAE,CAAC;QACvF,OAAO;IACT,CAAC;IAED,MAAM,UAAU,GAAG,eAAe,GAAG,iBAAiB,CAAC,MAAM,CAAC;IAC9D,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;IAE3D,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5C,OAAO;IACT,CAAC;IAED,MAAM,YAAY,GAAG,iCAAiC,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAC7E,MAAM,UAAU,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,GAAG,yBAAyB,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC,KAAK,CACxG,aAAa,CACd,EAAE,CAAC;IAEJ,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;QAC1B,MAAM,SAAS,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACxC,CAAC;AACH,CAAC;AAED,SAAS,yBAAyB,CAAC,YAAoB;IACrD,MAAM,mBAAmB,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACjF,MAAM,MAAM,GAAG,mBAAmB,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEhE,OAAO,eAAe,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,uCAAuC,CAAC;AACtF,CAAC;AAED,SAAS,iCAAiC,CAAC,UAAkB,EAAE,QAAgB;IAC7E,MAAM,iBAAiB,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC;IAE5C,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CAAC,gDAAgD,QAAQ,qBAAqB,CAAC,CAAC;IACjG,CAAC;IAED,MAAM,QAAQ,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAChD,IAAI,KAAK,GAAG,EAAE,CAAC;IAEf,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACxD,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QAE7B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,KAAK,IAAI,IAAI,CAAC;YACd,SAAS;QACX,CAAC;QAED,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAExC,IAAI,WAAW,KAAK,IAAI,IAAI,WAAW,KAAK,GAAG,IAAI,WAAW,KAAK,GAAG,EAAE,CAAC;YACvE,KAAK,IAAI,WAAW,CAAC;YACrB,KAAK,IAAI,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,KAAK,IAAI,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,6BAA6B,CAAC,MAAc,EAAE,IAAY,EAAE,EAAU;IAC7E,OAAO,MAAM;SACV,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;SACpD,UAAU,CAAC,IAAI,IAAI,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,4BAA4B,CAAC,IAAY;IAChD,OAAO,2BAA2B,CAAC,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,QAAgB;IACnD,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzE,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,uBAAuB,CAAC,QAAgB;IACrD,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzE,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;;;GASG;AACH,KAAK,UAAU,yBAAyB,CAAC,UAAkB,EAAE,UAAkB;IAC7E,MAAM,cAAc,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,CAAC;IAClD,MAAM,gBAAgB,GAAG,MAAM,uBAAuB,CAAC,UAAU,CAAC,CAAC;IAEnE,IAAI,gBAAgB,KAAK,IAAI,IAAI,gBAAgB,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC;QACzE,OAAO;IACT,CAAC;IAED,MAAM,SAAS,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;AAC9C,CAAC"}
|
package/docs/public/evals.md
CHANGED
|
@@ -40,6 +40,7 @@ import { Run } from "experimental-ash/evals/scores";
|
|
|
40
40
|
|
|
41
41
|
export default defineEvalSuite({
|
|
42
42
|
id: "weather-smoke",
|
|
43
|
+
model: "openai/gpt-5.4-mini",
|
|
43
44
|
name: "Weather Smoke",
|
|
44
45
|
description: "Basic message and tool-usage coverage for the weather agent.",
|
|
45
46
|
cases: [
|
|
@@ -59,6 +60,7 @@ can discover and run.
|
|
|
59
60
|
Every suite must provide:
|
|
60
61
|
|
|
61
62
|
- `id`
|
|
63
|
+
- `model`
|
|
62
64
|
- `name`
|
|
63
65
|
- `scores`
|
|
64
66
|
- either `cases` or `load`
|
|
@@ -68,6 +70,7 @@ Common optional suite fields:
|
|
|
68
70
|
- `description`
|
|
69
71
|
- `task`
|
|
70
72
|
- `thresholds`
|
|
73
|
+
- `modelOptions`
|
|
71
74
|
- `tags`
|
|
72
75
|
- `metadata`
|
|
73
76
|
- `maxConcurrency`
|
|
@@ -86,6 +89,7 @@ You can provide cases directly:
|
|
|
86
89
|
```ts
|
|
87
90
|
export default defineEvalSuite({
|
|
88
91
|
id: "smoke",
|
|
92
|
+
model: "openai/gpt-5.4-mini",
|
|
89
93
|
name: "Smoke",
|
|
90
94
|
cases: [
|
|
91
95
|
{
|
|
@@ -107,6 +111,7 @@ import { Text } from "experimental-ash/evals/scores";
|
|
|
107
111
|
|
|
108
112
|
export default defineEvalSuite({
|
|
109
113
|
id: "marketing-sql",
|
|
114
|
+
model: "openai/gpt-5.4-mini",
|
|
110
115
|
name: "Marketing SQL",
|
|
111
116
|
async load() {
|
|
112
117
|
const doc = await loadYaml("evals/data/cases.yaml");
|
|
@@ -127,6 +132,7 @@ message.
|
|
|
127
132
|
```ts
|
|
128
133
|
defineEvalSuite({
|
|
129
134
|
id: "weather-task",
|
|
135
|
+
model: "openai/gpt-5.4-mini",
|
|
130
136
|
name: "Weather Task",
|
|
131
137
|
task: {
|
|
132
138
|
prompt: (testCase) => `Answer the user: ${String(testCase.input)}`,
|
|
@@ -144,6 +150,7 @@ Suite thresholds let you relax a scorer from the default exact-match requirement
|
|
|
144
150
|
```ts
|
|
145
151
|
defineEvalSuite({
|
|
146
152
|
id: "weather",
|
|
153
|
+
model: "openai/gpt-5.4-mini",
|
|
147
154
|
name: "Weather",
|
|
148
155
|
cases: [{ id: "hello", input: "Hello", expected: "Hello" }],
|
|
149
156
|
scores: [Run.didNotFail(), Text.includes()],
|
|
@@ -157,6 +164,34 @@ defineEvalSuite({
|
|
|
157
164
|
Use `maxConcurrency` to limit how many cases the runner executes in parallel and `timeoutMs` to
|
|
158
165
|
bound each case. When omitted, `ash eval` runs up to 8 cases in parallel per suite.
|
|
159
166
|
|
|
167
|
+
## Model-Backed Scorers
|
|
168
|
+
|
|
169
|
+
Some scorers call a judge or embedding model. For those scorers, Ash uses the same model contract
|
|
170
|
+
as `agent.ts`:
|
|
171
|
+
|
|
172
|
+
- a string model id, such as `"openai/gpt-5.4-mini"`, routes through the Vercel AI Gateway
|
|
173
|
+
- an AI SDK model instance is used directly
|
|
174
|
+
|
|
175
|
+
Set a suite-level scorer model with `model`. Every eval suite requires one. This model is only
|
|
176
|
+
used for scoring; it does not change the Ash agent being evaluated.
|
|
177
|
+
|
|
178
|
+
```ts
|
|
179
|
+
import { defineEvalSuite } from "experimental-ash/evals";
|
|
180
|
+
import { Autoevals, Run } from "experimental-ash/evals/scores";
|
|
181
|
+
|
|
182
|
+
export default defineEvalSuite({
|
|
183
|
+
id: "support-quality",
|
|
184
|
+
name: "Support Quality",
|
|
185
|
+
model: "openai/gpt-5.4-mini",
|
|
186
|
+
cases: [{ id: "refund", input: "How do I get a refund?", expected: "refund policy" }],
|
|
187
|
+
scores: [Run.didNotFail(), Autoevals.factuality()],
|
|
188
|
+
});
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
Use `modelOptions.providerOptions` when the scorer model needs provider-specific AI SDK options.
|
|
192
|
+
Individual Autoevals scorers also accept `model` and `modelOptions`; scorer-level values override
|
|
193
|
+
the suite default.
|
|
194
|
+
|
|
160
195
|
## Loaders
|
|
161
196
|
|
|
162
197
|
Ash ships small loader helpers for eval data files:
|
|
@@ -170,20 +205,42 @@ These loaders are meant for eval fixtures and other authoring-time data, not for
|
|
|
170
205
|
|
|
171
206
|
## Scores
|
|
172
207
|
|
|
173
|
-
Ash exports scorer namespaces from `experimental-ash/evals/scores
|
|
208
|
+
Ash exports scorer namespaces from `experimental-ash/evals/scores`. Scorers receive flattened
|
|
209
|
+
`input`, `output`, and `expected` strings plus the full case and task result. They may return
|
|
210
|
+
`null` to skip scoring a case.
|
|
211
|
+
|
|
212
|
+
### Picking a Scorer
|
|
213
|
+
|
|
214
|
+
Pick the cheapest scorer that can express what "correct" means. Free scorers run instantly with no
|
|
215
|
+
API cost; an LLM judge runs once per case and costs tokens, so reach for it last.
|
|
216
|
+
|
|
217
|
+
| Need | Use |
|
|
218
|
+
| ------------------------------------------------------------ | ---------------------------------------------------------- |
|
|
219
|
+
| Assert agent behavior (run succeeded, used the right tools) | `Run.didNotFail()`, `Run.usedTool()`, `Run.maxToolCalls()` |
|
|
220
|
+
| Exact string match | `Text.exact()` or `Text.includes()` |
|
|
221
|
+
| Exact JSON match | `Json.deepEqual()` |
|
|
222
|
+
| Exact SQL match (after normalization) | `Sql.exactNormalized()` |
|
|
223
|
+
| Fuzzy text match (typos, whitespace differences) | `Text.levenshtein()` |
|
|
224
|
+
| LLM-judged factual correctness vs an expected answer | `Autoevals.factuality()` |
|
|
225
|
+
| LLM-judged summary quality | `Autoevals.summary()` |
|
|
226
|
+
| LLM-judged SQL semantic equivalence | `Autoevals.sql()` |
|
|
227
|
+
| LLM-judged free-form criteria (no `expected` value to match) | `Autoevals.closedQA({ criteria: "..." })` |
|
|
228
|
+
|
|
229
|
+
### Reference
|
|
174
230
|
|
|
175
|
-
- `Text.exact()` and `Text.includes()`
|
|
176
|
-
- `Json.deepEqual()`
|
|
177
|
-
- `Sql.exactNormalized()`
|
|
178
231
|
- `Run.didNotFail()`
|
|
179
232
|
- `Run.usedTool(name)`
|
|
180
233
|
- `Run.usedNoTools()`
|
|
181
234
|
- `Run.maxToolCalls(max)`
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
235
|
+
- `Text.exact()`
|
|
236
|
+
- `Text.includes()`
|
|
237
|
+
- `Text.levenshtein()`
|
|
238
|
+
- `Json.deepEqual()`
|
|
239
|
+
- `Sql.exactNormalized()`
|
|
240
|
+
- `Autoevals.factuality()`
|
|
241
|
+
- `Autoevals.summary()`
|
|
242
|
+
- `Autoevals.sql()`
|
|
243
|
+
- `Autoevals.closedQA({ criteria })`
|
|
187
244
|
|
|
188
245
|
## Reporters
|
|
189
246
|
|
|
@@ -173,7 +173,7 @@ import type { RunMode } from "experimental-ash";
|
|
|
173
173
|
|
|
174
174
|
```ts
|
|
175
175
|
import { defineEvalSuite } from "experimental-ash/evals";
|
|
176
|
-
import { Text, Run } from "experimental-ash/evals/scores";
|
|
176
|
+
import { Autoevals, Text, Run } from "experimental-ash/evals/scores";
|
|
177
177
|
import { loadYaml } from "experimental-ash/evals/loaders";
|
|
178
178
|
import { Braintrust } from "experimental-ash/evals/reporters";
|
|
179
179
|
```
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "experimental-ash",
|
|
3
|
-
"version": "0.3.0-alpha.
|
|
3
|
+
"version": "0.3.0-alpha.36",
|
|
4
4
|
"bin": {
|
|
5
5
|
"ash": "./bin/ash.js",
|
|
6
6
|
"experimental-ash": "./bin/ash.js"
|
|
@@ -158,6 +158,7 @@
|
|
|
158
158
|
"@workflow/core": "^4.2.4",
|
|
159
159
|
"@workflow/errors": "^4.1.1",
|
|
160
160
|
"ai": "7.0.0-canary.121",
|
|
161
|
+
"autoevals": "^0.0.132",
|
|
161
162
|
"chat": "^4.27.0",
|
|
162
163
|
"chokidar": "^5.0.0",
|
|
163
164
|
"commander": "^14.0.3",
|