@leostera/evalkit 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -1
- package/dist/cli.mjs +5186 -10301
- package/dist/dashboard/assets/{index-CTJpd8q8.js → index-BEBo_81Y.js} +4 -4
- package/dist/dashboard/assets/index-C24IquYZ.css +2 -0
- package/dist/dashboard/index.html +2 -2
- package/dist/index.d.mts +111 -6
- package/dist/index.d.mts.map +1 -1
- package/dist/index.mjs +2 -8
- package/dist/runner.d.mts +85 -3
- package/dist/runner.d.mts.map +1 -1
- package/dist/runner.mjs +2 -2
- package/dist/skills/evalkit-evals/SKILL.md +1 -1
- package/dist/skills/evalkit-judges/SKILL.md +3 -2
- package/dist/skills/evalkit-results/SKILL.md +1 -1
- package/dist/{src-CKqOzeBi.mjs → src-DhromKNu.mjs} +175 -29
- package/dist/{src-CZPufWN_.mjs → src-b2Wnmzbb.mjs} +53 -3
- package/package.json +1 -1
- package/dist/dashboard/assets/index-YAabTN3X.css +0 -2
package/dist/index.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.mts","names":[],"sources":["../../core/src/identity.ts","../../core/src/dataset.ts","../../core/src/aut-events.ts","../../core/src/catalog.ts","../../core/src/canonical-json.ts","../../core/src/matrix.ts","../../core/src/config.ts","../../core/src/json.ts","../../core/src/definition-site.ts","../../core/src/schema.ts","../../core/src/trajectory.ts","../../core/src/index.ts","../../agents/src/index.ts"],"mappings":";;qBAEa,oBAAkB,OAAA;YACnB,sBAAsB,mBAAmB;;wBAGrC,SAAS;;qBAOZ,YAAU,OAAA,cAAA,OAAA;YAKX,cAAc,WAAW;YAEzB,YAAY,cAAc,eAAe,2BACxC;qBAEA,mBAAiB,OAAA,cAAA,OAAA;wBAMd,kBAAkB,cAAc,cAAc,MAAM,QAAK,OAAA,cAAA,OAAA;wBAWzD,YAAY,cAAc,cACxC,MAAM,OACN,MAAM,OACL,YAAY;wBAIC,iBAAiB,cAAc,cAC7C,eACA,eAAe;EACZ,MAAM;EAAO,MAAM;EAAM,KAAK,YAAY;;wBAY/B,aAAa,eAAe,eAAe,eAAe;;;;YCvD9D;;YAGA,6BAA6B;EAAe;;;;;;YAM5C,wBAAwB,SAAS,eAAe;;YAGhD;EACV,QAAQ;;EAER;;YAGU;;EAEV;EACA,QAAQ;;YAGE;EACV;EACA;EACA;EACA;;EAEA;;EAEA,YAAY;EACZ,QAAQ;;KAGL;EACH;EACA;;EAEA,aAAa;;EAEb;;YAGU,0BAA0B;EACpC,QAAQ;;EAER;;EAEA;EACA;EACA;;YAGU,0BAA0B,gBAAgB,kBACpD;;EAEE,SAAS,MAAM,uBAAuB,UAAU,QAAQ;;EAExD,SACE,QAAQ,SACR,OAAO,wBACJ,iBAAiB,QAAQ;EAC9B;EACA;EACA;;YAGQ;EAEN;EACA,QAAQ;EACR;EACA;;EAGA;EACA,SACE,MAAM,uBACH,iBAAiB,QAAQ;EAC9B,SACE,QAAQ,gBACR,OAAO,wBACJ,iBAAiB,QAAQ;;;YAIxB;WACD;WACA;WACA;WACA,YAAY;;WAEZ;WACA,QAAQ;;;wBAIH,oBAAoB,eAAe;;;;;;;wBAuBnC,cAAc,gBAAgB,gBAC5C,YAAY,0BAA0B,0BAA0B,WAC/D;wBAmFa,UAAU,iBAAiB,SAAS;;;;;;;YC/MxC;EACN;EAAiB;;EAEjB;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB;EAAkB;;EACrC;EAAe,OAAO;EAAe;;YAE/B;EACV;EACA;;EAEA;EACA;;;YAIU;EAGN;EACA,qBAAqB;EACrB,QAAQ;;;wBAIE,kBACd,OAAO,eACP,WAAW,OACV;;;;;wBA0Da,iBACd,QAAQ,YACR,cACA,WAAW,OACV;;;;;;;;;YCtGS;;EAEV;;YAGU;EACV;EACA;EACA;EACA;EACA,UAAU;IAAQ,MAAM;IAAkB;;;EAE1C;;;YAIU;EACN;EAAe;;EACf;EAAgB;;EAChB;;YAEM;EACV;EACA;EACA,YAAY;EACZ;EACA;;EAEA;IAAU;;IAAwB;;;EAElC;IAAU,OAAO;;IAAiB;;;EAElC;IAAU;IAAc;;EACxB;;YAGU;EACV;EACA,MAAM;EACN;EACA;;EAEA;EACA;;YAGU;
|
|
1
|
+
{"version":3,"file":"index.d.mts","names":[],"sources":["../../core/src/identity.ts","../../core/src/dataset.ts","../../core/src/aut-events.ts","../../core/src/catalog.ts","../../core/src/canonical-json.ts","../../core/src/matrix.ts","../../core/src/config.ts","../../core/src/json.ts","../../core/src/definition-site.ts","../../core/src/schema.ts","../../core/src/trajectory.ts","../../core/src/index.ts","../../agents/src/index.ts"],"mappings":";;qBAEa,oBAAkB,OAAA;YACnB,sBAAsB,mBAAmB;;wBAGrC,SAAS;;qBAOZ,YAAU,OAAA,cAAA,OAAA;YAKX,cAAc,WAAW;YAEzB,YAAY,cAAc,eAAe,2BACxC;qBAEA,mBAAiB,OAAA,cAAA,OAAA;wBAMd,kBAAkB,cAAc,cAAc,MAAM,QAAK,OAAA,cAAA,OAAA;wBAWzD,YAAY,cAAc,cACxC,MAAM,OACN,MAAM,OACL,YAAY;wBAIC,iBAAiB,cAAc,cAC7C,eACA,eAAe;EACZ,MAAM;EAAO,MAAM;EAAM,KAAK,YAAY;;wBAY/B,aAAa,eAAe,eAAe,eAAe;;;;YCvD9D;;YAGA,6BAA6B;EAAe;;;;;;YAM5C,wBAAwB,SAAS,eAAe;;YAGhD;EACV,QAAQ;;EAER;;YAGU;;EAEV;EACA,QAAQ;;YAGE;EACV;EACA;EACA;EACA;;EAEA;;EAEA,YAAY;EACZ,QAAQ;;KAGL;EACH;EACA;;EAEA,aAAa;;EAEb;;YAGU,0BAA0B;EACpC,QAAQ;;EAER;;EAEA;EACA;EACA;;YAGU,0BAA0B,gBAAgB,kBACpD;;EAEE,SAAS,MAAM,uBAAuB,UAAU,QAAQ;;EAExD,SACE,QAAQ,SACR,OAAO,wBACJ,iBAAiB,QAAQ;EAC9B;EACA;EACA;;YAGQ;EAEN;EACA,QAAQ;EACR;EACA;;EAGA;EACA,SACE,MAAM,uBACH,iBAAiB,QAAQ;EAC9B,SACE,QAAQ,gBACR,OAAO,wBACJ,iBAAiB,QAAQ;;;YAIxB;WACD;WACA;WACA;WACA,YAAY;;WAEZ;WACA,QAAQ;;;wBAIH,oBAAoB,eAAe;;;;;;;wBAuBnC,cAAc,gBAAgB,gBAC5C,YAAY,0BAA0B,0BAA0B,WAC/D;wBAmFa,UAAU,iBAAiB,SAAS;;;;;;;YC/MxC;EACN;EAAiB;;EAEjB;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB;EAAkB;;EACrC;EAAe,OAAO;EAAe;;YAE/B;EACV;EACA;;EAEA;EACA;;;YAIU;EAGN;EACA,qBAAqB;EACrB,QAAQ;;;wBAIE,kBACd,OAAO,eACP,WAAW,OACV;;;;;wBA0Da,iBACd,QAAQ,YACR,cACA,WAAW,OACV;;;;;;;;;YCtGS;;EAEV;;YAGU;EACV;EACA;EACA;EACA;EACA,UAAU;IAAQ,MAAM;IAAkB;;;EAE1C;;;YAIU;EACN;EAAe;;EACf;EAAgB;;EAChB;;YAEM;EACV;EACA;EACA,YAAY;EACZ;EACA;;EAEA;IAAU;;IAAwB;;;EAElC;IAAU,OAAO;;IAAiB;;;EAElC;IAAU;IAAc;;EACxB;;YAGU;EACV;EACA,MAAM;EACN;EACA;;EAEA;EACA;;YAGU;;;EAEN;EAAc;;EACd;EAAe,aAAa;;EAC5B;EAA0B;EAAc,YAAY;;;;EAEpD;EAAoB,QAAQ;;YAEtB;EACV;EACA;EACA;EACA;EACA,OAAO;;EAEP,aAAa;EACb,UAAU;EACV;EACA;IAAU;IAAgB;IAAoB;;EAC9C,YAAY;EACZ,SAAS;EACT,WAAW;;EAEX;;wBAGc,aACd,YAAY,gBACZ,OAAO,uBACP,UAAS,iBACR;;;;wBC5Fa,oBAAoB,OAAO;;;KCK/B;;EAEV;;EAEA,aAAa,SAAS,wBAAwB;;EAE9C,YAAY;;KAEF;EACV;;EAEA;EACA,MAAM;EACN,YAAY;;KAEF;EACV;EACA;EACA,gBAAgB;EAChB,YAAY,SAAS,wBAAwB;EAC7C,WAAW;;EAEX,iBAAiB;;EAEjB,mBAAmB;;KAET,aAAa;EACvB;EACA,MAAM,YAAY;EAClB,MAAM,YAAY,kBAAkB,iBAAiB;;wBAWvC,uBAAuB,UAAU,sBAC/C,YAAY,IACX,aAAa;;;KCjDJ;;EAEV;EACA;EACA;EACA;;IAEE;;;EAGF,iBAAiB;EACjB,WAAW;;EAEX,SAAS,KAAK;IAA0C;;EACxD;IAAc;IAAsB;IAAiB;;EACrD;;EAEA;;;;;;IAME;;;EAGF;;EAEA,gBAAgB;EAChB,QAAQ;;KAGE;;qBAGC;;KAWD,eAAe;KAEf;;EAEV;;;;;;EAMA,UAAU,sBAAsB,eAAe,QAAQ;;;wBAIzC,mBAAmB,UAAU,eAC3C,eAAe,IACd;;;;qBC9DU,qBAAqB;WAGrB;WACA;WAHO;EAClB,YACW,cACA;;;;;;;;wBAYG,OAAO,gBAAgB,gBAAiB;;wBA4CxC,aACd,gBACA;GACI,cAAc;;;;;;;;;;;KC3DR;EAAmB;EAAc;;wBAqB7B,eAAe,iBAAiB;;;;qBCvBnC,iBAAe,OAAA;qBAOf,qBAAmB,OAAA;;;;;qBAMnB,mBAAiB,OAAA;;;;;;qBAQjB,iBAAiB,OAAO,OAAO;qBAoB/B,iBAAe,OAAA;;;;;;;;;;;;;;qBAiBf,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAejB,qBAAmB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;qBAyBnB,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAIjB,qBAAmB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;qBAKnB,aAAW,OAAA;;;;;qBAMX,oBAAkB,OAAA;;;;;;;;;;;;;;qBAalB,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAYjB,wBAAsB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAgBtB,oBAAkB,OAAA;;;;;;;;;;;;qBAQlB,oBAAkB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAUlB,qBAAmB,OAAA;;;;;qBAKnB,oBAAkB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBASlB,kBAAgB,OAAA;;;;;;;;;;;;;;qBAmJhB,uBAAuB,OAAO,OAAO;qBAGrC,gBAAc,OAAA;;;;;;;wBChVX,qBAAqB,mBAAmB;;;YCgB5C;YACA,YACV,gBAAgB;GAAiB,cAAc;;YACrC;GAAgB,cAAc;;YAE9B;;EAEV;EACA;EACA;EACA;;YAGU;;;;;YAMA;EACV;EACA,gBAAgB;;YAGN,gBAAgB,QAAQ,OAAO,kBAAkB;YAEjD;EACV;EACA;EACA;;YAGU;EACV;EACA;EACA;;YAGU;EACN;EAAiB;;EAEjB;EACA;EACA,SAAS;EACT;;EAGA;EACA;EACA;EACA,WAAW;EACX;;EAGA;EACA;EACA;EACA,QAAQ;EACR;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB,SAAS;EAAW;;EACvC;EAAe,OAAO;EAAe;;YAE/B;EACN;EAAuB;;EAEvB;EACA;;EAEA;EACA;;EAEA;EAAmC;EAAc;;EACjD;EAAiC;EAAc;;EAE/C;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAwB;EAAgB;;EAExC;EACA;;EAEA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAyB;;EACzB;EAAe,OAAO;EAAe;;YAE/B,mBACT;EAAa;MAAoB;EAAgB;;YAExC;;EAEV;;YAGU,qBAAqB;YACrB,qBAAqB;YAErB;EACV;EACA;EACA;EACA;EACA,UAAU;;EAEV,aAAa;EACb,UAAU;;EAEV,MAAM;;EAEN,WAAW;;EAEX,oBAAoB;;YAGV,WAAW;WACZ,WAAW;;WAEX,WAAW;EACpB,MAAM;IACJ,SAAS;IACT,UAAU;;IAEV,UAAU,OAAO,yBAAyB;MACxC,QAAQ,WAAW;;YAGb,WAAW;;;;;EAKrB,KAAK,SAAS,WAAW,eAAe;;EAExC,UAAU;;;qBAIC,wBAAwB;;wBAKrB,cAAc,OAAO,kBAAkB;;;;;wBAUvC,kBAAkB,UAAU,YAAY,OAAO,IAAI;;YAMvD;;EAEV,YAAY;;EAEZ,MAAM;;EAEN;EACA;;YAGU;EACV;;EAEA,oBAAoB,OAAO;;YAGjB;EACV;EACA,aAAa;;YAGH;EACV;EACA,kBAAkB;IAAa;;EAC/B,6BAA6B,QAAQ;IAAY;;IAC/C;IACA;;EAEF;IACE;IACA;IACA;IACA,WAAW;IACX;IACA,SAAS;;EAEX;;YAGU;EACV;EACA;EACA;IAAY;IAAc,YAAY;;;YAE5B,iBACV,WAAW,YAAY,cAAc;;wBAGvB,eACd,UAAU,iCACT;;;;;wBAUa,KAAK,SAAS,sBAAsB;wBAIpC,MAAM,aAAa,aAAa;YAIpC;EACV,QAAQ;EACR,QAAQ;;EAER,SAAS;;;YAIC;EAAiB;EAAe;;YAEhC,sBAER;EAEE;EACA;EACA;;EAEA;;EAEA,QAAQ;;;;;;wBAOE,KAAK,iBAAiB;YAK1B;EACV;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;;EAER;;YAGU;EACV,WAAW;EACX,WAAW;;YAGD;EACV,SAAS;EACT;IAAc,iBAAiB;;EAC/B,WAAW;;EAEX,OAAO;;EAEP,SAAS;;EAET,MAAM;;;;;;EAMN,sBAAsB;;YAGZ;EACV;EACA;EACA;;EAEA;EACA,IACE,SAAS,2BACE,aAAa,kBAAkB;;YAGlC;EACV;EACA;EACA;;EAEA;;YAGU,cAAc,kBAAkB;wBAE5B,UACd,cACA,KAAK,wBACL,UAAS,KAAK,iDACb;;;;;;YAqBS;EACV;EACA;;EAEA;;EAEA;EACA,IACE,SAAS,sCACa;;wBAGV,OACd,cACA,KAAK,eACL,UAAS,KAAK,6BACb;;YAMS;EACV;;EAEA;EACA;EACA;;EAEA;EACA,QAAQ;;wBAGM,MACd,cACA,SAAS,KAAK,2CACb;YAMS;EACV;;EAEA;;EAEA;;YAGU,eAAe,eAAe,aAAa;EACrD;EACA;EACA,OAAO;;EAEP,QAAQ;;EAER,oBAAoB;EACpB,YAAY;EACZ,SAAS;;EAET,mBAAmB;EACnB,SAAS;;EAET,WAAW;;wBAGG,iBAAiB,UAAU,gBAAgB,YAAY,IAAI;YAqD/D,UACV,wBAAwB,4BAA4B;EAEpD;EACA;EACA,OAAO;;wBAGO,kBAAkB,eAAe,WAC/C,OAAO,SACN;YAKS,mBAAmB,iBAAiB,YAAY;wBAE5C,YAAY;EAAS;;YAkBzB;;EAEV,gBAAgB;EAChB,iBAAiB;EACjB,mBAAmB;EACnB,IAAI,cAAc;EAClB,SAAS,cAAc;EACvB,YAAY;EACZ,iBAAiB;;EAEjB,QAAQ,UAAU,iBAAiB;;YAGzB;EACV;EACA;EACA;;YAGU;EACV;EACA;EACA;;;;;;wBAOc,oBACR,gCAAgC,oBACtC,eAAe,iBAAiB;YAuEtB;;;;;YAMA;;EAEV;;EAEA;;EAEA;EACA;EACA;;EAEA;EACA;IAAO;IAAkB;IAAmB;IAAc;;;;YAIhD;EACV,SAAS;EACT,OAAO;EACP,QAAQ;;;YAIE;EAAe;EAAa;;YAE5B;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,SAAS;;EAET,UAAU;;YAGA;EACV;EACA;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,WAAW;;YAGD;EACV;EACA,YAAY;EACZ;;EAEA;;YAGU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;EACR;IAAoB;IAAoB;;;;qBAI7B;YAMD;EACV,SAAS;EACT,UAAU;EACV,cAAc;EACd;EACA;EACA;;YAGU;;EAEV;EACA;EACA;;YAGU;EACV,QAAQ;EACR;EACA;EACA,UAAU;EACV,YAAY;EACZ,QAAQ;;YAGE;EACV,QAAQ;EACR;EACA;EACA;EACA;EACA;;EAEA;EACA,QAAQ;;YAGE;EACV,YAAY,OAAO,kBAAkB;EACrC,YAAY,SAAS,eAAe;;EAEpC,cAAc,cAAc,MAAM,aAAa;EAC/C,SAAS,SAAS,eAAe;;YAGvB;WACD;EACT,WAAW,UAAU,gBAAgB,QAAQ;EAC7C,SAAS,SAAS,aAAa;;YAGrB;EACV,SACE,UAAU,aACV;IAAY,UAAU;MACrB,QAAQ;;YAGD;EACV;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,UAAU;EACV,QAAQ;;YAGE;EACV;EACA;EACA;EACA;EACA;;YAGU,YAAY;EACtB;EACA;EACA;EACA;EACA,mBAAmB;;EAEnB,SAAS;;wBAYK,YAAY,iBAAiB;;;KC5zBjC;EACV;;EAEA;;;wBAIc,QAAQ,UAAS,eAAoB"}
|
package/dist/index.mjs
CHANGED
|
@@ -1,10 +1,4 @@
|
|
|
1
|
-
import { $ as
|
|
2
|
-
//#region ../core/src/config.ts
|
|
3
|
-
/** Pure, typed project configuration. Loading/discovery belongs to the CLI. */
|
|
4
|
-
function defineConfig(configuration) {
|
|
5
|
-
return configuration;
|
|
6
|
-
}
|
|
7
|
-
//#endregion
|
|
1
|
+
import { $ as ResourceUriSchema, A as RunSummarySchema, B as defineEvalMatrix, C as JudgeRunInfoSchema, D as RunMetadataSchema, E as RunManifestSchema, F as TrialScoringSchema, G as NotJsonError, H as DEFAULT_ARTIFACT_EXCLUDE, I as TrialSummarySchema, J as defineDataset, K as toJson, L as UsageSchema, M as TrajectoryEventSchema, N as TrialManifestSchema, O as RunSourceSchema, P as TrialMetadataSchema, Q as ResourceKindSchema, R as normalizeAutEvent, S as JsonValueSchema, T as RecordedErrorSchema, U as defineConfig, V as canonicalParameters, W as catalogEntry, X as relativeDatasetPath, Y as isDataset, Z as definitionSite, _ as parseTrajectoryJsonl, a as authoringId, at as resourceUuid, b as AutIdentitySchema, c as defineSuite, d as metric, et as UuidSchema, f as predicate, g as user, h as skip, i as agentRuntimes, it as resourceUriSchema, j as ScoreResultSchema, k as RunStatusSchema, l as expectToolCall, m as registerEvals, n as DEFAULT_AGENT_RUNTIMES, nt as parseResourceUri, o as defineAgent, p as recordError, q as toJsonObject, r as agent, rt as resourceUri, s as defineEval, t as BUILT_IN_METRICS, tt as authorId, u as judge, v as ApiErrorSchema, w as MetricResultSchema, x as CheckpointResultSchema, y as ArtifactEntrySchema, z as turnResultEvents } from "./src-b2Wnmzbb.mjs";
|
|
8
2
|
//#region ../agents/src/index.ts
|
|
9
3
|
/** A local, one-prompt-per-turn AUT backed by Pi's non-interactive print mode. */
|
|
10
4
|
function piAgent(options = {}) {
|
|
@@ -74,4 +68,4 @@ function piAgent(options = {}) {
|
|
|
74
68
|
});
|
|
75
69
|
}
|
|
76
70
|
//#endregion
|
|
77
|
-
export { ApiErrorSchema, ArtifactEntrySchema, AutIdentitySchema, CheckpointResultSchema, DEFAULT_AGENT_RUNTIMES, JsonValueSchema, JudgeRunInfoSchema, NotJsonError, RecordedErrorSchema, ResourceKindSchema, ResourceUriSchema, RunManifestSchema, RunMetadataSchema, RunSourceSchema, RunStatusSchema, RunSummarySchema, ScoreResultSchema, TrajectoryEventSchema, TrialManifestSchema, TrialMetadataSchema, TrialScoringSchema, TrialSummarySchema, UsageSchema, UuidSchema, agent, agentRuntimes, authorId, authoringId, canonicalParameters, catalogEntry, defineAgent, defineConfig, defineDataset, defineEval, defineEvalMatrix, defineSuite, definitionSite, expectToolCall, isDataset, judge, normalizeAutEvent, parseResourceUri, parseTrajectoryJsonl, piAgent, predicate, recordError, registerEvals, relativeDatasetPath, resourceUri, resourceUriSchema, resourceUuid, skip, toJson, toJsonObject, turnResultEvents, user };
|
|
71
|
+
export { ApiErrorSchema, ArtifactEntrySchema, AutIdentitySchema, BUILT_IN_METRICS, CheckpointResultSchema, DEFAULT_AGENT_RUNTIMES, DEFAULT_ARTIFACT_EXCLUDE, JsonValueSchema, JudgeRunInfoSchema, MetricResultSchema, NotJsonError, RecordedErrorSchema, ResourceKindSchema, ResourceUriSchema, RunManifestSchema, RunMetadataSchema, RunSourceSchema, RunStatusSchema, RunSummarySchema, ScoreResultSchema, TrajectoryEventSchema, TrialManifestSchema, TrialMetadataSchema, TrialScoringSchema, TrialSummarySchema, UsageSchema, UuidSchema, agent, agentRuntimes, authorId, authoringId, canonicalParameters, catalogEntry, defineAgent, defineConfig, defineDataset, defineEval, defineEvalMatrix, defineSuite, definitionSite, expectToolCall, isDataset, judge, metric, normalizeAutEvent, parseResourceUri, parseTrajectoryJsonl, piAgent, predicate, recordError, registerEvals, relativeDatasetPath, resourceUri, resourceUriSchema, resourceUuid, skip, toJson, toJsonObject, turnResultEvents, user };
|
package/dist/runner.d.mts
CHANGED
|
@@ -221,9 +221,11 @@ type EvalCatalogScorer = {
|
|
|
221
221
|
rubric?: string;
|
|
222
222
|
definedAt?: string;
|
|
223
223
|
};
|
|
224
|
-
type EvalCatalogStep =
|
|
224
|
+
type EvalCatalogStep =
|
|
225
|
+
/** `message` is absent when it is computed for each trial. */
|
|
226
|
+
{
|
|
225
227
|
kind: 'user';
|
|
226
|
-
message
|
|
228
|
+
message?: string;
|
|
227
229
|
} | {
|
|
228
230
|
kind: 'agent';
|
|
229
231
|
expectation: JsonValue;
|
|
@@ -292,6 +294,9 @@ type EvalMatrix = EvalMatrixDefinition & {
|
|
|
292
294
|
cells(selection?: MatrixSelection): IterableIterator<EvalMatrixCell>;
|
|
293
295
|
};
|
|
294
296
|
//#endregion
|
|
297
|
+
//#region ../core/src/config.d.ts
|
|
298
|
+
type KeepSandboxes = 'failed' | 'all' | 'none';
|
|
299
|
+
//#endregion
|
|
295
300
|
//#region ../core/src/index.d.ts
|
|
296
301
|
type JsonPrimitive = boolean | number | string | null;
|
|
297
302
|
type JsonValue = JsonPrimitive | JsonValue[] | {
|
|
@@ -469,9 +474,20 @@ type AutSession<TMessage = string> = {
|
|
|
469
474
|
/** Optional; defaults to doing nothing. */
|
|
470
475
|
close?(): Promise<void>;
|
|
471
476
|
};
|
|
477
|
+
/** What a computed user message may depend on: only what the agent is given too. */
|
|
478
|
+
type UserMessageInput = {
|
|
479
|
+
/** The matrix cell's parameters and CLI overrides. */
|
|
480
|
+
parameters: JsonObject;
|
|
481
|
+
/** Values of the scenario's candidate datasets, by dataset ID (never evaluator ones). */
|
|
482
|
+
data: JsonObject;
|
|
483
|
+
/** The trial's random seed (also in `context.metadata.randomSeed`). */
|
|
484
|
+
seed: number;
|
|
485
|
+
trialIndex: number;
|
|
486
|
+
};
|
|
472
487
|
type UserStep = {
|
|
473
488
|
kind: 'user';
|
|
474
|
-
message
|
|
489
|
+
/** A fixed message, or one computed for each trial. */
|
|
490
|
+
message: string | ((input: UserMessageInput) => string);
|
|
475
491
|
};
|
|
476
492
|
type AgentStep = {
|
|
477
493
|
kind: 'agent';
|
|
@@ -556,6 +572,12 @@ type ScoringContext = {
|
|
|
556
572
|
output?: JsonValue;
|
|
557
573
|
/** Values of all the scenario's datasets (candidate and evaluator), by dataset ID. */
|
|
558
574
|
data: JsonObject;
|
|
575
|
+
/**
|
|
576
|
+
* Results of the transcript's checkpoints so far, in order: all of them for
|
|
577
|
+
* final scorers. A checkpoint's `evidence` can carry what it observed (for
|
|
578
|
+
* example, in a container that is gone by final scoring).
|
|
579
|
+
*/
|
|
580
|
+
checkpoints: readonly CheckpointResult[];
|
|
559
581
|
};
|
|
560
582
|
type PredicateScorer = {
|
|
561
583
|
kind: 'predicate';
|
|
@@ -573,6 +595,31 @@ type JudgeRule = {
|
|
|
573
595
|
supportsPartial?: boolean;
|
|
574
596
|
};
|
|
575
597
|
type ScoringRule = PredicateScorer | JudgeRule;
|
|
598
|
+
/**
|
|
599
|
+
* A number measured for every trial, such as tokens, retries or web searches.
|
|
600
|
+
* Metrics never decide pass or fail; they are averaged per run and shown on
|
|
601
|
+
* the scoreboard. Return undefined when there is nothing to measure.
|
|
602
|
+
*/
|
|
603
|
+
type Metric = {
|
|
604
|
+
kind: 'metric';
|
|
605
|
+
name: string;
|
|
606
|
+
/** Shown next to values: `ms` formats as a duration, others as numbers. */
|
|
607
|
+
unit?: string;
|
|
608
|
+
/** Which direction is an improvement, for comparisons. */
|
|
609
|
+
better?: 'lower' | 'higher';
|
|
610
|
+
run(context: ScoringContext): number | undefined | Promise<number | undefined>;
|
|
611
|
+
};
|
|
612
|
+
/** A metric's value for one trial. */
|
|
613
|
+
type MetricResult = {
|
|
614
|
+
name: string;
|
|
615
|
+
/** Absent when the metric measured nothing or failed. */
|
|
616
|
+
value?: number;
|
|
617
|
+
unit?: string;
|
|
618
|
+
better?: 'lower' | 'higher';
|
|
619
|
+
/** Measured by EvalKit itself (agent time, tokens, tool calls). */
|
|
620
|
+
builtIn?: true;
|
|
621
|
+
error?: RecordedError;
|
|
622
|
+
};
|
|
576
623
|
type EvalPolicy = {
|
|
577
624
|
timeoutMs?: number;
|
|
578
625
|
/** Stop later transcript steps after a failed checkpoint assertion. Default false. */
|
|
@@ -590,6 +637,8 @@ type EvalDefinition<TAgent extends AutAdapter = AutAdapter> = {
|
|
|
590
637
|
datasets?: readonly Dataset[];
|
|
591
638
|
transcript: TranscriptStep[];
|
|
592
639
|
scoring: ScoringRule[];
|
|
640
|
+
/** Numbers to measure for every trial; they never affect pass or fail. */
|
|
641
|
+
metrics?: readonly Metric[];
|
|
593
642
|
policy?: EvalPolicy;
|
|
594
643
|
/** Any JSON-convertible object; converted when the eval is defined. */
|
|
595
644
|
metadata?: Record<string, unknown>;
|
|
@@ -700,6 +749,7 @@ type CheckpointResult = {
|
|
|
700
749
|
};
|
|
701
750
|
type TrialScoring = {
|
|
702
751
|
results: ScoreResult[];
|
|
752
|
+
metrics?: MetricResult[];
|
|
703
753
|
checkpoints?: CheckpointResult[];
|
|
704
754
|
skippedScorers?: string[];
|
|
705
755
|
overall?: number;
|
|
@@ -912,6 +962,18 @@ export declare const readTrialSummary: (root: string, runId: string, trialId: st
|
|
|
912
962
|
} | undefined;
|
|
913
963
|
readonly skipped?: true | undefined;
|
|
914
964
|
}[];
|
|
965
|
+
readonly metrics?: {
|
|
966
|
+
readonly name: string;
|
|
967
|
+
readonly value?: number | undefined;
|
|
968
|
+
readonly unit?: string | undefined;
|
|
969
|
+
readonly better?: "higher" | "lower" | undefined;
|
|
970
|
+
readonly builtIn?: true | undefined;
|
|
971
|
+
readonly error?: {
|
|
972
|
+
readonly name: string;
|
|
973
|
+
readonly message: string;
|
|
974
|
+
readonly stack?: string | undefined;
|
|
975
|
+
} | undefined;
|
|
976
|
+
}[] | undefined;
|
|
915
977
|
readonly checkpoints?: {
|
|
916
978
|
readonly step: number;
|
|
917
979
|
readonly kind: "expect-tool-call" | "judge" | "predicate";
|
|
@@ -991,6 +1053,18 @@ export declare const readTrialScoring: (root: string, runId: string, trialId: st
|
|
|
991
1053
|
} | undefined;
|
|
992
1054
|
readonly skipped?: true | undefined;
|
|
993
1055
|
}[];
|
|
1056
|
+
readonly metrics?: {
|
|
1057
|
+
readonly name: string;
|
|
1058
|
+
readonly value?: number | undefined;
|
|
1059
|
+
readonly unit?: string | undefined;
|
|
1060
|
+
readonly better?: "higher" | "lower" | undefined;
|
|
1061
|
+
readonly builtIn?: true | undefined;
|
|
1062
|
+
readonly error?: {
|
|
1063
|
+
readonly name: string;
|
|
1064
|
+
readonly message: string;
|
|
1065
|
+
readonly stack?: string | undefined;
|
|
1066
|
+
} | undefined;
|
|
1067
|
+
}[] | undefined;
|
|
994
1068
|
readonly checkpoints?: {
|
|
995
1069
|
readonly step: number;
|
|
996
1070
|
readonly kind: "expect-tool-call" | "judge" | "predicate";
|
|
@@ -1052,6 +1126,10 @@ export declare function runMatrix(matrix: EvalMatrix, options: Omit<RunEvalOptio
|
|
|
1052
1126
|
onResult?: (cell: EvalMatrixCell, result: RunResult) => void | Promise<void>;
|
|
1053
1127
|
}): Effect.Effect<MatrixSummary, unknown>;
|
|
1054
1128
|
//#endregion
|
|
1129
|
+
//#region ../runner/src/workspace.d.ts
|
|
1130
|
+
/** Where a trial's persistent workspace lives below `root`. */
|
|
1131
|
+
export declare function trialWorkspacePath(root: string, trialId: string): string;
|
|
1132
|
+
//#endregion
|
|
1055
1133
|
//#region ../runner/src/index.d.ts
|
|
1056
1134
|
export type RunEvalOptions = {
|
|
1057
1135
|
report: ReportStore;
|
|
@@ -1072,6 +1150,10 @@ export type RunEvalOptions = {
|
|
|
1072
1150
|
semaphore?: Effect.Semaphore;
|
|
1073
1151
|
/** Base directory for persistent local trial workspaces. Omit for an ephemeral OS temp workspace. */
|
|
1074
1152
|
workspaceRoot?: string;
|
|
1153
|
+
/** With `workspaceRoot`: which trial workspaces to keep afterwards. Default `'failed'`. */
|
|
1154
|
+
keepWorkspaces?: KeepSandboxes;
|
|
1155
|
+
/** Candidate paths left out of the artifact snapshot. Default `DEFAULT_ARTIFACT_EXCLUDE`. */
|
|
1156
|
+
artifactExclude?: readonly string[];
|
|
1075
1157
|
/** Repository and commit the eval code came from, recorded in the manifest. */
|
|
1076
1158
|
source?: RunSource;
|
|
1077
1159
|
/** The project's evals and suites at run time, saved beside the manifest. */
|
package/dist/runner.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.mts","names":["_$_evalkit_core0.JsonValue","_$_evalkit_core0.TrajectoryEvent"],"sources":["../../runner/src/errors.ts","../../core/src/identity.ts","../../core/src/dataset.ts","../../core/src/aut-events.ts","../../core/src/catalog.ts","../../core/src/matrix.ts","../../core/src/index.ts","../../runner/src/local-report-store.ts","../../runner/src/datasets.ts","../../runner/src/local-report-reader.ts","../../runner/src/matrix.ts","../../runner/src/index.ts"],"mappings":";;;;;;qBAEa,0BAA0B;EACrC;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBAGW,oBAAoB;EAC/B;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBASW,iCAAiC;EAG5C;EACA;EACA;EACA;;;;;;qBAIW,uBAAuB;EAClC;;;;cCrCW,oBAAkB,OAAA;KACnB,sBAAsB,mBAAmB;KAiBzC,YAAY,cAAc,eAAe,2BACxC;;;;KCdD;;KAGA,6BAA6B;EAAe;;;;;;KAM5C,wBAAwB,SAAS,eAAe;;KAGhD;EACV,QAAQ;;EAER;;KAGU;;EAEV;EACA,QAAQ;;KAGE;EACV;EACA;EACA;EACA;;EAEA;;EAEA,YAAY;EACZ,QAAQ;;KAoCE;EAEN;EACA,QAAQ;EACR;EACA;;EAGA;EACA,SACE,MAAM,uBACH,iBAAiB,QAAQ;EAC9B,SACE,QAAQ,gBACR,OAAO,wBACJ,iBAAiB,QAAQ;;;KAIxB;WACD;WACA;WACA;WACA,YAAY;;WAEZ;WACA,QAAQ;;;;;;;;KC/FP;EACN;EAAiB;;EAEjB;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB;EAAkB;;EACrC;EAAe,OAAO;EAAe;;KAE/B;EACV;EACA;;EAEA;EACA;;;KAIU;EAGN;EACA,qBAAqB;EACrB,QAAQ;;;;KC5BF;EACV;EACA;EACA;EACA;EACA,UAAU;IAAQ,MAAM;IAAkB;;;EAE1C;;;KAIU;EACN;EAAe;;EACf;EAAgB;;EAChB;;KAEM;EACV;EACA;EACA,YAAY;EACZ;EACA;;EAEA;IAAU;;IAAwB;;;EAElC;IAAU,OAAO;;IAAiB;;;EAElC;IAAU;IAAc;;EACxB;;KAGU;EACV;EACA,MAAM;EACN;EACA;;EAEA;EACA;;KAGU;
|
|
1
|
+
{"version":3,"file":"runner.d.mts","names":["_$_evalkit_core0.JsonValue","_$_evalkit_core0.TrajectoryEvent"],"sources":["../../runner/src/errors.ts","../../core/src/identity.ts","../../core/src/dataset.ts","../../core/src/aut-events.ts","../../core/src/catalog.ts","../../core/src/matrix.ts","../../core/src/config.ts","../../core/src/index.ts","../../runner/src/local-report-store.ts","../../runner/src/datasets.ts","../../runner/src/local-report-reader.ts","../../runner/src/matrix.ts","../../runner/src/workspace.ts","../../runner/src/index.ts"],"mappings":";;;;;;qBAEa,0BAA0B;EACrC;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBAGW,oBAAoB;EAC/B;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBASW,iCAAiC;EAG5C;EACA;EACA;EACA;;;;;;qBAIW,uBAAuB;EAClC;;;;cCrCW,oBAAkB,OAAA;KACnB,sBAAsB,mBAAmB;KAiBzC,YAAY,cAAc,eAAe,2BACxC;;;;KCdD;;KAGA,6BAA6B;EAAe;;;;;;KAM5C,wBAAwB,SAAS,eAAe;;KAGhD;EACV,QAAQ;;EAER;;KAGU;;EAEV;EACA,QAAQ;;KAGE;EACV;EACA;EACA;EACA;;EAEA;;EAEA,YAAY;EACZ,QAAQ;;KAoCE;EAEN;EACA,QAAQ;EACR;EACA;;EAGA;EACA,SACE,MAAM,uBACH,iBAAiB,QAAQ;EAC9B,SACE,QAAQ,gBACR,OAAO,wBACJ,iBAAiB,QAAQ;;;KAIxB;WACD;WACA;WACA;WACA,YAAY;;WAEZ;WACA,QAAQ;;;;;;;;KC/FP;EACN;EAAiB;;EAEjB;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB;EAAkB;;EACrC;EAAe,OAAO;EAAe;;KAE/B;EACV;EACA;;EAEA;EACA;;;KAIU;EAGN;EACA,qBAAqB;EACrB,QAAQ;;;;KC5BF;EACV;EACA;EACA;EACA;EACA,UAAU;IAAQ,MAAM;IAAkB;;;EAE1C;;;KAIU;EACN;EAAe;;EACf;EAAgB;;EAChB;;KAEM;EACV;EACA;EACA,YAAY;EACZ;EACA;;EAEA;IAAU;;IAAwB;;;EAElC;IAAU,OAAO;;IAAiB;;;EAElC;IAAU;IAAc;;EACxB;;KAGU;EACV;EACA,MAAM;EACN;EACA;;EAEA;EACA;;KAGU;;;EAEN;EAAc;;EACd;EAAe,aAAa;;EAC5B;EAA0B;EAAc,YAAY;;;;EAEpD;EAAoB,QAAQ;;KAEtB;EACV;EACA;EACA;EACA;EACA,OAAO;;EAEP,aAAa;EACb,UAAU;EACV;EACA;IAAU;IAAgB;IAAoB;;EAC9C,YAAY;EACZ,SAAS;EACT,WAAW;;EAEX;;;;KChFU;;EAEV;;EAEA,aAAa,SAAS,wBAAwB;;EAE9C,YAAY;;KAEF;EACV;;EAEA;EACA,MAAM;EACN,YAAY;;KAEF;EACV;EACA;EACA,gBAAgB;EAChB,YAAY,SAAS,wBAAwB;EAC7C,WAAW;;EAEX,iBAAiB;;EAEjB,mBAAmB;;KAET,aAAa;EACvB;EACA,MAAM,YAAY;EAClB,MAAM,YAAY,kBAAkB,iBAAiB;;;;KCJ3C;;;KCdA;KACA,YACV,gBAAgB;GAAiB,cAAc;;KACrC;GAAgB,cAAc;;KAE9B;;EAEV;EACA;EACA;EACA;;KAGU;;;;;KAMA;EACV;EACA,gBAAgB;;KAGN,gBAAgB,QAAQ,OAAO,kBAAkB;KAEjD;EACV;EACA;EACA;;KAGU;EACV;EACA;EACA;;KAGU;EACN;EAAiB;;EAEjB;EACA;EACA,SAAS;EACT;;EAGA;EACA;EACA;EACA,WAAW;EACX;;EAGA;EACA;EACA;EACA,QAAQ;EACR;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB,SAAS;EAAW;;EACvC;EAAe,OAAO;EAAe;;KAE/B;EACN;EAAuB;;EAEvB;EACA;;EAEA;EACA;;EAEA;EAAmC;EAAc;;EACjD;EAAiC;EAAc;;EAE/C;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAwB;EAAgB;;EAExC;EACA;;EAEA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAyB;;EACzB;EAAe,OAAO;EAAe;;KAE/B,mBACT;EAAa;MAAoB;EAAgB;;KAExC;;EAEV;;KAGU,qBAAqB;KACrB,qBAAqB;KAErB;EACV;EACA;EACA;EACA;EACA,UAAU;;EAEV,aAAa;EACb,UAAU;;EAEV,MAAM;;EAEN,WAAW;;EAEX,oBAAoB;;KAGV,WAAW;WACZ,WAAW;;WAEX,WAAW;EACpB,MAAM;IACJ,SAAS;IACT,UAAU;;IAEV,UAAU,OAAO,yBAAyB;MACxC,QAAQ,WAAW;;KAGb,WAAW;;;;;EAKrB,KAAK,SAAS,WAAW,eAAe;;EAExC,UAAU;;;KAyBA;;EAEV,YAAY;;EAEZ,MAAM;;EAEN;EACA;;KAGU;EACV;;EAEA,oBAAoB,OAAO;;KAGjB;EACV;EACA,aAAa;;KAGH;EACV;EACA,kBAAkB;IAAa;;EAC/B,6BAA6B,QAAQ;IAAY;;IAC/C;IACA;;EAEF;IACE;IACA;IACA;IACA,WAAW;IACX;IACA,SAAS;;EAEX;;KAGU;EACV;EACA;EACA;IAAY;IAAc,YAAY;;;KAE5B,iBACV,WAAW,YAAY,cAAc;KAuB3B;EACV,QAAQ;EACR,QAAQ;;EAER,SAAS;;;KAIC;EAAiB;EAAe;;KAEhC,sBAER;EAEE;EACA;EACA;;EAEA;;EAEA,QAAQ;;KAYF;EACV;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;;EAER;;KAGU;EACV,WAAW;EACX,WAAW;;KAGD;EACV,SAAS;EACT;IAAc,iBAAiB;;EAC/B,WAAW;;EAEX,OAAO;;EAEP,SAAS;;EAET,MAAM;;;;;;EAMN,sBAAsB;;KAGZ;EACV;EACA;EACA;;EAEA;EACA,IACE,SAAS,2BACE,aAAa,kBAAkB;;KAGlC;EACV;EACA;EACA;;EAEA;;KAGU,cAAc,kBAAkB;;;;;;KA2BhC;EACV;EACA;;EAEA;;EAEA;EACA,IACE,SAAS,sCACa;;;KAad;EACV;;EAEA;EACA;EACA;;EAEA;EACA,QAAQ;;KAYE;EACV;;EAEA;;EAEA;;KAGU,eAAe,eAAe,aAAa;EACrD;EACA;EACA,OAAO;;EAEP,QAAQ;;EAER,oBAAoB;EACpB,YAAY;EACZ,SAAS;;EAET,mBAAmB;EACnB,SAAS;;EAET,WAAW;;KAwGD;EACV;EACA;EACA;;KAGU;EACV;EACA;EACA;;KAgFU;;;;;KAMA;;EAEV;;EAEA;;EAEA;EACA;EACA;;EAEA;EACA;IAAO;IAAkB;IAAmB;IAAc;;;;KAIhD;EACV,SAAS;EACT,OAAO;EACP,QAAQ;;;KAIE;EAAe;EAAa;;KAE5B;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,SAAS;;EAET,UAAU;;KAGA;EACV;EACA;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,WAAW;;KAGD;EACV;EACA,YAAY;EACZ;;EAEA;;KAGU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;EACR;IAAoB;IAAoB;;;KAU9B;EACV,SAAS;EACT,UAAU;EACV,cAAc;EACd;EACA;EACA;;KAGU;;EAEV;EACA;EACA;;KAGU;EACV,QAAQ;EACR;EACA;EACA,UAAU;EACV,YAAY;EACZ,QAAQ;;KAGE;EACV,QAAQ;EACR;EACA;EACA;EACA;EACA;;EAEA;EACA,QAAQ;;KAGE;EACV,YAAY,OAAO,kBAAkB;EACrC,YAAY,SAAS,eAAe;;EAEpC,cAAc,cAAc,MAAM,aAAa;EAC/C,SAAS,SAAS,eAAe;;KAGvB;WACD;EACT,WAAW,UAAU,gBAAgB,QAAQ;EAC7C,SAAS,SAAS,aAAa;;KAGrB;EACV,SACE,UAAU,aACV;IAAY,UAAU;MACrB,QAAQ;;KAGD;EACV;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,UAAU;EACV,QAAQ;;KAGE;EACV;EACA;EACA;EACA;EACA;;KAGU,YAAY;EACtB;EACA;EACA;EACA;EACA,mBAAmB;;EAEnB,SAAS;;;;;wBCvrBK,iBAAiB,wBAAwB;;;;KCiI7C;EAAqB;EAAc;;;;;;;wBAOzB,mBACpB,SAAS,SACT,eACC,QAAQ;;wBA6BW,kBACpB,SAAS,SACT,cACA,eACC,QAAQ,WAAW;;;qBC3RT,kBAAe,cAAgB,kBAAe;;;;;;0BAA9CA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAEA,iBAAc,cAAgB,kBAAe;;;;;;;;;;;;;;qBAE7C,oBAAiB,cAChB,eACC,oBACE;;;;;;;0BAHJA;;;;;;;;;;;;;;;;;;;;;qBASA,mBAAgB,cACf,eACC,oBACE;;;;;;;;;;;0BAHJA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBASA,mBAAgB,cACf,eACC,oBACE;;;;;;;wBAHJA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;wBASS,gBACpB,cACA,eACA,kBAAe,QAHKC;;;;;wBAiBN,YAAY,OAAO;;;;KC7DvB;EACV;EACA;EACA;EACA;;;wBAIc,UACd,QAAQ,YACR,SAAS,KACP;EAGA,YAAY;;EAEZ,YAAY,MAAM;EAClB,YACE,MAAM,gBACN,QAAQ,qBACE;IAEb,OAAO,OAAO;;;;wBChBD,mBAAmB,cAAc;;;YCyDrC;EACV,QAAQ;EACR;EACA;EACA;;EAEA;;EAEA,aAAa;;EAEb,SAAS;;EAET,UAAU;;EAEV;;EAEA,YAAY,OAAO;;EAEnB;;EAEA,iBAAiB;;EAEjB;;EAEA,SAAS;;EAET,UAAU;;EAEV;;;;;EAKA,SAAS;;;;;EAKT,WAAW,OAAO,UAAU,OAAO;EACnC,YAAY;;;YAIF;EACV;EACA;EACA;EACA;EACA;EACA,aAAa;;;wBASC,QACd,YAAY,gBACZ,SAAS,iBACR,OAAO,OAAO"}
|
package/dist/runner.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { _ as
|
|
2
|
-
export { AutExecutionError, CancelledError, CheckpointExecutionError, DatasetError, ReportError, ScoringError, localReportStore, ownerIsGone, readRunManifest, readRunSummary, readTrialEvents, readTrialManifest, readTrialScoring, readTrialSummary, runEval, runMatrix, staticDatasetFile, staticDatasetFiles };
|
|
1
|
+
import { _ as CheckpointExecutionError, a as readRunSummary, b as ScoringError, c as readTrialScoring, d as trialWorkspacePath, f as staticDatasetFile, g as CancelledError, h as AutExecutionError, i as readRunManifest, l as readTrialSummary, n as runMatrix, o as readTrialEvents, p as staticDatasetFiles, r as ownerIsGone, s as readTrialManifest, t as runEval, u as localReportStore, v as DatasetError, y as ReportError } from "./src-DhromKNu.mjs";
|
|
2
|
+
export { AutExecutionError, CancelledError, CheckpointExecutionError, DatasetError, ReportError, ScoringError, localReportStore, ownerIsGone, readRunManifest, readRunSummary, readTrialEvents, readTrialManifest, readTrialScoring, readTrialSummary, runEval, runMatrix, staticDatasetFile, staticDatasetFiles, trialWorkspacePath };
|
|
@@ -8,7 +8,7 @@ description: Define or modify EvalKit scenarios (evals) and suites. Use when add
|
|
|
8
8
|
Read `../evalkit/SKILL.md`, `../../../sites/evalkit.leostera.dev/src/content/docs/manual/project-structure.md` and `../../../sites/evalkit.leostera.dev/src/content/docs/manual/scoring-and-evals.md`, `../../../examples/starter/scenarios/greeting.eval.ts`, and `../../../examples/configured-matrix/evalkit.config.ts`.
|
|
9
9
|
|
|
10
10
|
1. Default-export one scenario per file from `scenarios/**/*.eval.ts` using `defineEval({ id: 'my-scenario', name, agent, transcript, scoring, datasets?, policy? })`. The CLI discovers these files automatically; no registry or config is required. Choose a stable lowercase kebab-case ID. `metadata` is optional; `name` is a display label. Import shared agents, judges and datasets; keep each scenario short and concrete (a suite of a hundred cases is a hundred files), and never generate scenarios in a loop.
|
|
11
|
-
2. Executable transcript steps are `user('message')`, inline `predicate(...)`, `judge(...)`, and `expectToolCall(...)`. User messages call `session.send` in order on one fresh session per trial; predicates/judges/tool-call assertions run at checkpoints. `judge(...)` requires a separate eval-level `judge` adapter. `agent(...)` transcript steps are not executable. `{
|
|
11
|
+
2. Executable transcript steps are `user('message')`, inline `predicate(...)`, `judge(...)`, and `expectToolCall(...)`. User messages call `session.send` in order on one fresh session per trial; predicates/judges/tool-call assertions run at checkpoints. `judge(...)` requires a separate eval-level `judge` adapter. `agent(...)` transcript steps are not executable. `user(({ parameters, data, seed, trialIndex }) => ...)` computes a message per trial, so one scenario can vary its prompt by matrix axis or candidate dataset instead of being copied per variant; `policy.trials` controls independent trials (default 1, positive integer); `policy.timeoutMs` is not enforced.
|
|
12
12
|
3. Suites are directories: every file below `scenarios/billing/` is in suite `billing` (kebab-case directory names; deeper directories stay in the top one). Files directly in `scenarios/` have no suite; `discovery.suites: false` turns this off. Only use an explicit registry (`registry: registerEvals([defineSuite({ id, evals })])` in the config) when you need full control; it replaces discovery, and duplicate eval IDs throw.
|
|
13
13
|
4. For an intervention comparison, hold task/model/prompt constant and vary one explicit matrix axis (for example `memory: ['off', 'on']`). Repeat trials and include a negative control where possible. Record whether the treatment injects a saved memory or performs live retrieval: those answer different questions. A memory dataset can be evaluator-visible to the trusted adapter, but do not expose evaluator roots or reference answers to the AUT unless revealing the chosen treatment is intentional.
|
|
14
14
|
5. From the project directory run `bun run evalkit run <eval-id>`. Use `bun run evalkit run --suite <suite-id>` for a suite directory. IDs are exact human-defined strings. Inspect per-trial `scoring.json` and the run `summary.json`; completed execution is not the same as passing scores.
|
|
@@ -9,9 +9,10 @@ Read `../evalkit/SKILL.md`, `../../../sites/evalkit.leostera.dev/src/content/doc
|
|
|
9
9
|
|
|
10
10
|
EvalKit currently executes deterministic `predicate(name, callback)` scorers and agent-backed `judge(name, { rubric })` rules. A judge rule requires a **separate eval-level `judge` adapter**; it is not inferred from or shared with the AUT. The `judgeScorer(...)` API is not implemented—do not suggest it.
|
|
11
11
|
|
|
12
|
-
1. Add a named `predicate(...)` for deterministic checks or a `judge(...)` descriptor for rubric-based LLM assessment, then reference it from the eval's `scoring` array or supported transcript checkpoint. The predicate callback receives `{ context, output, data, trajectory, artifacts, turn? }
|
|
12
|
+
1. Add a named `predicate(...)` for deterministic checks or a `judge(...)` descriptor for rubric-based LLM assessment, then reference it from the eval's `scoring` array or supported transcript checkpoint. The predicate callback receives `{ context, output, data, trajectory, artifacts, checkpoints, turn? }`; `checkpoints` are earlier checkpoint results (all of them in final scoring), so a checkpoint that inspects live state before the session closes can hand its `evidence` to final scorers. Use `output` for the final assistant reply and `data[id]` for dataset values (expected answers belong in an evaluator dataset); only filter `trajectory.events` (by `source: 'aut'` and `kind`) for tool calls or intermediate messages. `context` includes per-trial IDs, metadata, random seed and optional parameters; workspace roots are absolute local paths.
|
|
13
13
|
2. A predicate returns a boolean, a finite score from 0 to 1, `{ value, passed?, explanation?, evidence? }` (evidence may be any plain value), or `skip(reason)` when the check does not apply. Without `passed`, only `value === 1` passes; give `predicate(name, fn, { passAt: 0.8 })` for a threshold. A skipped result neither passes nor fails and is left out of averages; never fake "not applicable" as a pass. Use clear evidence for debugging.
|
|
14
14
|
3. A `judge(...)` rule sends a rubric plus the current turn or trajectory to the separate judge adapter, which must emit a JSON verdict. Validate that the judge is independent from the AUT, runs in an appropriate workspace, and does not receive secrets. Judge events/prompts may contain sensitive data and incur model cost; cost is not inferred unless the adapter emits usage.
|
|
15
15
|
4. Normal scorers are skipped after primary execution errors unless declared with `{ supportsPartial: true }` and the required context/workspace exists. Scorer exceptions and invalid scores become recorded failures; never mask them as passes.
|
|
16
16
|
5. Scorers run after the AUT closes; use `node:fs/promises` to inspect candidate/evaluator files before workspace cleanup. Keep evaluator references and secrets out of AUT events, evidence, and exported reports. Reading one evaluator dataset in a trusted adapter and deliberately injecting it as an experiment treatment is valid, but measures injected context—not whether the AUT retrieved it itself.
|
|
17
|
-
6.
|
|
17
|
+
6. Numbers to compare rather than gate on (tokens, time, retries, mistakes, web searches) are **metrics**, not scorers: `metric(name, (context) => number | undefined, { unit?, better? })` in the eval's `metrics` array. Never fake one as a predicate that always passes. Agent time, tokens (when the adapter reports usage) and tool calls are recorded automatically; have the adapter emit `turn-completed` usage so tokens appear. Means show on the run page and scoreboard.
|
|
18
|
+
7. Test passing, failing, and missing-output trajectories; inspect `scoring.json` (`results`, `overall`, `passed`) and run `summary.json`. `overall` is an unweighted mean of valid scores, not the pass rate. Execution `status: 'completed'` can still have failed scoring; run-level `failed` is the meaningful aggregate gate.
|
|
@@ -7,7 +7,7 @@ description: Run Evalkit locally and interpret or process evaluation results. Us
|
|
|
7
7
|
|
|
8
8
|
Read `../evalkit/SKILL.md` and `../../../sites/evalkit.leostera.dev/src/content/docs/index.md` (First run) and `../../../sites/evalkit.leostera.dev/src/content/docs/manual/cli-and-matrices.md` and `../../../sites/evalkit.leostera.dev/src/content/docs/manual/results.md`. Verify report semantics in `../../../packages/runner/src/index.ts`, `../../../packages/runner/src/local-report-store.ts`, and `../../../packages/cli/src/index.ts` before automation.
|
|
9
9
|
|
|
10
|
-
1. Run from the project root containing default-exported `scenarios/**/*.eval.ts` (as in `examples/starter`, or use an optional config as in `examples/configured-matrix`), not this skill directory. Start with `bun run evalkit run <eval-id>`; run all evals only when intended (Pi-backed evals can cost tokens). `bun run evalkit run --suite <suite-id>`, `bun run evalkit run --matrix <matrix-id>`, and `bun run evalkit dev` are also available; `bun run evalkit build [--out dir]` writes a read-only static dashboard snapshot (reports plus candidate artifacts, never evaluator workspaces) that must still be reviewed before sharing; it asks to run evals first (always runs them when there are no results; `--run`/`--no-run` skip the prompt). `bun run evalkit tidy` archives runs of evals no longer in the project (marker `archived.json`, nothing deleted); archived runs are hidden in the dashboard and excluded from builds. Runs record `source` (repo, project, commit, ref, dirty) in `manifest.json`; `bun run evalkit share` uploads runs for HEAD to the configured share server (`share.url` or `EVALKIT_SHARE_URL`; `share.headers` (object or function) adds headers for an auth proxy in front of the server; `--all`, `--allow-dirty`, `--no-workspace`) and prints the URL. `bun run evalkit share serve` runs a local share server.
|
|
10
|
+
1. Run from the project root containing default-exported `scenarios/**/*.eval.ts` (as in `examples/starter`, or use an optional config as in `examples/configured-matrix`), not this skill directory. Start with `bun run evalkit run <eval-id>`; run all evals only when intended (Pi-backed evals can cost tokens). `bun run evalkit run --suite <suite-id>`, `bun run evalkit run --matrix <matrix-id>`, and `bun run evalkit dev` are also available; `bun run evalkit build [--out dir]` writes a read-only static dashboard snapshot (reports plus candidate artifacts, never evaluator workspaces) that must still be reviewed before sharing; it asks to run evals first (always runs them when there are no results; `--run`/`--no-run` skip the prompt). `bun run evalkit tidy` archives runs of evals no longer in the project (marker `archived.json`, nothing deleted); archived runs are hidden in the dashboard and excluded from builds. Runs record `source` (repo, project, commit, ref, dirty) in `manifest.json`; `bun run evalkit share` uploads runs for HEAD to the configured share server (`share.url` or `EVALKIT_SHARE_URL`; `share.headers` (object or function) adds headers for an auth proxy in front of the server; `--all`, `--allow-dirty`, `--no-workspace`) and prints the URL. `bun run evalkit share serve` runs a local share server. Pass `dev --no-open` or `dev --port <n>` (`-p`) for dashboard behavior, and `--log-level debug` (on `run`, `dev` or `build`) to see the runner's structured logs. `evalkit <command> --help` lists every flag. `--concurrency` bounds cell parallelism. In CI, add `--junit <file>` (one test case per scenario and matrix cell) and gate with `bun run evalkit compare --base <branch> --fail-on regression` (or `--base-dir <results>`; `--markdown <file>` for an MR comment, `--fail-on score-drop=0.05`). `run` exits 1 on any failure or a selected file that failed to load (`--skip-broken` runs the rest), 130 when cancelled with Ctrl+C; cancelled runs are never failures. Datasets' digests are recorded per trial; `compare` lists changed datasets. For baseline/treatment studies, inspect a `--dry-run` first, keep model/task/prompt constant, and identify whether the treatment is injected context or live tool retrieval.
|
|
11
11
|
2. Each `_evalkit-results/<run-uuid>/` has `manifest.json`, `summary.json`, and `trials/<trial-uuid>/{manifest.json,trajectory.jsonl,scoring.json,summary.json,artifacts/candidate/...}`. `trajectory.jsonl` contains one JSON event per line; filter by `source` and `kind`. `scoring.json` records scorer values/explanations/errors; candidate snapshot files are report artifacts.
|
|
12
12
|
3. Read run `summary.json` fields `passed`, `failed`, `trialCount` for success rates. Trial `status: 'completed'` means execution finished, **not** that its scorers passed. `scoring.overall` is an unweighted mean of valid values, not a pass rate. For CI, use `jq -e '.failed == 0 and .trialCount > 0' "_evalkit-results/<run-uuid>/summary.json"` after replacing the placeholder; CLI exits nonzero when a matrix cell fails; the report supplies trial-level detail.
|
|
13
13
|
4. `--json` emits one JSON object per completed matrix cell with its key, eval ID, effective parameters, and run result. Read the saved trial trajectory for adapter-emitted turn latency or token usage; EvalKit does not infer provider cost, latency, or token counts. Compare matrix conditions at both cell and trial level and avoid claiming significance from a small pilot.
|