@leostera/evalkit 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -1
- package/dist/cli.mjs +182 -73
- package/dist/dashboard/assets/index-CFJzcw3o.css +2 -0
- package/dist/dashboard/assets/{index-DNR4shrM.js → index-DgoX2x_M.js} +34 -32
- package/dist/dashboard/index.html +2 -2
- package/dist/{index-db-B1FeXzmS.mjs → index-db-DI7xKMSX.mjs} +237 -47
- package/dist/index.d.mts +837 -220
- package/dist/index.d.mts.map +1 -1
- package/dist/index.mjs +241 -52
- package/dist/runner.d.mts +568 -227
- package/dist/runner.d.mts.map +1 -1
- package/dist/runner.mjs +2 -2
- package/dist/share-worker.d.mts +1 -1
- package/dist/share-worker.d.mts.map +1 -1
- package/dist/share-worker.mjs +1 -1
- package/dist/share.d.mts +9 -6
- package/dist/share.d.mts.map +1 -1
- package/dist/share.mjs +2 -2
- package/dist/skills/evalkit/SKILL.md +2 -2
- package/dist/skills/evalkit-agents/SKILL.md +3 -3
- package/dist/skills/evalkit-datasets/SKILL.md +1 -1
- package/dist/skills/evalkit-evals/SKILL.md +1 -1
- package/dist/skills/evalkit-judges/SKILL.md +1 -1
- package/dist/skills/evalkit-results/SKILL.md +1 -1
- package/dist/{src-Bccv9igQ.mjs → src-CXzbHAKN.mjs} +1 -1
- package/dist/{src-B14fXAGp.mjs → src-D6Ax2t9b.mjs} +379 -206
- package/dist/src-Dc3GXacB.mjs +3637 -0
- package/dist/upgrade-rMsGzUEJ.mjs +230 -0
- package/dist/upgrade-transcript-C8DUykLd.mjs +402 -0
- package/package.json +2 -1
- package/dist/dashboard/assets/index-TPYprg75.css +0 -2
- package/dist/src-BfSQKOJP.mjs +0 -1877
- package/dist/upgrade-G-uFPJKg.mjs +0 -111
package/dist/runner.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.mts","names":["_$_evalkit_core0.JsonValue","_$_evalkit_core0.
|
|
1
|
+
{"version":3,"file":"runner.d.mts","names":["TreeRecord","Transcript","_$_evalkit_core0.JsonValue","Transcript","_$_evalkit_core0.RunEvent"],"sources":["../../core/src/identity.ts","../../core/src/dataset.ts","../../core/src/usage.ts","../../core/src/agent-transcript.ts","../../core/src/isolation.ts","../../core/src/limits.ts","../../core/src/transcript.ts","../../core/src/judge.ts","../../core/src/catalog.ts","../../core/src/matrix.ts","../../core/src/config.ts","../../core/src/index.ts","../../runner/src/errors.ts","../../runner/src/shell/fixtures.ts","../../runner/src/shell/container-shell.ts","../../runner/src/local-report-store.ts","../../runner/src/datasets.ts","../../runner/src/local-report-reader.ts","../../runner/src/matrix.ts","../../runner/src/upgrade.ts","../../runner/src/workspace.ts","../../runner/src/index.ts"],"mappings":";;;;cAEa,oBAAkB,OAAA;KACnB,sBAAsB,mBAAmB;KAiBzC,YAAY,cAAc,eAAe,2BACxC;;;;KCdD;;KAGA,6BAA6B;EAAe;;;;;;KAM5C,wBAAwB,SAAS,eAAe;;KAGhD;EACV,QAAQ;;EAER;;KAGU;;EAEV;EACA,QAAQ;;KAGE;EACV;EACA;;EAEA,QAAQ;;EAER;;EAEA,YAAY;EACZ,QAAQ;;KAoCE;EAEN;EACA,QAAQ;EACR;EACA;;EAGA;EACA,SACE,MAAM,uBACH,iBAAiB,QAAQ;EAC9B,SACE,QAAQ,gBACR,KAAK,sBACF,iBAAiB,QAAQ;;;KAIxB;WACD;WACA;WACA;WACA,YAAY;;WAEZ;WACA,QAAQ;;;;;;;;;;;;KC1FP;EACV;EACA;EACA;EACA;EACA;;EAEA;;;;KCRG,iBAAiB,GAAG,UAAU,eAAe,oBAC9C,KAAK,GAAG;;KAIA,cAAc,iBACxB;EAEI;EAAa;EAAe;;;KAGtB;;EAEV,kBAAkB;;EAElB,iBAAiB;;EAEjB,QAAQ;EACR,OAAO;EACP;IAAU;IAAiB;;;;;;;;;KASjB;;EAEV,KAAK,kBAAkB,aAAa;IAAY,SAAS;;EACzD,UAAU,SAAS;;;;;;EAMnB,WACE,gBACA,iBACA;IAAY;IAAe;;;EAG7B,OAAO,QAAQ;;;;;;;;EAQf,OAAO,YAAY;;;;;EAKnB,KACE,wBACA;IACE;IACA;IACA;MAAU;MAAiB;;;;;;;;;;EAU/B,SAAS,YAAY;;KAGX;EACV;EACA;;EAEA;EACA;;;KAIU;;EAIN;EACA,qBAAqB;EACrB,QAAQ;;EAER,iBAAiB;;;;;;;;;;KC/FX;EAAgB;;;;;;KAMhB;;KAGA;EAAmB;EAAc;;;KAGjC;EACV;;EAEA;EACA,SAAS;;EAET;;EAEA;EACA,iBAAiB;;;KAIP;EACV;;EAEA;;EAEA;;EAEA;;;;;;EAMA;;KAUU,YAAY,cAAc,qBAAqB;;;;;;KAsG/C;;EAEV;;EAEA;;EAEA;;EAEA;;;KAIU;;EAEV;;;KAIU;EACV;;EAEA;;EAEA;;KAGU,cAAc;EAAc;EAAgB;;KAE5C;;EAEV;;EAEA,MAAM,SAAS;;EAEf,SAAS;;EAET;;EAEA;;;KAIU;;WAED;;EAET;EACA;;EAEA,SACE,WAAW,eAAe;;EAG5B,QAAQ,QAAQ;;EAEhB,KAAK,kBAAkB;;;;;;;;KASb;WACD,WAAW;;WAEX;;;;;;WAMA;EACT,KACE,qCACA,UAAU,eACT,QAAQ;EACX,MACE,qCACA,UAAU,KAAK,6BACd;;EAEH,SAAS,eAAe;EACxB,UAAU,cAAc,kBAAkB,aAAa;;EAEvD,KAAK,gBAAgB;;;;;KClPX;;EAEV;;EAEA;;EAEA;;KAGU,kBAAkB;;KAKlB;EACV;EACA;;EAEA;;;KAIU;EAAiB,OAAO;EAAW;EAAe;;;;KCyBlD,mBAAmB;EAAkB;;;;;;;;;;;KCvCrC;EACN;;EACA;EAAiB,SAAS;;EAC1B;EAAkB,UAAU;;;KAGtB;EAAgB;EAAa;EAAe;;;KAG5C;EACV;EACA;EACA;;;;;;;;;KAuFU,kBAAkB,mBAAmB;KAErC;EACV;;EAEA;;EAEA;;EAEA;;EAEA,WAAW;;KAGD;EACV;EACA;EACA;EACA;;;KAIU,kBACP,KAAK;EAAkC;KACxC;;;KCrGQ;EACV;EACA;EACA;EACA;EACA,UAAU;IAAQ,MAAM;IAAkB;;;EAE1C;;;KAIU;EACN;EAAe;;EACf;EAAgB;;EAChB;;KAEM;EACV;EACA;EACA,YAAY;EACZ;EACA;;EAEA;IAAU;;IAAwB;;;EAElC;IAAU,OAAO;;IAAiB;;;EAElC;IAAU;IAAc;;EACxB;;KAGU;EACV;EACA,MAAM;;EAEN;EACA;EACA;;EAEA;;EAEA,QAAQ;;EAER;;EAEA,SAAS;;EAET;EACA;;KAGU;;;EAEN;EAAc;;EACd;EAAe,aAAa;;EAC5B;EAA0B;EAAc,YAAY;;;;EAEpD;EAAoB,QAAQ;;KAEtB;EACV;EACA;EACA;EACA;EACA,OAAO;;EAEP,aAAa;EACb,UAAU;;EAEV;EACA;IAAU;;;EAEV,SAAS;EACT,YAAY;EACZ,SAAS;EACT,WAAW;;EAEX;;;;KCjGU;;EAEV;;EAEA,aAAa,SAAS,wBAAwB;;EAE9C,YAAY;;KAEF;EACV;;EAEA;EACA,MAAM;EACN,YAAY;;KAEF;EACV;EACA;EACA,gBAAgB;EAChB,YAAY,SAAS,wBAAwB;EAC7C,WAAW;;EAEX,iBAAiB;;EAEjB,mBAAmB;;;;;EAKnB,SAAS;IACP;IACA;IACA;;;KAGQ,aAAa;EACvB;EACA,MAAM,YAAY;EAClB,MAAM,YAAY,kBAAkB,iBAAiB;;;;KCmB3C;KAEA;;EAEV;;EAEA;;;;KCVU;KACA,YACV,gBAAgB;GAAiB,cAAc;;KACrC;GAAgB,cAAc;;KAE9B;;EAEV;EACA;EACA;EACA;;KAGU;;;;;KAMA;EACV;EACA,gBAAgB;;KAGN,gBAAgB,QAAQ,OAAO,kBAAkB;KAKjD;EACV;EACA;EACA;;KAGU;EACN;EAAqB;;EAErB;EACA;;EAEA;EACA;;EAEA;EAAmC;EAAc;;EACjD;EAAiC;EAAc;;EAE/C;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAwB;EAAgB;;EAExC;EACA;;EAEA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEC;EAAuB;IAAsB;EAC9C;EAAuB;;EACvB;EAAe,OAAO;EAAe;;KAmB/B,WAAW;EAAgB;;KAE3B;;EAEV;;KAGU,qBAAqB;KACrB,qBAAqB;KAErB;EACV;EACA;;EAEA,QAAQ;EACR,UAAU;;EAEV,aAAa;EACb,UAAU;;EAEV,MAAM;;;;;EAKN,QAAQ;;EAER,QAAQ;;;;;EAKR,OAAO;;EAEP,UAAU,SAAS,eAAe;;EAElC,WAAW;;EAEX,oBAAoB;;KAGV,WAAW;WACZ,WAAW;;WAEX,WAAW;EACpB,MAAM;IACJ,SAAS;IACT,UAAU;;;;;IAKV,YAAY;;;;;;;;IAQZ,SAAS,cAAc,kBAAkB,eAAe;MACtD,QAAQ,WAAW;;KAGb,WAAW;;;;;EAKrB,KAAK,SAAS,WAAW,eAAe;;EAExC,UAAU;;;KAyBA;;EAEV,YAAY;;EAEZ,MAAM;;EAEN;;EAEA,QAAQ;;KAGE;EACV;;EAEA,oBAAoB,OAAO;;KAGjB;EACV;EACA,aAAa;;;KAIH;EACV;;EAEA,kBAAkBA;EAClB,4BAA4B;EAC5B;;IAEE;IACA;IACA;IACA,WAAW;IACX;IACA,SAAS;;EAEX;;KAGU;EACV;EACA;EACA;IAAY;IAAc,YAAY;;;KAE5B,iBACV,WAAW,YAAY,cAAc;;;;;;KA4B3B;EACV,QAAQ;;EAER;;EAEA,QAAQ;;EAER;;EAEA,QAAQ;;EAER;;EAEA,aAAaC;;;KAIH;EAAiB;EAAe;;KAEhC,WAAW,+BAEnB;;EAGE;EACA;EACA;;;;;EAKA,WAAW;;;;;EAKX,WAAW;;EAEX,QAAQ;;KAYF;EACV;EACA;EACA;EACA;;EAEA;EACA;EACA,WAAW;;EAEX;EACA,WAAW;;EAEX;EACA,QAAQ;EACR;EACA,QAAQ;;EAER;;KAGU;EACV,WAAW;EACX,WAAW;;KAGD;EACV,SAAS;;;;;;EAMT,YAAYA;;EAEZ,WAAW,SAAS,eAAeA;;EAEnC,oBAAoB;EACpB,WAAW;;EAEX,OAAO;;EAEP;;EAEA,MAAM;;;;;;EAMN,sBAAsB;;;;;;;EAOtB,WAAW,UACT,MAAM,gBAAgB,YACrB,kBAAkB;EACrB,WACE,MAAM,YAAY,qBACjB,kBAAkB;;;;;;EAMrB,OAAO,cAAc,kBAAkB,aAAa;;;;;;;KAQ1C,kBAAkB,YAAY,KAAK;EAC7C,WAAW;;KAGD,gBAAgB;EAC1B;EACA;;EAEA;EACA;;EAEA;;;;;;EAMA,WAAW;EACX,IACE,SAAS,2BACE,WAAW,YAAY,kBAAkB,WAAW;;;KAIvD,cACV,SAAS,mBACN,wBAAwB,QAAQ;;;;;;;KAQzB;EACV;EACA;EACA;;EAEA;;;;;;EAMA,QAAQ;;;;;EAKR,SAAS,SAAS,eAAe;;EAEjC,QAAQ;;EAER;;EAEA;;KAGU,cAAc,kBAAkB;;;;;;KA6ChC;EACV;EACA;;EAEA;;EAEA;EACA,IACE,SAAS,sCACa;;;KAcd;EACV,MAAM;;EAEN;EACA;EACA;;EAEA;EACA,QAAQ;;KAyCE;;EAEV;;KAGU,eAAe,eAAe,aAAa;EACrD;EACA;EACA,OAAO;;EAEP,oBAAoB;EACpB,YAAY;EACZ,SAAS;;EAET,mBAAmB;EACnB,SAAS;;;;;EAKT;;;;;EAKA,SAAS;;EAET,WAAW;EACX,YAAY;;;KAIF;;;;;;;;;EASV;;KAqHU;EACV;EACA;EACA;;KAGU;EACV;EACA;EACA;;;;;;;;KAsFU;;KAIA;EACV;;EAEA;EACA;;;;;;KAOU;;EAEV;;EAEA;;EAEA;EACA;EACA;;EAEA;EACA;IAAO;IAAkB;IAAmB;IAAc;;;;KAIhD;EACV,SAAS;EACT,OAAO;EACP,QAAQ;;;KAIE;EAAe;EAAa;;KAE5B;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,SAAS;EACT,UAAU;;EAEV,QAAQ;;EAER;;EAEA,WAAW;;EAEX,SAAS;;EAET,YAAY;;;KAIF;EACN;EAAmB;EAAe,SAAS;;EAC3C;;KAEM;EACV;EACA,YAAY;EACZ;;EAEA;;KAGU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX;EACA,WAAW;;EAEX;EACA,QAAQ;EACR;EACA,QAAQ;;EAER;IAAoB;IAAmB;;;;;;;;;;;;;;;;;cAiB5B;KAUD,4BAA4B;;KAE5B,aAAa;KAEb;EACV,SAAS;EACT,UAAU;EACV,cAAc;EACd;EACA;EACA;;KAGU;;EAEV;EACA;EACA;;;KAIU;EACV;;EAEA;;EAEA;;KAGU;EACV,QAAQ;EACR;EACA;EACA,UAAU;EACV,YAAY;EACZ,cAAc;EACd,QAAQ;;EAER,QAAQ;;KAGE;WACD;;EAET,YAAY,OAAO,WAAW;;EAE9B,iBACE,MAAMA,yBAAuBA,kCAC5B;;EAEH,cAAc,YAAY,eAAe;EACzC,YAAY,SAAS,aAAa;;EAElC,cAAc,cAAc,MAAM,aAAa;EAC/C,SAAS,SAAS,aAAa;;KAGrB;EACV,SACE,UAAU,aACV;IAAY,UAAU;MACrB,QAAQ;;;KAID;EACV;EACA,QAAQ;EACR;EACA,QAAQ;EACR,QAAQ;EACR;EACA;EACA,UAAU;EACV,QAAQ;;EAER,QAAQ;;;;;;;qBC5jCG,0BAA0B;EACrC;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBAGW,oBAAoB;EAC/B;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBASW,iCAAiC;EAG5C;EACA;EACA;EACA;;;;;;qBAIW,uBAAuB;EAClC;;;;KCtBU;;EAEV,UAAU;;WAED;;EAET,UAAU,WAAW,YAAY,eAAe;;EAEhD,QAAQ,kBAAkB;;EAE1B,QAAQ;IAAQ;IAAc;;;EAE9B,QAAQ;;;;;;wBAuDM,WACd,UAAU,SAAS,eAAe,WAClC;EAAW;IACV;;;;wBCrCmB,mBACpB,eACA,mBACC;;;;wBC8Ea,iBAAiB,wBAAwB;;;;KCoI7C;EAAqB;EAAc;;;;;;;wBAOzB,mBACpB,SAAS,SACT,eACC,QAAQ;;wBA6BW,kBACpB,SAAS,SACT,cACA,eACC,QAAQ,WAAW;;;qBCtRT,kBAAe,cAAgB,kBAAe;;;;;;0BAA9CC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAEA,iBAAc,cAAgB,kBAAe;;;;;;;;;;;;;4BAA7CA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAEA,iBAAc,cAAgB,kBAAe;;;;;;;;;0BAA7CA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;wBAGS,cAAc,cAAc,gBAAa,QAAzCE;;wBAOA,iBACpB,cACA,gBACC,QAAQ,eAAeD;;;;;wBAsBJ,kBACpB,cACA,gBACC,QAAQA;;;;;wBASK,YAAY,OAAO;;;;;;;KClEvB;EACV;EACA;EACA;EACA;EACA;EACA;;;wBAIc,UACd,QAAQ,YACR,SAAS,KAAK;EACZ,YAAY;;EAEZ,YAAY,MAAM;EAClB,YACE,MAAM,gBACN,eAAe,uBACL;IAEb,OAAO,OAAO;;;;KCCL;;EAEV;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA,QAAQ;IAAQ;IAAe;;;;;;;;;;;;;;;;;wBAuBX,eACpB,cACA;EAAW;IACV,QAAQ;;;;wBCzDK,iBAAiB,cAAc;;;YCiFnC;EACV,QAAQ;;EAER;EACA;;EAEA;;EAEA,aAAa;;EAEb,YAAY;;EAEZ,WAAW;;EAEX,QAAQ;;EAER,SAAS;;EAET,UAAU;;EAEV;;EAEA,YAAY,OAAO;;EAEnB;;EAEA,iBAAiB;;;;;EAKjB;;EAEA,mBAAmB;;EAEnB,SAAS;;EAET,UAAU;;EAEV;;;;;EAKA,SAAS;;;;;;EAMT;IAAW,OAAO;IAAQ,WAAW;;;;;;EAKrC,YAAY,KAAK,QAAQ,QAAQ;EACjC,YAAY;;;YAIF;EACR;EAAgB,QAAQ;;EAAkB;EAAe,OAAO;;;YAGxD;EACV;EACA,QAAQ;EACR;EACA;EACA,aAAa;;;;;;;wBAUC,QACd,YAAY,gBACZ,SAAS,iBACR,OAAO,OAAO"}
|
package/dist/runner.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { _ as
|
|
2
|
-
export { AutExecutionError, CancelledError, CheckpointExecutionError, DatasetError, ReportError, ScoringError, localReportStore, ownerIsGone, readRunEvents, readRunManifest, readRunScoring, readRunSummary, runEval, runMatrix, runWorkspacePath, staticDatasetFile, staticDatasetFiles, upgradeReports };
|
|
1
|
+
import { C as ScoringError, S as ReportError, _ as removeRunContainer, a as readRunEvents, b as CheckpointExecutionError, c as readRunSubagents, d as localReportStore, f as runWorkspacePath, g as fixtureSet, i as ownerIsGone, l as readRunSummary, m as staticDatasetFiles, n as upgradeReports, o as readRunManifest, p as staticDatasetFile, r as runMatrix, s as readRunScoring, t as runEval, u as readRunTranscript, v as AutExecutionError, x as DatasetError, y as CancelledError } from "./src-Dc3GXacB.mjs";
|
|
2
|
+
export { AutExecutionError, CancelledError, CheckpointExecutionError, DatasetError, ReportError, ScoringError, fixtureSet, localReportStore, ownerIsGone, readRunEvents, readRunManifest, readRunScoring, readRunSubagents, readRunSummary, readRunTranscript, removeRunContainer, runEval, runMatrix, runWorkspacePath, staticDatasetFile, staticDatasetFiles, upgradeReports };
|
package/dist/share-worker.d.mts
CHANGED
|
@@ -20,7 +20,7 @@ type R2BucketLike = {
|
|
|
20
20
|
//#endregion
|
|
21
21
|
//#region ../share/src/upload.d.ts
|
|
22
22
|
/** Finished runs only: a shared run's files are immutable once stored. */
|
|
23
|
-
type SharedRunStatus = 'passed' | 'failed' | 'errored';
|
|
23
|
+
type SharedRunStatus = 'passed' | 'failed' | 'errored' | 'limited';
|
|
24
24
|
type SharedSource = {
|
|
25
25
|
repo: string;
|
|
26
26
|
project: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"share-worker.d.mts","names":[],"sources":["../../share/src/files.ts","../../share/src/upload.ts","../../share/src/index-db.ts","../../share/src/index-do.ts","../../share/src/worker.ts"],"mappings":";;KAUY;EACV,IAAI,cAAc;IAAU,MAAM,eAAe;;EACjD,IAAI,aAAa,OAAO,sBAAsB;EAC9C,OAAO,cAAc;EACrB,KAAK;IAAW;IAAgB;MAAoB;IAClD;MAAW;;IACX;IACA;;;;;;
|
|
1
|
+
{"version":3,"file":"share-worker.d.mts","names":[],"sources":["../../share/src/files.ts","../../share/src/upload.ts","../../share/src/index-db.ts","../../share/src/index-do.ts","../../share/src/worker.ts"],"mappings":";;KAUY;EACV,IAAI,cAAc;IAAU,MAAM,eAAe;;EACjD,IAAI,aAAa,OAAO,sBAAsB;EAC9C,OAAO,cAAc;EACrB,KAAK;IAAW;IAAgB;MAAoB;IAClD;MAAW;;IACX;IACA;;;;;;KCGQ;KAEA;EACV;EACA;EACA;EACA;EACA;EACA;;;KAIU;EACV;EACA;EACA;EACA,QAAQ;EACR;EACA;EACA,QAAQ;;EAER;;EAEA;;;;KC5BU;KAEA;EACV;EACA;EACA;EACA;EACA;EACA;;KAGU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;;;KAIU;EACV,OAAO,gBAAgB;EACvB,UAAU,KAAK,YAAY;;EAE3B,UAAU,gBAAgB;EAC1B,SAAS,QAAQ;EACjB,QAAQ,eAAe,QAAQ;EAC/B,KAAK,cAAc,cAAc;;EAEjC,QAAQ,cAAc,cAAc;EACpC,OAAO,eAAe;;;;;KC5CnB;EACH;IACE;MACE,KAAK,kBAAkB,QAAQ;QAAe;;;IAEhD,gBAAgB,GAAG,YAAY,IAAI;;;KAIlC;EACH,WAAW;EACX,IAAI;IAAgB,MAAM,SAAS,UAAU,QAAQ;;;;;;;;cAoB1C;mBACM;EAEjB,YAAY,OAAO;EAYb,MAAM,SAAS,UAAU,QAAQ;;;iBAezB,mBACd,WAAW,6BACV;;;;;;;;;;KCrDS;EACV,SAAS;EACT,OAAO,kBAAkB;EACzB;IAAU,MAAM,SAAS,UAAU,QAAQ;;EAC3C;EACA;;;EAIA,MAAK,SAAU,SAAO,KAAO,WAAW,QAAQ,YAAY"}
|
package/dist/share-worker.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { r as createShareApp, t as ShareIndexDb } from "./index-db-
|
|
1
|
+
import { r as createShareApp, t as ShareIndexDb } from "./index-db-DI7xKMSX.mjs";
|
|
2
2
|
import { t as r2Files } from "./files-DkDQs2JL.mjs";
|
|
3
3
|
//#region ../share/src/index-do.ts
|
|
4
4
|
const METHODS = [
|
package/dist/share.d.mts
CHANGED
|
@@ -49,7 +49,7 @@ export declare function decodeBundle(body: Uint8Array, limits: {
|
|
|
49
49
|
//#endregion
|
|
50
50
|
//#region ../share/src/upload.d.ts
|
|
51
51
|
/** Finished runs only: a shared run's files are immutable once stored. */
|
|
52
|
-
type SharedRunStatus = 'passed' | 'failed' | 'errored';
|
|
52
|
+
type SharedRunStatus = 'passed' | 'failed' | 'errored' | 'limited';
|
|
53
53
|
type SharedSource = {
|
|
54
54
|
repo: string;
|
|
55
55
|
project: string;
|
|
@@ -176,6 +176,8 @@ type ShareEvent = {
|
|
|
176
176
|
event: 'share.upgrade';
|
|
177
177
|
upgraded: number;
|
|
178
178
|
runs: number;
|
|
179
|
+
transcripts: number;
|
|
180
|
+
failed: number;
|
|
179
181
|
remaining: number;
|
|
180
182
|
} | {
|
|
181
183
|
event: 'share.error';
|
|
@@ -212,7 +214,7 @@ export declare function normalizeBasePath(basePath: string | undefined): string;
|
|
|
212
214
|
*
|
|
213
215
|
* POST /api/v1/uploads one run bundle
|
|
214
216
|
* POST /api/v1/reindex rebuild the index from stored run.json files
|
|
215
|
-
* POST /api/v1/upgrade convert stored runs from
|
|
217
|
+
* POST /api/v1/upgrade convert stored runs from older EvalKit versions (RFD0008, RFD0009)
|
|
216
218
|
* GET /api/v1/repos indexed repositories
|
|
217
219
|
* GET /api/v1/repos/<repo>/-/commits commits, newest first
|
|
218
220
|
* GET /api/v1/repos/<repo>/-/<sha>/data/… snapshot data
|
|
@@ -282,11 +284,12 @@ export declare function createDirectoryShareApp(options: {
|
|
|
282
284
|
//#endregion
|
|
283
285
|
//#region ../share/src/paths.d.ts
|
|
284
286
|
/**
|
|
285
|
-
* The only files a run bundle may contain (snapshot format version
|
|
286
|
-
* `evalkit
|
|
287
|
-
* attachments. Never evaluator workspaces.
|
|
287
|
+
* The only files a run bundle may contain (snapshot format version 3, from
|
|
288
|
+
* `evalkit share`, for one run): its JSON, candidate workspace files and
|
|
289
|
+
* attachments. Never evaluator workspaces. Version 2 (EvalKit 0.9 and
|
|
290
|
+
* 0.10) has `events.json` where version 3 has `transcript.json`.
|
|
288
291
|
*/
|
|
289
|
-
export declare function allowedRunPath(path: string): boolean;
|
|
292
|
+
export declare function allowedRunPath(path: string, version?: 2 | 3): boolean;
|
|
290
293
|
/**
|
|
291
294
|
* Repository key: normalized host/path, plus `/~<project>` for an eval
|
|
292
295
|
* project below the repository root. Used in storage keys and URLs.
|
package/dist/share.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"share.d.mts","names":[],"sources":["../../share/src/files.ts","../../share/src/bundle.ts","../../share/src/upload.ts","../../share/src/index-db.ts","../../share/src/app.ts","../../share/src/fs-files.ts","../../share/src/html.ts","../../share/src/index-sqlite.ts","../../share/src/node.ts","../../share/src/paths.ts"],"mappings":";;KACY;EACV,IAAI,cAAc,QAAQ,eAAe,cAAc;EACvD,IAAI,aAAa,MAAM,sBAAsB;;EAE7C,KAAK,iBAAiB;EACtB,OAAO,cAAc;;;KAIX;EACV,IAAI,cAAc;IAAU,MAAM,eAAe;;EACjD,IAAI,aAAa,OAAO,sBAAsB;EAC9C,OAAO,cAAc;EACrB,KAAK;IAAW;IAAgB;MAAoB;IAClD;MAAW;;IACX;IACA;;;wBAIY,QAAQ,QAAQ,eAAe;;;;;;;;;;;qBCblC;KAGD;EAAe;EAAc,MAAM;;wBAUzB,aAAa,OAAO,eAAe,QAAQ;;wBA2B3C,aACpB,MAAM,YACN;EAAU;IACT,QAAQ;;;;
|
|
1
|
+
{"version":3,"file":"share.d.mts","names":[],"sources":["../../share/src/files.ts","../../share/src/bundle.ts","../../share/src/upload.ts","../../share/src/index-db.ts","../../share/src/app.ts","../../share/src/fs-files.ts","../../share/src/html.ts","../../share/src/index-sqlite.ts","../../share/src/node.ts","../../share/src/paths.ts"],"mappings":";;KACY;EACV,IAAI,cAAc,QAAQ,eAAe,cAAc;EACvD,IAAI,aAAa,MAAM,sBAAsB;;EAE7C,KAAK,iBAAiB;EACtB,OAAO,cAAc;;;KAIX;EACV,IAAI,cAAc;IAAU,MAAM,eAAe;;EACjD,IAAI,aAAa,OAAO,sBAAsB;EAC9C,OAAO,cAAc;EACrB,KAAK;IAAW;IAAgB;MAAoB;IAClD;MAAW;;IACX;IACA;;;wBAIY,QAAQ,QAAQ,eAAe;;;;;;;;;;;qBCblC;KAGD;EAAe;EAAc,MAAM;;wBAUzB,aAAa,OAAO,eAAe,QAAQ;;wBA2B3C,aACpB,MAAM,YACN;EAAU;IACT,QAAQ;;;;KC/BC;KAEA;EACV;EACA;EACA;EACA;EACA;EACA;;;KAIU;EACV;EACA;EACA;EACA,QAAQ;EACR;EACA;EACA,QAAQ;;EAER;;EAEA;;;KAUU;EACV,KAAK;EACL,SAAS;EACT,UAAU;;EAEV,OAAO;;;;;;;;wBASO,YAAY,iBAAiB,eAAe;;;;;;;;;;KCzDhD;EACV,IAAI,GAAG,kBAAkB,QAAQ,aAAa;EAC9C,IAAI,kBAAkB,QAAQ;EAC9B,YAAY,GAAG,YAAY,IAAI;;KAErB;KAEA;EACV;EACA;EACA;EACA;EACA;EACA;;KAGU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;;;KAIU;EACV,OAAO,gBAAgB;EACvB,UAAU,KAAK,YAAY;;EAE3B,UAAU,gBAAgB;EAC1B,SAAS,QAAQ;EACjB,QAAQ,eAAe,QAAQ;EAC/B,KAAK,cAAc,cAAc;;EAEjC,QAAQ,cAAc,cAAc;EACpC,OAAO,eAAe;;qBA+CX;mBAEQ;mBACA;EAFnB,YACmB,KAAK,KACL,YAAW;EAK9B,OAAO;;EAKP,UAAU,KAAK;EAwDf,UAAU;EAuBV,SAAS;EAST,QAAQ,eAAe;EAevB,KAAK,cAAc;EAUnB,QAAQ,cAAc;EAiBtB,OAAO;;;wBASO,WAAW,IAAI,eAAe;;;;KC9NlC;EACV,SAAS;EACT,IAAI,eAAe,QAAQ;;;KAIjB;EAEN;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;EAAuB;EAAqB;;EAC5C;EAAwB;EAAc;;EAEtC;EACA;EACA;EACA;EACA;EACA;;EAEA;EAAsB;EAAgB;EAAc;;KAE9C;EACV,OAAO;EACP,OAAO;EACP,QAAQ;;EAER;;EAEA,OAAO,OAAO;;;;;EAKd;;;;;;;EAOA;;;wBAIc,kBAAkB;;;;;;;;;;;;;;;;;;;;;wBA2ClB,eAAe,SAAS,2CAAe,8BAAA,+BAAA;;;;wBC1GvC,eAAe,eAAe;;;;;;;;KCPlC;;EAEV;;EAEA;;EAEA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;wBAGc,eACd,cACA;EAAW;EAAc;EAAiB,QAAQ;;;;;;;;;wBCX9B,YAAY,eAAe,QAAQ;;;;wBCAzC,gBAAgB,eAAe;;wBAsBzB,wBAAwB;EAC5C;EACA;EACA;EACA;IACD,iCAAA,8BAAA,+BAAA;;;;;;;;;wBCZe,eAAe,cAAc;;;;;wBA2B7B,QAAQ;EAAU;EAAc;;wBAIhC,aAAa"}
|
package/dist/share.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { a as parseUpload, c as validRepoKey, d as decodeBundle, f as encodeBundle, i as normalizeBasePath, l as dashboardIndex, n as asyncIndex, o as allowedRunPath, r as createShareApp, s as repoKey, t as ShareIndexDb, u as BUNDLE_FORMAT } from "./index-db-
|
|
1
|
+
import { a as parseUpload, c as validRepoKey, d as decodeBundle, f as encodeBundle, i as normalizeBasePath, l as dashboardIndex, n as asyncIndex, o as allowedRunPath, r as createShareApp, s as repoKey, t as ShareIndexDb, u as BUNDLE_FORMAT } from "./index-db-DI7xKMSX.mjs";
|
|
2
2
|
import { t as r2Files } from "./files-DkDQs2JL.mjs";
|
|
3
|
-
import { i as directoryFiles, n as directoryAssets, r as sqliteIndex, t as createDirectoryShareApp } from "./src-
|
|
3
|
+
import { i as directoryFiles, n as directoryAssets, r as sqliteIndex, t as createDirectoryShareApp } from "./src-CXzbHAKN.mjs";
|
|
4
4
|
export { BUNDLE_FORMAT, ShareIndexDb, allowedRunPath, asyncIndex, createDirectoryShareApp, createShareApp, dashboardIndex, decodeBundle, directoryAssets, directoryFiles, encodeBundle, normalizeBasePath, parseUpload, r2Files, repoKey, sqliteIndex, validRepoKey };
|
|
@@ -26,7 +26,7 @@ For a cross-cutting change, use all relevant subskills; for a focused task, load
|
|
|
26
26
|
- A scenario is one concrete conversation in its own small file. Share agents (`agents/`), judges (`judges/`) and datasets (`datasets/`) by importing them; never generate scenarios in loops or from datasets.
|
|
27
27
|
- Authored suites, scenarios, datasets, agents, and matrices use stable lowercase kebab-case IDs. Only runs and runs receive generated runtime URIs.
|
|
28
28
|
- Keep evaluator datasets/answers private from the AUT and model unless a trusted adapter deliberately reads them and includes selected content in its prompt. Such inclusion is a deliberate treatment (for example, a memory-on condition), not a live retrieval test. Local sandboxes retain evaluator files; review reports and sandboxes before sharing.
|
|
29
|
-
- Executable transcript steps are `user(...)`, `predicate(...)`, `judge(...)`, and `expectToolCall(...)`. Final scoring supports both `predicate(...)` and `judge(...)`; `judge(...)` uses its own `agent` or the project's `judge` in `evalkit.config.ts`, never the AUT. `agent(...)` transcript steps are not executable. `agentsSdk()` is unimplemented, `judgeScorer(...)` is not part of the current API
|
|
30
|
-
-
|
|
29
|
+
- Executable transcript steps are `user(...)`, `predicate(...)`, `judge(...)`, and `expectToolCall(...)`. Final scoring supports both `predicate(...)` and `judge(...)`; `judge(...)` uses its own `agent` or the project's `judge` in `evalkit.config.ts`, never the AUT. `agent(...)` transcript steps are not executable. `agentsSdk()` is unimplemented, `judgeScorer(...)` is not part of the current API. Runs record ATXF transcripts and can be capped with `limits` (steps, tokens, time; `--limit` on the CLI). Verify current code/docs before proposing capabilities.
|
|
30
|
+
- Built-in harnesses `piAgent()`, `codexAgent()`, `opencodeAgent()` and `claudeCodeAgent()` run those coding agents with their own tools, one session per run, and import their native sessions as the transcript. They use a private, empty config by default (`configFiles`, `env`; or `config: 'user'`). For other agents, implement an adapter that records observed tool calls and results; do not claim a tool was used unless the transcript records it.
|
|
31
31
|
- For an intervention/A-B eval, hold task, model, and other settings constant; vary only the intervention parameter, use repeated runs and a negative control, and report exactly what was varied. A memory snapshot injected into the prompt tests memory-content usefulness, not retrieval quality. Start with a dry run and narrow smoke test; Pi-backed evaluations can incur provider costs.
|
|
32
32
|
- Run `bun run check` and relevant `bun test` suites when changing code. Check `git status --short` first and preserve unrelated changes.
|
|
@@ -9,10 +9,10 @@ Read `../evalkit/SKILL.md` for shared constraints and `../../../sites/evalkit.le
|
|
|
9
9
|
|
|
10
10
|
1. Identify the **Agent Under Test (AUT)**: the assistant, process, or service whose behavior the eval measures, not the scorer that grades it. Its adapter starts a fresh session per run, receives user messages, and emits the evidence scorers inspect. Decide whether to reuse `piAgent()` for the local Pi CLI or implement `defineAgent({ identity, start })` for a different agent (`runtimes` defaults to in-process local). Do not add new agent kinds; adapt the agent with `defineAgent`. `agentsSdk()` currently throws; do not claim it is a usable remote adapter.
|
|
11
11
|
2. Give the new agent a stable, unique lowercase kebab-case `identity.id`, meaningful `name`/`kind` and optional version. A declared runtime has `kind` and optional serializable `configuration`; the local CLI selects the first declared runtime among local/remote/sandbox, or lets an adapter without runtimes use its default. Only declare a runtime your adapter really supports.
|
|
12
|
-
3. In `start({ context, runtime,
|
|
13
|
-
4. Return a session whose `send(message)` resolves after the turn finishes and **returns the turn's result**: a reply string, or `{ reply, toolCalls: [{ name, arguments, result?, id? }], usage }`. EvalKit records it
|
|
12
|
+
3. In `start({ context, runtime, transcript, attach })`, initialize a **new run session**. Use `context.workspace.root` for candidate-visible data. Treat `context.evaluatorWorkspace.root` as trusted-only: never expose its path or all its contents to the AUT. `context.data[id]` holds candidate dataset values. A controlled experiment may deliberately read a specific evaluator dataset and pass only that treatment input to the model (for example, a `memory=on` arm); document that this tests injected context, not retrieval. Read and validate `context.parameters` if flags or matrix axes should affect agent behavior. Run tools and agent CLIs through `context.shell` (`exec`, `spawn`, `readFile`/`writeFile`/`list`), so they run where the project's `isolation` puts them (`none()` host default, `container({ image, network })`, `sandboxExec({ allow, read, deny, env })`); services from config `fixtures` are at `context.fixtures[name].url`. Pass `context.signal` to model calls and tool processes: it aborts when the run reaches a limit (`context.limits`: steps, tokens, time) or is cancelled.
|
|
13
|
+
4. Return a session whose `send(message)` resolves after the turn finishes and **returns the turn's result**: a reply string, or `{ reply, toolCalls: [{ name, arguments, result?, id? }], usage, model }`. EvalKit records it in the run's ATXF transcript. `close()` is optional. To record each model call as it happens, write to `transcript` instead: `transcript.assistant({ content, model, usage })` (content is text or ATXF blocks such as `tool_call`), `transcript.toolResult(callId, result, { name, isError })`, optional `transcript.turn('start' | 'end')`, `transcript.subagent(childTranscript)` for sub-agents. For agents ATXF converts (Pi, Claude Code, Codex, Gemini CLI, OpenCode), import the session after each turn: `transcript.import(pi.toAtxf(text))` from `@leostera/atxf/pi`. Usage is ATXF buckets: `{ input, output, cacheRead?, cacheWrite?, reasoning? }`, with cached tokens kept **out** of `input`. Plain values are converted to JSON once; no `JSON.parse(JSON.stringify(...))` casts. Propagate failures so the runner can record them.
|
|
14
14
|
5. Import the adapter from an eval; importing an agent alone does not make a run available to the CLI. Add or adapt a predicate; `output` holds the final reply. Test a single eval before a costly suite.
|
|
15
15
|
|
|
16
|
-
|
|
16
|
+
For a coding agent, use a built-in harness: `piAgent()`, `codexAgent()`, `opencodeAgent()` or `claudeCodeAgent()` run the CLI with its own tools, one session per run, through `context.shell` with the message on stdin, and import its native session through ATXF (tool calls, usage and steps included). They default to `config: 'private'`, an empty config directory in `context.shell.home`: supply provider settings with `configFiles` and keys with `env`, or `config: 'user'` for the person's own setup. `model` may be `({ model }) => String(model)` for a matrix axis. For another CLI use `harnessAgent(spec, options)`. For a custom agent, explicitly control its MCP/tool configuration and record real observed tool calls and results. Read and validate `context.parameters` when the model or treatment varies by matrix cell. Do not claim a memory snapshot ablation measures live memory retrieval.
|
|
17
17
|
|
|
18
18
|
To keep the agent's raw log (or any file) when a passing run's sandbox is removed, call `attach(name, content)` from `start()`'s options, usually in `close()`: `await attach('logs/agent.log', log)`. Names are relative paths of `[A-Za-z0-9._-]` segments; files over 5 MB keep their end; a run's attachments are capped at 20 MB (`artifacts.attachments` in the config). Scorers have the same `attach` on their context.
|
|
@@ -14,4 +14,4 @@ Read `../evalkit/SKILL.md` and `../../../sites/evalkit.leostera.dev/src/content/
|
|
|
14
14
|
5. Files land below `mount` (default: the dataset ID; `'.'` is the workspace root). Paths may not escape the mount; two datasets in one scenario may not write the same file. Each run gets fresh copies.
|
|
15
15
|
6. For a file-creating agent, inspect candidate files via `artifacts.candidate.root` in predicates and the report's `<run-id>/artifacts/candidate/`. Each run records a digest per dataset; `evalkit compare` lists datasets whose digest changed.
|
|
16
16
|
|
|
17
|
-
Datasets never template or fan out scenarios: write one concrete scenario file per case and share the dataset. When dataset setup fails, the run's `summary.json` and `
|
|
17
|
+
Datasets never template or fan out scenarios: write one concrete scenario file per case and share the dataset. When dataset setup fails, the run's `summary.json` and `run-events.jsonl` record the error naming the dataset. Test the smallest scenario from the project directory.
|
|
@@ -8,7 +8,7 @@ description: Define or modify EvalKit scenarios (evals) and suites. Use when add
|
|
|
8
8
|
Read `../evalkit/SKILL.md`, `../../../sites/evalkit.leostera.dev/src/content/docs/manual/project-structure.md` and `../../../sites/evalkit.leostera.dev/src/content/docs/manual/scoring-and-evals.md`, `../../../examples/starter/scenarios/greeting.eval.ts`, and `../../../examples/configured-matrix/evalkit.config.ts`.
|
|
9
9
|
|
|
10
10
|
1. Default-export one scenario per file from `scenarios/**/*.eval.ts` using `defineEval({ id: 'my-scenario', name, agent, transcript, scoring, datasets?, repeat?, policy? })`. The CLI discovers these files automatically; no registry or config is required. Choose a stable lowercase kebab-case ID. `metadata` is optional; `name` is a display label. Import shared agents, judges and datasets; keep each scenario short and concrete (a suite of a hundred cases is a hundred files), and never generate scenarios in a loop.
|
|
11
|
-
2. Executable transcript steps are `user('message')`, inline `predicate(...)`, `judge(...)`, and `expectToolCall(...)`. User messages call `session.send` in order on one fresh session per run; predicates/judges/tool-call assertions run at checkpoints. A `judge(...)` names its judge `agent`, or uses `judge` in `evalkit.config.ts`; it is never the AUT. `agent(...)` transcript steps are not executable. `user(({ parameters, data, seed, batch }) => ...)` computes a message per run, so one scenario can vary its prompt by matrix axis or candidate dataset instead of being copied per variant; `repeat: n` starts `n` independent runs as one batch (default 1, positive integer; `--repeat` and `execution.repeat` override it); `
|
|
11
|
+
2. Executable transcript steps are `user('message')`, inline `predicate(...)`, `judge(...)`, and `expectToolCall(...)`. User messages call `session.send` in order on one fresh session per run; predicates/judges/tool-call assertions run at checkpoints. A `judge(...)` names its judge `agent`, or uses `judge` in `evalkit.config.ts`; it is never the AUT. `agent(...)` transcript steps are not executable. `user(({ parameters, data, seed, batch }) => ...)` computes a message per run, so one scenario can vary its prompt by matrix axis or candidate dataset instead of being copied per variant; `repeat: n` starts `n` independent runs as one batch (default 1, positive integer; `--repeat` and `execution.repeat` override it); `limits: { steps?, tokens?, time? }` caps each run over the project's `limits` (a run past one ends `limited`, graded only by `supportsPartial` scorers).
|
|
12
12
|
3. Suites are directories: every file below `scenarios/billing/` is in suite `billing` (kebab-case directory names; deeper directories stay in the top one). Files directly in `scenarios/` have no suite; `discovery.suites: false` turns this off. Only use an explicit registry (`registry: registerEvals([defineSuite({ id, evals })])` in the config) when you need full control; it replaces discovery, and duplicate eval IDs throw.
|
|
13
13
|
4. For an intervention comparison, hold task/model/prompt constant and vary one explicit matrix axis (for example `memory: ['off', 'on']`). Repeat runs and include a negative control where possible. Record whether the treatment injects a saved memory or performs live retrieval: those answer different questions. A memory dataset can be evaluator-visible to the trusted adapter, but do not expose evaluator roots or reference answers to the AUT unless revealing the chosen treatment is intentional.
|
|
14
14
|
5. From the project directory run `bun run evalkit run <eval-id>`. Use `bun run evalkit run --suite <suite-id>` for a suite directory. IDs are exact human-defined strings. Inspect per-run `scoring.json` and the run `summary.json`; completed execution is not the same as passing scores.
|
|
@@ -9,7 +9,7 @@ Read `../evalkit/SKILL.md`, `../../../sites/evalkit.leostera.dev/src/content/doc
|
|
|
9
9
|
|
|
10
10
|
EvalKit currently executes deterministic `predicate(name, callback)` scorers and agent-backed `judge(name, { rubric })` rules. A judge rule runs on its own `agent` or the project's `judge` (`defineConfig({ judge })`); it is never the AUT. Give every rule a `description` saying what it checks: the dashboard, `--dry-run` and failure lines show it. The `judgeScorer(...)` API is not implemented—do not suggest it.
|
|
11
11
|
|
|
12
|
-
1. Add a named `predicate(...)` for deterministic checks or a `judge(...)` descriptor for rubric-based LLM assessment, then reference it from the eval's `scoring` array or supported transcript checkpoint. The predicate callback receives `{ context, output, data,
|
|
12
|
+
1. Add a named `predicate(...)` for deterministic checks or a `judge(...)` descriptor for rubric-based LLM assessment, then reference it from the eval's `scoring` array or supported transcript checkpoint. The predicate callback receives `{ context, output, data, transcript, subagents, runEvents, artifacts, checkpoints, turn? }`; `checkpoints` are earlier checkpoint results (all of them in final scoring), so a checkpoint that inspects live state before the session closes can hand its `evidence` to final scorers. Use `output` for the final assistant reply and `data[id]` for dataset values (expected answers belong in an evaluator dataset); for tool calls or intermediate messages use the helpers `toolCalls(ctx, name?)` (`{ name, arguments, result, isError }`), `messages(ctx, role?)` (`{ role, text }`), `lastReply(ctx)`, `usage(ctx)` and `modelContext(ctx)`; `transcript` is the raw ATXF transcript. `context` includes per-run IDs, metadata, random seed and optional parameters; workspace roots are absolute local paths.
|
|
13
13
|
2. A predicate returns a boolean, a finite score from 0 to 1, `{ value, passed?, explanation?, evidence? }` (evidence may be any plain value), or `skip(reason)` when the check does not apply. Without `passed`, only `value === 1` passes; give `predicate(name, fn, { passAt: 0.8 })` for a threshold. A skipped result neither passes nor fails and is left out of averages; never fake "not applicable" as a pass. Use clear evidence for debugging.
|
|
14
14
|
3. A `judge(...)` rule: EvalKit builds the prompt from `rubric`, `inputs` (`{ name: (ctx) => value }`, default `conversation` and `reply`) and `answer` (`choices({ A: [label, value] })`, `criteria({ name: weight })`, or the default 0–1 verdict), reads the reply, and asks once more if it cannot. Set `passAt` (default only 1). The judge agent receives `context.parameters`, so a matrix axis such as `judgeModel` can choose its model; it should return `{ reply, usage, model }`. Results save the prompt, reply, conversation, usage, model and `passAt`; judging tokens are the built-in `judge tokens` metric, apart from `tokens`. A predicate may return `criteria` instead of `value`; the score is their weighted mean. Validate that the judge is independent from the AUT, runs in an appropriate workspace, and does not receive secrets. Judge events/prompts may contain sensitive data and incur model cost; cost is not inferred unless the adapter emits usage.
|
|
15
15
|
4. Normal scorers are skipped after primary execution errors unless declared with `{ supportsPartial: true }` and the required context/workspace exists. Scorer exceptions and invalid scores become recorded failures; never mask them as passes.
|
|
@@ -8,7 +8,7 @@ description: Run Evalkit locally and interpret or process evaluation results. Us
|
|
|
8
8
|
Read `../evalkit/SKILL.md` and `../../../sites/evalkit.leostera.dev/src/content/docs/index.md` (First run) and `../../../sites/evalkit.leostera.dev/src/content/docs/manual/cli-and-matrices.md` and `../../../sites/evalkit.leostera.dev/src/content/docs/manual/results.md`. Verify report semantics in `../../../packages/runner/src/index.ts`, `../../../packages/runner/src/local-report-store.ts`, and `../../../packages/cli/src/index.ts` before automation.
|
|
9
9
|
|
|
10
10
|
1. Run from the project root containing default-exported `scenarios/**/*.eval.ts` (as in `examples/starter`, or use an optional config as in `examples/configured-matrix`), not this skill directory. Start with `bun run evalkit run <eval-id>`; run all evals only when intended (Pi-backed evals can cost tokens). `bun run evalkit run --suite <suite-id>`, `bun run evalkit run --matrix <matrix-id>`, and `bun run evalkit dev` are also available; `bun run evalkit build [--out dir]` writes a read-only static dashboard snapshot (reports plus candidate artifacts, never evaluator workspaces) that must still be reviewed before sharing; it asks to run evals first (always runs them when there are no results; `--run`/`--no-run` skip the prompt). `bun run evalkit tidy` archives runs of evals no longer in the project (marker `archived.json`, nothing deleted); archived runs are hidden in the dashboard and excluded from builds. Runs record `source` (repo, project, commit, ref, dirty) in `manifest.json`; `bun run evalkit share` uploads runs for HEAD to the configured share server (`share.url` or `EVALKIT_SHARE_URL`; `share.headers` (object or function) adds headers for an auth proxy in front of the server; `--all`, `--allow-dirty`, `--no-workspace`) and prints the URL. `bun run evalkit share serve` runs a local share server (`--base-path /evals` and `--repo <key>` match a Worker's `BASE_PATH`/`REPO` vars: below a path, one repository only). Pass `dev --no-open` or `dev --port <n>` (`-p`) for dashboard behavior, and `--log-level debug` (on `run`, `dev` or `build`) to see the runner's structured logs. `evalkit <command> --help` lists every flag. `--concurrency` bounds cell parallelism. In CI, add `--junit <file>` (one test case per scenario and matrix cell) and gate with `bun run evalkit compare --base <branch> --fail-on regression` (or `--base-dir <results>`; `--markdown <file>` for an MR comment, `--fail-on score-drop=0.05`). `run` exits 1 on any failure or a selected file that failed to load (`--skip-broken` runs the rest), 130 when cancelled with Ctrl+C; cancelled runs are never failures. Datasets' digests are recorded per run; `compare` lists changed datasets. For baseline/treatment studies, inspect a `--dry-run` first, keep model/task/prompt constant, and identify whether the treatment is injected context or live tool retrieval.
|
|
11
|
-
2. Each `_evalkit-results/<run-uuid>/` is one run (one execution): `manifest.json`, `catalog.json`, `
|
|
11
|
+
2. Each `_evalkit-results/<run-uuid>/` is one run (one execution): `manifest.json`, `catalog.json`, `transcript.atxf.jsonl` (the agent's ATXF transcript, sealed), `run-events.jsonl` (EvalKit's step/scorer/limit events), optional `transcripts/` (sub-agents), `scoring.json`, `summary.json`, and `artifacts/{candidate,attachments}/`. Status can be `limited` (hit `limits`; `summary.limit` says which). `repeat: n` starts `n` runs sharing `manifest.batch: { id, index, size }`. Results saved by older versions (before 0.9 a run held `runs/`; 0.9 and 0.10 saved `trajectory.jsonl`) are skipped until `bun run evalkit upgrade` converts them; it only reads the config, so run it first when updating.
|
|
12
12
|
3. A run's `summary.json` `status` is `passed`, `failed` (finished, but a score or checkpoint did not pass), `errored` (could not finish), or `cancelled` (never a failure). `scoring.overall` is an unweighted mean of valid values, not a pass rate. For a batch, count how many of its runs passed. For CI, use `jq -e '.status == "passed"' "_evalkit-results/<run-uuid>/summary.json"` after replacing the placeholder; the CLI exits nonzero when any selected run does not pass.
|
|
13
13
|
4. `--json` emits one JSON object per finished run with its cell key, eval ID, effective parameters, and run result (including `batch`). Read the saved run trajectory for adapter-emitted turn latency or token usage; EvalKit does not infer provider cost, latency, or token counts. Compare matrix conditions at both cell and run level and avoid claiming significance from a small pilot.
|
|
14
14
|
5. The dashboard can start runs and inspect events, scoring, and candidate files. Run workspaces live outside the project, in `$TMPDIR/evalkit/<project>-<hash>/`<run-uuid>/{candidate,evaluator}/ (`sandboxDir` overrides it; inside the project, agents can read its source and other runs, and `run` warns), and kept ones retain **both** workspaces; evaluator secrets can live there. Treat reports and sandboxes as sensitive, apply Git ignore/retention rules, and don't share raw data without review. They may be deleted after inspection.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { n as asyncIndex, r as createShareApp, t as ShareIndexDb } from "./index-db-
|
|
1
|
+
import { n as asyncIndex, r as createShareApp, t as ShareIndexDb } from "./index-db-DI7xKMSX.mjs";
|
|
2
2
|
import { mkdir, readFile, readdir, rename, rm, writeFile } from "node:fs/promises";
|
|
3
3
|
import { dirname, join, relative, resolve, sep } from "node:path";
|
|
4
4
|
//#region ../share/src/fs-files.ts
|