@leostera/evalkit 0.4.0 → 0.6.0

package/dist/runner.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"runner.d.mts","names":["_$_evalkit_core0.JsonValue","_$_evalkit_core0.TrajectoryEvent"],"sources":["../../runner/src/errors.ts","../../runner/src/local-report-store.ts","../../runner/src/local-report-reader.ts","../../runner/src/matrix.ts","../../runner/src/index.ts"],"mappings":";;;;;;qBAEa,0BAA0B;EACrC;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBAGW,oBAAoB;EAC/B;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBASW,iCAAiC;EAG5C;EACA;EACA;EACA;;;;;wBCqGc,iBAAiB,wBAAwB;;;qBCjH5C,kBAAe,cAAgB,kBAAe;;;;;;0BAA9CA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAEA,iBAAc,cAAgB,kBAAe;;;;;;;;;;;;;qBAE7C,oBAAiB,cAChB,eACC,oBACE;;;;;;;0BAHJA;;;;;;;;;;;;;;;qBASA,mBAAgB,cACf,eACC,oBACE;;;;;;;;;;;0BAHJA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBASA,mBAAgB,cACf,eACC,oBACE;;;;;;;wBAHJA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;wBASS,gBACpB,cACA,eACA,kBAAe,QAHKC;;;;wBC3CN,UACd,QAAQ,YACR,SAAS,KACP;EAGA,YAAY;EACZ,YACE,MAAM,gBACN,QAAQ,qBACE;IAEb,OAAO;EAAS;EAAe;EAAgB;;;;YCsBtC;EACV,QAAQ;EACR;EACA;EACA;;EAEA;;EAEA,aAAa;;EAEb,SAAS;;EAET,UAAU;;EAEV;;EAEA,YAAY,OAAO;;EAEnB;;EAEA,SAAS;;EAET,UAAU;EACV,YAAY;;;wBASE,QACd,YAAY,gBACZ,SAAS,iBACR,OAAO,OAAO"}
+{"version":3,"file":"runner.d.mts","names":["_$_evalkit_core0.JsonValue","_$_evalkit_core0.TrajectoryEvent"],"sources":["../../runner/src/errors.ts","../../core/src/identity.ts","../../core/src/dataset.ts","../../core/src/aut-events.ts","../../core/src/catalog.ts","../../core/src/matrix.ts","../../core/src/index.ts","../../runner/src/local-report-store.ts","../../runner/src/datasets.ts","../../runner/src/local-report-reader.ts","../../runner/src/matrix.ts","../../runner/src/index.ts"],"mappings":";;;;;;qBAEa,0BAA0B;EACrC;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBAGW,oBAAoB;EAC/B;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBASW,iCAAiC;EAG5C;EACA;EACA;EACA;;;;;;qBAIW,uBAAuB;EAClC;;;;cCrCW,oBAAkB,OAAA;KACnB,sBAAsB,mBAAmB;KAiBzC,YAAY,cAAc,eAAe,2BACxC;;;;KCdD;;KAGA,6BAA6B;EAAe;;;;;;KAM5C,wBAAwB,SAAS,eAAe;;KAGhD;EACV,QAAQ;;EAER;;KAGU;;EAEV;EACA,QAAQ;;KAGE;EACV;EACA;EACA;EACA;;EAEA;;EAEA,YAAY;EACZ,QAAQ;;KAoCE;EAEN;EACA,QAAQ;EACR;EACA;;EAGA;EACA,SACE,MAAM,uBACH,iBAAiB,QAAQ;EAC9B,SACE,QAAQ,gBACR,OAAO,wBACJ,iBAAiB,QAAQ;;;KAIxB;WACD;WACA;WACA;WACA,YAAY;;WAEZ;WACA,QAAQ;;;;;;;;KC/FP;EACN;EAAiB;;EAEjB;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB;EAAkB;;EACrC;EAAe,OAAO;EAAe;;KAE/B;EACV;EACA;;EAEA;EACA;;;KAIU;EAGN;EACA,qBAAqB;EACrB,QAAQ;;;;KC5BF;EACV;EACA;EACA;EACA;EACA,UAAU;IAAQ,MAAM;IAAkB;;;EAE1C;;;KAIU;EACN;EAAe;;EACf;EAAgB;;EAChB;;KAEM;EACV;EACA;EACA,YAAY;EACZ;EACA;;EAEA;IAAU;;IAAwB;;;EAElC;IAAU,OAAO;;IAAiB;;;EAElC;IAAU;IAAc;;EACxB;;KAGU;EACV;EACA,MAAM;EACN;EACA;;EAEA;EACA;;KAGU;EACN;EAAc;;EACd;EAAe,aAAa;;EAC5B;EAA0B;EAAc,YAAY;;;;EAEpD;EAAoB,QAAQ;;KAEtB;EACV;EACA;EACA;EACA;EACA,OAAO;;EAEP,aAAa;EACb,UAAU;EACV;EACA;IAAU;IAAgB;IAAoB;;EAC9C,YAAY;EACZ,SAAS;EACT,WAAW;;EAEX;;;;KC/EU;;EAEV;;EAEA,aAAa,SAAS,wBAAwB;;EAE9C,YAAY;;KAEF;EACV;;EAEA;EACA,MAAM;EACN,YAAY;;KAEF;EACV;EACA;EACA,gBAAgB;EAChB,YAAY,SAAS,wBAAwB;EAC7C,WAAW;;EAEX,iBAAiB;;EAEjB,mBAAmB;;KAET,aAAa;EACvB;EACA,MAAM,YAAY;EAClB,MAAM,YAAY,kBAAkB,iBAAiB;;;;KCpB3C;KACA,YACV,gBAAgB;GAAiB,cAAc;;KACrC;GAAgB,cAAc;;KAE9B;;EAEV;EACA;EACA;EACA;;KAGU;;;;;KAMA;EACV;EACA,gBAAgB;;KAGN,gBAAgB,QAAQ,OAAO,kBAAkB;KAEjD;EACV;EACA;EACA;;KAGU;EACV;EACA;EACA;;KAGU;EACN;EAAiB;;EAEjB;EACA;EACA,SAAS;EACT;;EAGA;EACA;EACA;EACA,WAAW;EACX;;EAGA;EACA;EACA;EACA,QAAQ;EACR;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB,SAAS;EAAW;;EACvC;EAAe,OAAO;EAAe;;KAE/B;EACN;EAAuB;;EACvB;EAAiC;EAAc;;EAC/C;EAAmC;EAAc;;EACjD;EAAiC;EAAc;;EAE/C;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAwB;EAAgB;;EAExC;EACA;;EAEA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAyB;;EACzB;EAAe,OAAO;EAAe;;KAE/B,mBACT;EAAa;MAAoB;EAAgB;;KAExC;;EAEV;;KAGU,qBAAqB;KACrB,qBAAqB;KAErB;EACV;EACA;EACA;EACA;EACA,UAAU;;EAEV,aAAa;EACb,UAAU;;EAEV,MAAM;;EAEN,WAAW;;EAEX,oBAAoB;;KAGV,WAAW;WACZ,WAAW;;WAEX,WAAW;EACpB,MAAM;IACJ,SAAS;IACT,UAAU;;IAEV,UAAU,OAAO,yBAAyB;MACxC,QAAQ,WAAW;;KAGb,WAAW;;;;;EAKrB,KAAK,SAAS,WAAW,eAAe;;EAExC,UAAU;;KAwBA;EACV;EACA;;KAGU;EACV;EACA,aAAa;;KAGH;EACV;EACA,kBAAkB;IAAa;;EAC/B,6BAA6B,QAAQ;IAAY;;IAC/C;IACA;;EAEF;IACE;IACA;IACA;IACA,WAAW;IACX;IACA,SAAS;;EAEX;;KAGU;EACV;EACA;EACA;IAAY;IAAc,YAAY;;;KAE5B,iBACV,WAAW,YAAY,cAAc;KAmB3B;EACV,QAAQ;EACR,QAAQ;;EAER,SAAS;;;KAIC;EAAiB;EAAe;;KAEhC,sBAER;EAEE;EACA;EACA;;EAEA;;EAEA,QAAQ;;KAYF;EACV;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;;EAER;;KAGU;EACV,WAAW;EACX,WAAW;;KAGD;EACV,SAAS;EACT;IAAc,iBAAiB;;EAC/B,WAAW;;EAEX,OAAO;;EAEP,SAAS;;EAET,MAAM;;KAGI;EACV;EACA;EACA;;EAEA;EACA,IACE,SAAS,2BACE,aAAa,kBAAkB;;KAGlC;EACV;EACA;EACA;;EAEA;;KAGU,cAAc,kBAAkB;KA+BhC;EACV;;EAEA;;EAEA;;KAGU,eAAe,eAAe,aAAa;EACrD;EACA;EACA,OAAO;;EAEP,QAAQ;;EAER,oBAAoB;EACpB,YAAY;EACZ,SAAS;EACT,SAAS;;EAET,WAAW;;KA4FD;EACV;EACA;EACA;;KAGU;EACV;EACA;EACA;;KAgFU;;;;;KAMA;;EAEV;;EAEA;;EAEA;EACA;EACA;;EAEA;EACA;IAAO;IAAkB;IAAmB;IAAc;;;;KAIhD;EACV,SAAS;EACT,OAAO;EACP,QAAQ;;;KAIE;EAAe;EAAa;;KAE5B;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,SAAS;;EAET,UAAU;;KAGA;EACV;EACA;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,WAAW;;KAGD;EACV;EACA,YAAY;EACZ;;EAEA;;KAGU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;EACR;IAAoB;IAAoB;;;KAG9B;EACV,SAAS;EACT,cAAc;EACd;EACA;EACA;;KAGU;;EAEV;EACA;EACA;;KAGU;EACV,QAAQ;EACR;EACA;EACA,UAAU;EACV,YAAY;EACZ,QAAQ;;KAGE;EACV,QAAQ;EACR;EACA;EACA;EACA;EACA;;EAEA;EACA,QAAQ;;KAGE;EACV,YAAY,OAAO,kBAAkB;EACrC,YAAY,SAAS,eAAe;;EAEpC,cAAc,cAAc,MAAM,aAAa;EAC/C,SAAS,SAAS,eAAe;;KAGvB;WACD;EACT,WAAW,UAAU,gBAAgB,QAAQ;EAC7C,SAAS,SAAS,aAAa;;KAGrB;EACV,SACE,UAAU,aACV;IAAY,UAAU;MACrB,QAAQ;;KAGD;EACV;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,UAAU;EACV,QAAQ;;KAGE;EACV;EACA;EACA;EACA;EACA;;KAGU,YAAY;EACtB;EACA;EACA;EACA;EACA,mBAAmB;;EAEnB,SAAS;;;;;wBC7lBK,iBAAiB,wBAAwB;;;;KCiI7C;EAAqB;EAAc;;;;;;;wBAOzB,mBACpB,SAAS,SACT,eACC,QAAQ;;wBA6BW,kBACpB,SAAS,SACT,cACA,eACC,QAAQ,WAAW;;;qBC3RT,kBAAe,cAAgB,kBAAe;;;;;;0BAA9CA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAEA,iBAAc,cAAgB,kBAAe;;;;;;;;;;;;;;qBAE7C,oBAAiB,cAChB,eACC,oBACE;;;;;;;0BAHJA;;;;;;;;;;;;;;;;;;;;;qBASA,mBAAgB,cACf,eACC,oBACE;;;;;;;;;;;0BAHJA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBASA,mBAAgB,cACf,eACC,oBACE;;;;;;;wBAHJA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;wBASS,gBACpB,cACA,eACA,kBAAe,QAHKC;;;;;wBAiBN,YAAY,OAAO;;;;KC7DvB;EACV;EACA;EACA;EACA;;;wBAIc,UACd,QAAQ,YACR,SAAS,KACP;EAGA,YAAY;;EAEZ,YAAY,MAAM;EAClB,YACE,MAAM,gBACN,QAAQ,qBACE;IAEb,OAAO,OAAO;;;YCkCL;EACV,QAAQ;EACR;EACA;EACA;;EAEA;;EAEA,aAAa;;EAEb,SAAS;;EAET,UAAU;;EAEV;;EAEA,YAAY,OAAO;;EAEnB;;EAEA,SAAS;;EAET,UAAU;;EAEV;;;;;EAKA,SAAS;;;;;EAKT,WAAW,OAAO,UAAU,OAAO;EACnC,YAAY;;;YAIF;EACV;EACA;EACA;EACA;EACA;EACA,aAAa;;;wBASC,QACd,YAAY,gBACZ,SAAS,iBACR,OAAO,OAAO"}

package/dist/runner.mjs

@@ -1,2 +1,2 @@
-import { a as readTrialEvents, c as readTrialSummary, d as CheckpointExecutionError, f as FixtureError, i as readRunSummary, l as localReportStore, m as ScoringError, n as runMatrix, o as readTrialManifest, p as ReportError, r as readRunManifest, s as readTrialScoring, t as runEval, u as AutExecutionError } from "./src-DoPs9Yug.mjs";
-export { AutExecutionError, CheckpointExecutionError, FixtureError, ReportError, ScoringError, localReportStore, readRunManifest, readRunSummary, readTrialEvents, readTrialManifest, readTrialScoring, readTrialSummary, runEval, runMatrix };
+import { _ as ReportError, a as readRunSummary, c as readTrialScoring, d as staticDatasetFile, f as staticDatasetFiles, g as DatasetError, h as CheckpointExecutionError, i as readRunManifest, l as readTrialSummary, m as CancelledError, n as runMatrix, o as readTrialEvents, p as AutExecutionError, r as ownerIsGone, s as readTrialManifest, t as runEval, u as localReportStore, v as ScoringError } from "./src-CVSCn3bZ.mjs";
+export { AutExecutionError, CancelledError, CheckpointExecutionError, DatasetError, ReportError, ScoringError, localReportStore, ownerIsGone, readRunManifest, readRunSummary, readTrialEvents, readTrialManifest, readTrialScoring, readTrialSummary, runEval, runMatrix, staticDatasetFile, staticDatasetFiles };

package/dist/share-worker.d.mts

@@ -1,4 +1,79 @@
-import { h as R2BucketLike, o as SqlValue, r as ShareIndex } from "./index-db-HdZXelOS.mjs";
+//#region ../share/src/files.d.ts
+/** The subset of the Workers R2 binding used here. */
+type R2BucketLike = {
+  get(key: string): Promise<{
+    body: ReadableStream<Uint8Array>;
+  } | null>;
+  put(key: string, value: Uint8Array | string): Promise<unknown>;
+  list(options: {
+    prefix: string;
+    cursor?: string;
+  }): Promise<{
+    objects: {
+      key: string;
+    }[];
+    truncated: boolean;
+    cursor?: string;
+  }>;
+};
+//#endregion
+//#region ../share/src/upload.d.ts
+/** Finished runs only: a shared run's files are immutable once stored. */
+type SharedRunStatus = 'passed' | 'failed' | 'errored';
+type SharedSource = {
+  repo: string;
+  project: string;
+  commit: string;
+  ref?: string;
+  committedAt?: string;
+  dirty: boolean;
+};
+/** The fields of an uploaded run.json the server relies on. */
+type SharedRun = {
+  id: string;
+  evalId: string;
+  suiteId?: string;
+  status: SharedRunStatus;
+  score?: number;
+  startedAt: string;
+  source: SharedSource;
+  /** Repository key (`repo` plus `/~project`), checked to be URL- and path-safe. */
+  repoKey: string;
+  /** The whole run.json object, served as-is in runs.json. */
+  summary: unknown;
+};
+//#endregion
+//#region ../share/src/index-db.d.ts
+type SqlValue = string | number | null;
+type RepoRow = {
+  key: string;
+  url: string;
+  project: string;
+  firstSeenAt: string;
+  lastUploadAt: string;
+  commits: number;
+};
+type CommitRow = {
+  commit: string;
+  ref?: string;
+  committedAt?: string;
+  firstSeenAt: string;
+  runs: number;
+  passed: number;
+  failed: number;
+};
+/** The operations the app needs; async so a Durable Object can sit behind it. */
+type ShareIndex = {
+  hasRun(runId: string): Promise<boolean>;
+  recordRun(run: SharedRun): Promise<void>;
+  repos(): Promise<RepoRow[]>;
+  commits(repo: string): Promise<CommitRow[]>;
+  runs(repo: string, sha: string): Promise<unknown[]>;
+  /** Full SHA for a full SHA, a unique prefix (≥ 7), or a ref's latest commit. */
+  resolve(repo: string, ref: string): Promise<string | undefined>;
+  latest(repo: string): Promise<string | undefined>;
+};
+//#endregion
 //#region ../share/src/index-do.d.ts
 /** The subset of Durable Object state used by the index. */
 type DurableObjectStateLike = {

package/dist/share-worker.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"share-worker.d.mts","names":[],"sources":["../../share/src/index-do.ts","../../share/src/worker.ts"],"mappings":";;;KAGK;EACH;IACE;MACE,KAAK,kBAAkB,QAAQ;QAAe;;;IAEhD,gBAAgB,GAAG,YAAY,IAAI;;;KAIlC;EACH,WAAW;EACX,IAAI;IAAgB,MAAM,SAAS,UAAU,QAAQ;;;;;;;;cAmB1C;mBACM;EAEjB,YAAY,OAAO;EAYb,MAAM,SAAS,UAAU,QAAQ;;;iBAezB,mBACd,WAAW,6BACV;;;;;;;;KCtDS;EACV,SAAS;EACT,OAAO,kBAAkB;EACzB;IAAU,MAAM,SAAS,UAAU,QAAQ;;;;EAI3C,MAAK,SAAU,SAAO,KAAO,WAAW,QAAQ,YAAY"}
+{"version":3,"file":"share-worker.d.mts","names":[],"sources":["../../share/src/files.ts","../../share/src/upload.ts","../../share/src/index-db.ts","../../share/src/index-do.ts","../../share/src/worker.ts"],"mappings":";;KASY;EACV,IAAI,cAAc;IAAU,MAAM,eAAe;;EACjD,IAAI,aAAa,OAAO,sBAAsB;EAC9C,KAAK;IAAW;IAAgB;MAAoB;IAClD;MAAW;;IACX;IACA;;;;;;KCFQ;KAEA;EACV;EACA;EACA;EACA;EACA;EACA;;;KAIU;EACV;EACA;EACA;EACA,QAAQ;EACR;EACA;EACA,QAAQ;;EAER;;EAEA;;;;KCrBU;KAEA;EACV;EACA;EACA;EACA;EACA;EACA;;KAGU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;;;KAIU;EACV,OAAO,gBAAgB;EACvB,UAAU,KAAK,YAAY;EAC3B,SAAS,QAAQ;EACjB,QAAQ,eAAe,QAAQ;EAC/B,KAAK,cAAc,cAAc;;EAEjC,QAAQ,cAAc,cAAc;EACpC,OAAO,eAAe;;;;;KC1CnB;EACH;IACE;MACE,KAAK,kBAAkB,QAAQ;QAAe;;;IAEhD,gBAAgB,GAAG,YAAY,IAAI;;;KAIlC;EACH,WAAW;EACX,IAAI;IAAgB,MAAM,SAAS,UAAU,QAAQ;;;;;;;;cAmB1C;mBACM;EAEjB,YAAY,OAAO;EAYb,MAAM,SAAS,UAAU,QAAQ;;;iBAezB,mBACd,WAAW,6BACV;;;;;;;;KCtDS;EACV,SAAS;EACT,OAAO,kBAAkB;EACzB;IAAU,MAAM,SAAS,UAAU,QAAQ;;;;EAI3C,MAAK,SAAU,SAAO,KAAO,WAAW,QAAQ,YAAY"}

package/dist/share-worker.mjs

@@ -69,5 +69,3 @@ var worker_default = { fetch(request, env) {
 } };
 //#endregion
 export { ShareIndexObject as ShareIndex, worker_default as default };
-
-//# sourceMappingURL=share-worker.mjs.map

package/dist/share.d.mts

@@ -1,5 +1,144 @@
-import { _ as r2Files, a as Sql, c as SharedRun, d as BUNDLE_FORMAT, f as BundleFile, g as ShareFiles, h as R2BucketLike, i as ShareIndexDb, l as SharedRunStatus, m as encodeBundle, n as RepoRow, p as decodeBundle, r as ShareIndex, s as asyncIndex, t as CommitRow, u as parseUpload } from "./index-db-HdZXelOS.mjs";
 import { Hono } from "hono";
+//#region ../share/src/files.d.ts
+/** Where uploaded result files live: R2 in a Worker, a directory elsewhere. */
+type ShareFiles = {
+  get(key: string): Promise<ReadableStream<Uint8Array> | Uint8Array | null>;
+  put(key: string, data: Uint8Array | string): Promise<void>;
+  /** Every key below `prefix`, in no particular order. */
+  list(prefix: string): AsyncIterable<string>;
+};
+/** The subset of the Workers R2 binding used here. */
+type R2BucketLike = {
+  get(key: string): Promise<{
+    body: ReadableStream<Uint8Array>;
+  } | null>;
+  put(key: string, value: Uint8Array | string): Promise<unknown>;
+  list(options: {
+    prefix: string;
+    cursor?: string;
+  }): Promise<{
+    objects: {
+      key: string;
+    }[];
+    truncated: boolean;
+    cursor?: string;
+  }>;
+};
+export declare function r2Files(bucket: R2BucketLike): ShareFiles;
+//#endregion
+//#region ../share/src/bundle.d.ts
+/**
+ * A run bundle: the snapshot files for one run, as one gzip-compressed body.
+ *
+ *   "EKBUNDLE" (8 bytes) · header length (u32 BE) · header JSON · file bytes…
+ *   header = { format: 1, files: [{ path, size }, …] }, bytes in that order.
+ *
+ * Both ends are EvalKit, so this is simpler than tar and has no path-length limit.
+ */
+export declare const BUNDLE_FORMAT = 1;
+type BundleFile = {
+  path: string;
+  data: Uint8Array;
+};
+export declare function encodeBundle(files: BundleFile[]): Promise<Uint8Array>;
+/** Decode and check the framing; path rules are enforced by the caller. */
+export declare function decodeBundle(body: Uint8Array, limits: {
+  maxBytes: number;
+}): Promise<BundleFile[]>;
+//#endregion
+//#region ../share/src/upload.d.ts
+/** Finished runs only: a shared run's files are immutable once stored. */
+type SharedRunStatus = 'passed' | 'failed' | 'errored';
+type SharedSource = {
+  repo: string;
+  project: string;
+  commit: string;
+  ref?: string;
+  committedAt?: string;
+  dirty: boolean;
+};
+/** The fields of an uploaded run.json the server relies on. */
+type SharedRun = {
+  id: string;
+  evalId: string;
+  suiteId?: string;
+  status: SharedRunStatus;
+  score?: number;
+  startedAt: string;
+  source: SharedSource;
+  /** Repository key (`repo` plus `/~project`), checked to be URL- and path-safe. */
+  repoKey: string;
+  /** The whole run.json object, served as-is in runs.json. */
+  summary: unknown;
+};
+/** A run bundle that passed every check; storing it cannot fail on its contents. */
+type RunUpload = {
+  run: SharedRun;
+  runJson: Uint8Array;
+  catalog?: Uint8Array;
+  /** trials.json and trials/** files. */
+  files: BundleFile[];
+};
+export declare function parseUpload(bundle: readonly BundleFile[]): RunUpload;
+//#endregion
+//#region ../share/src/index-db.d.ts
+/**
+ * The share server's metadata index: repositories, commits and runs.
+ * R2 (or a directory) holds the files; this index answers "what exists" and
+ * every listing query. It is derived from stored run.json files and can be
+ * rebuilt from them (`POST /api/v1/reindex`).
+ */
+/** Synchronous SQLite access, provided by Durable Object storage or bun:sqlite. */
+type Sql = {
+  all<T>(query: string, ...params: SqlValue[]): T[];
+  run(query: string, ...params: SqlValue[]): void;
+  transaction<T>(work: () => T): T;
+};
+type SqlValue = string | number | null;
+type RepoRow = {
+  key: string;
+  url: string;
+  project: string;
+  firstSeenAt: string;
+  lastUploadAt: string;
+  commits: number;
+};
+type CommitRow = {
+  commit: string;
+  ref?: string;
+  committedAt?: string;
+  firstSeenAt: string;
+  runs: number;
+  passed: number;
+  failed: number;
+};
+/** The operations the app needs; async so a Durable Object can sit behind it. */
+type ShareIndex = {
+  hasRun(runId: string): Promise<boolean>;
+  recordRun(run: SharedRun): Promise<void>;
+  repos(): Promise<RepoRow[]>;
+  commits(repo: string): Promise<CommitRow[]>;
+  runs(repo: string, sha: string): Promise<unknown[]>;
+  /** Full SHA for a full SHA, a unique prefix (≥ 7), or a ref's latest commit. */
+  resolve(repo: string, ref: string): Promise<string | undefined>;
+  latest(repo: string): Promise<string | undefined>;
+};
+export declare class ShareIndexDb {
+  private readonly sql;
+  private readonly now;
+  constructor(sql: Sql, now?: () => Date);
+  hasRun(runId: string): boolean;
+  /** Idempotent: recording an indexed run again changes nothing. */
+  recordRun(run: SharedRun): void;
+  repos(): RepoRow[];
+  commits(repo: string): CommitRow[];
+  runs(repo: string, sha: string): unknown[];
+  resolve(repo: string, ref: string): string | undefined;
+  latest(repo: string): string | undefined;
+}
+/** Adapt the synchronous index (bun:sqlite) to the app's async interface. */
+export declare function asyncIndex(db: ShareIndexDb): ShareIndex;
+//#endregion
 //#region ../share/src/app.d.ts
 /** The bundled dashboard: its index.html template and its JS/CSS files. */
 type DashboardAssets = {
@@ -83,14 +222,18 @@ export declare function dashboardIndex(html: string, options: {
   share?: ShareContext;
 }): string;
 //#endregion
-//#region ../share/src/index-bun.d.ts
-/** The index in a SQLite file, for `evalkit share serve` and tests. */
-export declare function sqliteIndex(path: string): ShareIndex;
+//#region ../share/src/index-sqlite.d.ts
+/**
+ * The index in a SQLite file, for `evalkit share serve` and tests: bun:sqlite
+ * on Bun, node:sqlite on Node. Imported lazily so neither runtime loads the
+ * other's module.
+ */
+export declare function sqliteIndex(path: string): Promise<ShareIndex>;
 //#endregion
 //#region ../share/src/node.d.ts
 /** Serve the bundled dashboard from a directory (`dist/dashboard`). */
 export declare function directoryAssets(root: string): DashboardAssets;
-/** A share server on Bun: files and `index.sqlite` live in `dir`. */
+/** A share server on Bun or Node: files and `index.sqlite` live in `dir`. */
 export declare function createDirectoryShareApp(options: {
   dir: string;
   dashboardRoot: string;
@@ -112,5 +255,5 @@ export declare function repoKey(source: {
 }): string;
 export declare function validRepoKey(key: string): boolean;
 //#endregion
-export { BUNDLE_FORMAT, type BundleFile, type CommitRow, type DashboardAssets, type R2BucketLike, type RepoRow, type ShareAppOptions, type ShareEvent, type ShareFiles, type ShareIndex, ShareIndexDb, type SharedRun, type SharedRunStatus, type Sql, asyncIndex, decodeBundle, encodeBundle, parseUpload, r2Files };
+export type { BundleFile, CommitRow, DashboardAssets, R2BucketLike, RepoRow, ShareAppOptions, ShareEvent, ShareFiles, ShareIndex, SharedRun, SharedRunStatus, Sql };
 //# sourceMappingURL=share.d.mts.map

package/dist/share.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"share.d.mts","names":[],"sources":["../../share/src/app.ts","../../share/src/fs-files.ts","../../share/src/html.ts","../../share/src/index-bun.ts","../../share/src/node.ts","../../share/src/paths.ts"],"mappings":";;;;KAiBY;EACV,SAAS;EACT,IAAI,eAAe,QAAQ;;;KAIjB;EAEN;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;EAAuB;EAAqB;;EAC5C;EAAwB;EAAc;;EACtC;EAAsB;EAAgB;EAAc;;KAE9C;EACV,OAAO;EACP,OAAO;EACP,QAAQ;;EAER;;EAEA,OAAO,OAAO;;;;;;;;;;;;;;;;;;wBAiCA,eAAe,SAAS,kBAAe,0BAAA,+BAAA;;;;wBCxEvC,eAAe,eAAe;;;;;;;;KCAlC;;EAEV;;EAEA;;EAEA;EACA;;EAEA;;wBAGc,eACd,cACA;EAAW;EAAc;EAAiB,QAAQ;;;;;wBCfpC,YAAY,eAAe;;;;wBCU3B,gBAAgB,eAAe;;wBAsBzB,wBAAwB;EAC5C;EACA;IACD,uBAAA,0BAAA,+BAAA;;;;;;;wBCfe,eAAe;;;;;wBAef,QAAQ;EAAU;EAAc;;wBAIhC,aAAa"}
+{"version":3,"file":"share.d.mts","names":[],"sources":["../../share/src/files.ts","../../share/src/bundle.ts","../../share/src/upload.ts","../../share/src/index-db.ts","../../share/src/app.ts","../../share/src/fs-files.ts","../../share/src/html.ts","../../share/src/index-sqlite.ts","../../share/src/node.ts","../../share/src/paths.ts"],"mappings":";;;KACY;EACV,IAAI,cAAc,QAAQ,eAAe,cAAc;EACvD,IAAI,aAAa,MAAM,sBAAsB;;EAE7C,KAAK,iBAAiB;;;KAIZ;EACV,IAAI,cAAc;IAAU,MAAM,eAAe;;EACjD,IAAI,aAAa,OAAO,sBAAsB;EAC9C,KAAK;IAAW;IAAgB;MAAoB;IAClD;MAAW;;IACX;IACA;;;wBAIY,QAAQ,QAAQ,eAAe;;;;;;;;;;;qBCXlC;KAGD;EAAe;EAAc,MAAM;;wBAUzB,aAAa,OAAO,eAAe,QAAQ;;wBA2B3C,aACpB,MAAM,YACN;EAAU;IACT,QAAQ;;;;KCtCC;KAEA;EACV;EACA;EACA;EACA;EACA;EACA;;;KAIU;EACV;EACA;EACA;EACA,QAAQ;EACR;EACA;EACA,QAAQ;;EAER;;EAEA;;;KAUU;EACV,KAAK;EACL,SAAS;EACT,UAAU;;EAEV,OAAO;;wBAGO,YAAY,iBAAiB,eAAe;;;;;;;;;;KC5ChD;EACV,IAAI,GAAG,kBAAkB,QAAQ,aAAa;EAC9C,IAAI,kBAAkB,QAAQ;EAC9B,YAAY,GAAG,YAAY,IAAI;;KAErB;KAEA;EACV;EACA;EACA;EACA;EACA;EACA;;KAGU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;;;KAIU;EACV,OAAO,gBAAgB;EACvB,UAAU,KAAK,YAAY;EAC3B,SAAS,QAAQ;EACjB,QAAQ,eAAe,QAAQ;EAC/B,KAAK,cAAc,cAAc;;EAEjC,QAAQ,cAAc,cAAc;EACpC,OAAO,eAAe;;qBA4CX;mBAEQ;mBACA;EAFnB,YACmB,KAAK,KACL,YAAW;EAK9B,OAAO;;EAKP,UAAU,KAAK;EAwDf,SAAS;EAST,QAAQ,eAAe;EAevB,KAAK,cAAc;EAUnB,QAAQ,cAAc;EAiBtB,OAAO;;;wBASO,WAAW,IAAI,eAAe;;;;KCzMlC;EACV,SAAS;EACT,IAAI,eAAe,QAAQ;;;KAIjB;EAEN;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;EAAuB;EAAqB;;EAC5C;EAAwB;EAAc;;EACtC;EAAsB;EAAgB;EAAc;;KAE9C;EACV,OAAO;EACP,OAAO;EACP,QAAQ;;EAER;;EAEA,OAAO,OAAO;;;;;;;;;;;;;;;;;;wBAiCA,eAAe,SAAS,kBAAe,0BAAA,+BAAA;;;;wBCxEvC,eAAe,eAAe;;;;;;;;KCAlC;;EAEV;;EAEA;;EAEA;EACA;;EAEA;;wBAGc,eACd,cACA;EAAW;EAAc;EAAiB,QAAQ;;;;;;;;;wBCL9B,YAAY,eAAe,QAAQ;;;;wBCAzC,gBAAgB,eAAe;;wBAsBzB,wBAAwB;EAC5C;EACA;IACD,uBAAA,0BAAA,+BAAA;;;;;;;wBCfe,eAAe;;;;;wBAef,QAAQ;EAAU;EAAc;;wBAIhC,aAAa"}

package/dist/share.mjs

@@ -1,4 +1,4 @@
 import { a as allowedRunPath, c as dashboardIndex, d as encodeBundle, i as parseUpload, l as BUNDLE_FORMAT, n as asyncIndex, o as repoKey, r as createShareApp, s as validRepoKey, t as ShareIndexDb, u as decodeBundle } from "./index-db-vbqEGFaU.mjs";
 import { t as r2Files } from "./files-C-U0-PWW.mjs";
-import { i as directoryFiles, n as directoryAssets, r as sqliteIndex, t as createDirectoryShareApp } from "./src-DzZueFn3.mjs";
+import { i as directoryFiles, n as directoryAssets, r as sqliteIndex, t as createDirectoryShareApp } from "./src-CZnDXP1K.mjs";
 export { BUNDLE_FORMAT, ShareIndexDb, allowedRunPath, asyncIndex, createDirectoryShareApp, createShareApp, dashboardIndex, decodeBundle, directoryAssets, directoryFiles, encodeBundle, parseUpload, r2Files, repoKey, sqliteIndex, validRepoKey };

package/dist/skills/evalkit/SKILL.md

@@ -0,0 +1,32 @@
+---
+name: evalkit
+description: Work on or use this Evalkit repository end to end. Use for onboarding, choosing an Evalkit workflow, running evaluations, or coordinating changes across agents, datasets, scenarios, scoring, matrices, and results.
+---
+
+# Evalkit (project guide)
+
+This is a Bun workspace whose `new` CLI command generates standalone projects. `packages/evalkit` builds the single `@leostera/evalkit` artifact; the root `package.json` owns the npm name and manual publishing workflow. Check npm for the actual published version rather than assuming a release is live. The `@evalkit/*` packages are internal to the workspace. Read the Markdown guide `../../../sites/evalkit.leostera.dev/src/content/docs/index.md` and topic pages under `../../../sites/evalkit.leostera.dev/src/content/docs/manual/` for the user workflow; consult the implementation when behavior matters: `../../../packages/core/src/index.ts`, `../../../packages/runner/src/index.ts`, `../../../packages/cli/src/index.ts`, and `../../../examples/starter/` (paths relative to this skill directory). Check `git status --short` first and preserve unrelated changes.
+
+## Route the task
+
+Each sibling directory is a separately discoverable skill. Read its `SKILL.md` when relevant (resolve paths relative to this skill directory):
+
+- `../evalkit-agents/SKILL.md` — add or change an Agent Under Test (AUT) adapter.
+- `../evalkit-datasets/SKILL.md` — files and values for scenarios, expected answers, computed inputs, and per-trial workspaces.
+- `../evalkit-evals/SKILL.md` — author scenarios: transcripts, policies, suites, and registries.
+- `../evalkit-judges/SKILL.md` — write executable predicate scorers and evaluate outcomes.
+- `../evalkit-matrices/SKILL.md` — parameter sweeps and matrix registration.
+- `../evalkit-results/SKILL.md` — run the CLI/dashboard, inspect reports, analyze failures, CI reporting and `compare`.
+
+For a cross-cutting change, use all relevant subskills; for a focused task, load just that subskill. If code and docs disagree, verify the current code and update the manual/skills as appropriate.
+
+## Invariants
+
+- Work from the eval project's directory (for the included project, `examples/starter`). By default the CLI discovers default-exported `scenarios/**/*.eval.ts` without a config or registry, and each directory directly below `scenarios/` is a suite. The starter demonstrates this zero-config discovery path; `examples/configured-matrix` demonstrates optional config and a runnable sweep. An optional `evalkit.config.js`/`.ts` customizes discovery and execution, or can supply an explicit registry/eval list. Dataset paths and `_evalkit-results` / `_evalkit-sandbox` resolve relative to the config directory or project root.
+- A scenario is one concrete conversation in its own small file. Share agents (`agents/`), judges (`judges/`) and datasets (`datasets/`) by importing them; never generate scenarios in loops or from datasets.
+- Authored suites, scenarios, datasets, agents, and matrices use stable lowercase kebab-case IDs. Only runs and trials receive generated runtime URIs.
+- Keep evaluator datasets/answers private from the AUT and model unless a trusted adapter deliberately reads them and includes selected content in its prompt. Such inclusion is a deliberate treatment (for example, a memory-on condition), not a live retrieval test. Local sandboxes retain evaluator files; review reports and sandboxes before sharing.
+- Executable transcript steps are `user(...)`, `predicate(...)`, `judge(...)`, and `expectToolCall(...)`. Final scoring supports both `predicate(...)` and `judge(...)`; `judge(...)` requires a separate eval-level judge adapter. `agent(...)` transcript steps are not executable. `agentsSdk()` is unimplemented, `judgeScorer(...)` is not part of the current API, and `policy.timeoutMs` is not enforced. Verify current code/docs before proposing capabilities.
+- The built-in `piAgent()` invokes Pi with `--no-tools`, once per user message, and does not preserve conversational state. It cannot test live MCP retrieval/tool use. For that, implement a custom AUT adapter that explicitly enables the desired tools and emits observed `tool-call`/`tool-result` events; do not claim a tool was used unless the adapter records it.
+- For an intervention/A-B eval, hold task, model, and other settings constant; vary only the intervention parameter, use repeated trials and a negative control, and report exactly what was varied. A memory snapshot injected into the prompt tests memory-content usefulness, not retrieval quality. Start with a dry run and narrow smoke test; Pi-backed evaluations can incur provider costs.
+- Run `bun run check` and relevant `bun test` suites when changing code. Check `git status --short` first and preserve unrelated changes.

package/dist/skills/evalkit-agents/SKILL.md

@@ -0,0 +1,16 @@
+---
+name: evalkit-agents
+description: Add, adapt, or debug Evalkit Agents Under Test (AUTs). Use when connecting an agent or process, implementing the session adapter, emitting events, or handling trial workspaces and runtime parameters.
+---
+
+# Add an Evalkit agent
+
+Read `../evalkit/SKILL.md` for shared constraints and `../../../sites/evalkit.leostera.dev/src/content/docs/manual/agents.md`. Inspect `../../../packages/core/src/index.ts` (`AutAdapter`, `AutContext`, `AutEvent`), `../../../examples/starter/agents/greeting-agent.ts`, and `../../../packages/agents/src/index.ts` before editing.
+
+1. Identify the **Agent Under Test (AUT)**: the assistant, process, or service whose behavior the eval measures, not the scorer that grades it. Its adapter starts a fresh session per trial, receives user messages, and emits the evidence scorers inspect. Decide whether to reuse `piAgent()` for the local Pi CLI or implement `defineAgent({ identity, start })` for a different agent (`runtimes` defaults to in-process local). Do not add new agent kinds; adapt the agent with `defineAgent`. `agentsSdk()` currently throws; do not claim it is a usable remote adapter.
+2. Give the new agent a stable, unique lowercase kebab-case `identity.id`, meaningful `name`/`kind` and optional version. A declared runtime has `kind` and optional serializable `configuration`; the local CLI selects the first declared runtime among local/remote/sandbox, or lets an adapter without runtimes use its default. Only declare a runtime your adapter really supports.
+3. In `start({ context, runtime, onEvent })`, initialize a **new trial session**. Use `context.workspace.root` for candidate-visible data. Treat `context.evaluatorWorkspace.root` as trusted-only: never expose its path or all its contents to the AUT. `context.data[id]` holds candidate dataset values. A controlled experiment may deliberately read a specific evaluator dataset and pass only that treatment input to the model (for example, a `memory=on` arm); document that this tests injected context, not retrieval. Read and validate `context.parameters` if flags or matrix axes should affect agent behavior.
+4. Return a session whose `send(message)` resolves after the turn finishes and **returns the turn's result**: a reply string, or `{ reply, toolCalls: [{ name, arguments, result?, id? }], usage }`. EvalKit records it (tool calls, then the assistant message, then usage). `close()` is optional. To stream activity, call `onEvent` during the turn instead or as well; lifecycle events (`started`, `turn-started`, `turn-completed`, `completed`) are optional and `timestamp` defaults to now. Plain values are converted to JSON once; no `JSON.parse(JSON.stringify(...))` casts. Propagate failures so the runner can record them.
+5. Import the adapter from an eval; importing an agent alone does not make a run available to the CLI. Add or adapt a predicate; `output` holds the final reply. Test a single eval before a costly suite.
+
+The built-in Pi adapter starts `pi --print --no-session --no-tools` once **per message**, with no session continuity, and does not consume CLI `--model`/budget parameters. It is not suitable for evaluating live MCP/tool retrieval. For custom launch args use `piAgent({ args: [...] })`; for a tool-enabled Pi or persistent conversation, build a custom adapter, explicitly control its MCP/tool configuration, and emit real observed `tool-call`/`tool-result` events. Read and validate `context.parameters` when the model or treatment varies by matrix cell. Do not claim a memory snapshot ablation measures live Hivemind retrieval.

package/dist/skills/evalkit-datasets/SKILL.md

@@ -0,0 +1,17 @@
+---
+name: evalkit-datasets
+description: Add or troubleshoot EvalKit datasets and trial workspaces. Use for files and values scenarios need, expected answers kept from the agent, computed or cached inputs (onLoad/onRun), per-trial data, or dataset path failures.
+---
+
+# Add EvalKit datasets
+
+Read `../evalkit/SKILL.md` and `../../../sites/evalkit.leostera.dev/src/content/docs/manual/datasets.md`. Check `../../../packages/core/src/dataset.ts` and `../../../packages/runner/src/datasets.ts` for current behavior.
+
+1. Define each dataset once with `defineDataset({ id, ... })` in `datasets/<name>.dataset.ts`, and import it from every scenario that uses it: `defineEval({ datasets: [a, b], ... })`. Nothing is discovered from `datasets/`. IDs are lowercase kebab-case; a scenario may use each dataset once.
+2. **Static** datasets give `files` (a directory relative to the project root, e.g. `'datasets/repo'`, or a map `{ 'path/in/workspace': 'text' | bytes | { file: 'src' } }`), `value` (any JSON-convertible value), or `valueFile` (`.json`, or `.jsonl` read as an array). Both files and a value are allowed.
+3. **Computed** datasets give `onLoad({ root, signal })` instead, returning `{ files?, value? }`. It runs once per process when a trial first needs it (never at import) and is cached; a failure is not cached. `onRun(loaded, trial)` derives each trial's copy using `trial.seed`, `trial.trialIndex` or `trial.parameters`. Put expensive or credentialed work (fetching, generating) here, not at module top level.
+4. Choose `visibility` deliberately. `candidate` (default): files go into the agent's workspace (`context.workspace.root`) and the value is `context.data[id]`. `evaluator`: files go into the evaluator workspace and the value is only on the scoring context. Scorers see every value as `data[id]`. Keep expected answers in evaluator datasets; never pass evaluator paths or contents to a model.
+5. Files land below `mount` (default: the dataset ID; `'.'` is the workspace root). Paths may not escape the mount; two datasets in one scenario may not write the same file. Each trial gets fresh copies.
+6. For a file-creating agent, inspect candidate files via `artifacts.candidate.root` in predicates and the report's `trials/<trial-id>/artifacts/candidate/`. Each trial records a digest per dataset; `evalkit compare` lists datasets whose digest changed.
+
+Datasets never template or fan out scenarios: write one concrete scenario file per case and share the dataset. When dataset setup fails, the trial's `summary.json` and `trajectory.jsonl` record the error naming the dataset. Test the smallest scenario from the project directory.

package/dist/skills/evalkit-evals/SKILL.md

@@ -0,0 +1,19 @@
+---
+name: evalkit-evals
+description: Define or modify EvalKit scenarios (evals) and suites. Use when adding prompts/transcripts, choosing datasets or scorers, configuring trial count, changing registry entries, or debugging unknown eval/suite IDs.
+---
+
+# Add a scenario or suite
+
+Read `../evalkit/SKILL.md`, `../../../sites/evalkit.leostera.dev/src/content/docs/manual/project-structure.md` and `../../../sites/evalkit.leostera.dev/src/content/docs/manual/scoring-and-evals.md`, `../../../examples/starter/scenarios/greeting.eval.ts`, and `../../../examples/configured-matrix/evalkit.config.ts`.
+
+1. Default-export one scenario per file from `scenarios/**/*.eval.ts` using `defineEval({ id: 'my-scenario', name, agent, transcript, scoring, datasets?, policy? })`. The CLI discovers these files automatically; no registry or config is required. Choose a stable lowercase kebab-case ID. `metadata` is optional; `name` is a display label. Import shared agents, judges and datasets; keep each scenario short and concrete (a suite of a hundred cases is a hundred files), and never generate scenarios in a loop.
+2. Executable transcript steps are `user('message')`, inline `predicate(...)`, `judge(...)`, and `expectToolCall(...)`. User messages call `session.send` in order on one fresh session per trial; predicates/judges/tool-call assertions run at checkpoints. `judge(...)` requires a separate eval-level `judge` adapter. `agent(...)` transcript steps are not executable. `{{randomSeed}}` is replaced per trial; `policy.trials` controls independent trials (default 1, positive integer); `policy.timeoutMs` is not enforced.
+3. Suites are directories: every file below `scenarios/billing/` is in suite `billing` (kebab-case directory names; deeper directories stay in the top one). Files directly in `scenarios/` have no suite; `discovery.suites: false` turns this off. Only use an explicit registry (`registry: registerEvals([defineSuite({ id, evals })])` in the config) when you need full control; it replaces discovery, and duplicate eval IDs throw.
+4. For an intervention comparison, hold task/model/prompt constant and vary one explicit matrix axis (for example `memory: ['off', 'on']`). Repeat trials and include a negative control where possible. Record whether the treatment injects a saved memory or performs live retrieval: those answer different questions. A memory dataset can be evaluator-visible to the trusted adapter, but do not expose evaluator roots or reference answers to the AUT unless revealing the chosen treatment is intentional.
+5. From the project directory run `bun run evalkit run <eval-id>`. Use `bun run evalkit run --suite <suite-id>` for a suite directory. IDs are exact human-defined strings. Inspect per-trial `scoring.json` and the run `summary.json`; completed execution is not the same as passing scores.
+6. Update relevant tests and manual/examples if changing documented behavior. Run `bun run check`, inspect a `--dry-run`, and smoke-test one narrow scenario before a potentially costly Pi-backed matrix or suite.
+
+For parameter sweeps, read `../evalkit-matrices/SKILL.md`. For scoring details, read `../evalkit-judges/SKILL.md`.
+
+A scenario file that throws while importing (often a missing credential) becomes a load error for that file only; `run` refuses to run its scenarios unless `--skip-broken`. Do credentialed setup in the agent's `start()` or a dataset's `onLoad`, not at module top level.

package/dist/skills/evalkit-judges/SKILL.md

@@ -0,0 +1,17 @@
+---
+name: evalkit-judges
+description: Write or debug EvalKit scoring rules and judges. Use for predicate scorers, grading trajectories/artifacts, pass/fail thresholds, partial scoring, evidence, or interpreting scorer errors.
+---
+
+# Score EvalKit trials
+
+Read `../evalkit/SKILL.md`, `../../../sites/evalkit.leostera.dev/src/content/docs/manual/scoring-and-evals.md` and `../../../sites/evalkit.leostera.dev/src/content/docs/manual/results.md`, `../../../packages/runner/src/index.ts` (`normalizeScore`, `scoreSummary`, `runPredicate`), and `../../../examples/starter/judges/greeting.ts`.
+
+EvalKit currently executes deterministic `predicate(name, callback)` scorers and agent-backed `judge(name, { rubric })` rules. A judge rule requires a **separate eval-level `judge` adapter**; it is not inferred from or shared with the AUT. The `judgeScorer(...)` API is not implemented—do not suggest it.
+
+1. Add a named `predicate(...)` for deterministic checks or a `judge(...)` descriptor for rubric-based LLM assessment, then reference it from the eval's `scoring` array or supported transcript checkpoint. The predicate callback receives `{ context, output, data, trajectory, artifacts, turn? }`. Use `output` for the final assistant reply and `data[id]` for dataset values (expected answers belong in an evaluator dataset); only filter `trajectory.events` (by `source: 'aut'` and `kind`) for tool calls or intermediate messages. `context` includes per-trial IDs, metadata, random seed and optional parameters; workspace roots are absolute local paths.
+2. A predicate returns a boolean, a finite score from 0 to 1, `{ value, passed?, explanation?, evidence? }` (evidence may be any plain value), or `skip(reason)` when the check does not apply. Without `passed`, only `value === 1` passes; give `predicate(name, fn, { passAt: 0.8 })` for a threshold. A skipped result neither passes nor fails and is left out of averages; never fake "not applicable" as a pass. Use clear evidence for debugging.
+3. A `judge(...)` rule sends a rubric plus the current turn or trajectory to the separate judge adapter, which must emit a JSON verdict. Validate that the judge is independent from the AUT, runs in an appropriate workspace, and does not receive secrets. Judge events/prompts may contain sensitive data and incur model cost; cost is not inferred unless the adapter emits usage.
+4. Normal scorers are skipped after primary execution errors unless declared with `{ supportsPartial: true }` and the required context/workspace exists. Scorer exceptions and invalid scores become recorded failures; never mask them as passes.
+5. Scorers run after the AUT closes; use `node:fs/promises` to inspect candidate/evaluator files before workspace cleanup. Keep evaluator references and secrets out of AUT events, evidence, and exported reports. Reading one evaluator dataset in a trusted adapter and deliberately injecting it as an experiment treatment is valid, but measures injected context—not whether the AUT retrieved it itself.
+6. Test passing, failing, and missing-output trajectories; inspect `scoring.json` (`results`, `overall`, `passed`) and run `summary.json`. `overall` is an unweighted mean of valid scores, not the pass rate. Execution `status: 'completed'` can still have failed scoring; run-level `failed` is the meaningful aggregate gate.

package/dist/skills/evalkit-matrices/SKILL.md

@@ -0,0 +1,14 @@
+---
+name: evalkit-matrices
+description: Configure, run, or debug Evalkit parameter matrices and model sweeps. Use for matrix axes, defaults, eval ID filtering, CLI overrides, or making agents consume per-cell parameters.
+---
+
+# Configure an Evalkit matrix
+
+Read `../evalkit/SKILL.md`, `../../../sites/evalkit.leostera.dev/src/content/docs/manual/project-structure.md` and `../../../sites/evalkit.leostera.dev/src/content/docs/manual/cli-and-matrices.md`, the runnable `../../../examples/configured-matrix/`, `../../../packages/core/src/matrix.ts`, `../../../packages/cli/src/run-command.ts`, and `../../../packages/runner/src/matrix.ts`.
+
+1. In a discovery-based project, default-export each scenario from `scenarios/**/*.eval.ts` and configure `defineConfig({ matrix: { id: 'models', parameters: { model: ['a', 'b'] }, defaults? }, execution: { maxCells: 100 } })` in `evalkit.config.js`. The CLI loads the config and attaches all discovered evals. For an explicit registry instead, create `defineEvalMatrix({ id: 'models', evals: [myEval], parameters: { model: ['a', 'b'] }, defaults? })`. Axes must have JSON-serializable choices; `cells()` lazily expands eligible cells per eval. Use `cases` for correlated complete parameter assignments (e.g. language/framework pairs) and `exclude: [{ os: 'windows', arch: 'arm64' }]` for partial patterns that skip unsupported combinations; `count()` and `--dry-run` report only eligible cells.
+2. For an explicit registry, register the matrix with `registerEvals([myEval, matrix])` (or register `myEval` via a suite): a matrix **does not** register its evals for normal `run`. Do not register an eval twice outside matrices. Use eval IDs with `--eval`.
+3. In the AUT adapter read `context.parameters` and apply the values. Cells merge `defaults`, then selected axis values; `--model`, `--mode`, or `--select axis=value` **select** declared axis choices, while `--param` and budget flags override non-axis parameters. You cannot override an axis with `--param`. The built-in `piAgent()` currently ignores these parameters: using it unchanged does **not** compare different models. A constrained matrix can be run with the CLI; the dashboard lists eligible cells in bounded pages and rejects invalid one-cell submissions.
+4. Run from the eval project: `bun run evalkit run --matrix <matrix-id> --eval <eval-id> --dry-run` before dispatch; then remove `--dry-run`, optionally narrow with `--model model-a` or `--concurrency 4`. Plans exceeding `execution.maxCells` (default 100) require `--all`. `--eval` is a filter, not a matrix axis. Matrix cell keys and effective parameters are persisted in report manifests.
+5. Compare trial scores/pass counts across runs in `_evalkit-results`; verify that the AUT really applied the axis values before interpreting differences. Read `../evalkit-results/SKILL.md` for processing reports.

package/dist/skills/evalkit-results/SKILL.md

@@ -0,0 +1,14 @@
+---
+name: evalkit-results
+description: Run Evalkit locally and interpret or process evaluation results. Use for the CLI, dashboard, JSONL trajectories, trial artifacts, aggregate scores, failures, CI gates, or report retention.
+---
+
+# Run and process Evalkit results
+
+Read `../evalkit/SKILL.md` and `../../../sites/evalkit.leostera.dev/src/content/docs/index.md` (First run) and `../../../sites/evalkit.leostera.dev/src/content/docs/manual/cli-and-matrices.md` and `../../../sites/evalkit.leostera.dev/src/content/docs/manual/results.md`. Verify report semantics in `../../../packages/runner/src/index.ts`, `../../../packages/runner/src/local-report-store.ts`, and `../../../packages/cli/src/index.ts` before automation.
+
+1. Run from the project root containing default-exported `scenarios/**/*.eval.ts` (as in `examples/starter`, or use an optional config as in `examples/configured-matrix`), not this skill directory. Start with `bun run evalkit run <eval-id>`; run all evals only when intended (Pi-backed evals can cost tokens). `bun run evalkit run --suite <suite-id>`, `bun run evalkit run --matrix <matrix-id>`, and `bun run evalkit dev` are also available; `bun run evalkit build [--out dir]` writes a read-only static dashboard snapshot (reports plus candidate artifacts, never evaluator workspaces) that must still be reviewed before sharing; it asks to run evals first (always runs them when there are no results; `--run`/`--no-run` skip the prompt). `bun run evalkit tidy` archives runs of evals no longer in the project (marker `archived.json`, nothing deleted); archived runs are hidden in the dashboard and excluded from builds. Runs record `source` (repo, project, commit, ref, dirty) in `manifest.json`; `bun run evalkit share` uploads runs for HEAD to the configured share server (`share.url` or `EVALKIT_SHARE_URL`; `share.headers` (object or function) adds headers for an auth proxy in front of the server; `--all`, `--allow-dirty`, `--no-workspace`) and prints the URL. `bun run evalkit share serve` runs a local share server. Set `EVALKIT_NO_OPEN=1` or `PORT=...` for dashboard behavior, and `EVALKIT_LOG_LEVEL=debug` to see the runner's structured logs. `--concurrency` bounds cell parallelism. In CI, add `--junit <file>` (one test case per scenario and matrix cell) and gate with `bun run evalkit compare --base <branch> --fail-on regression` (or `--base-dir <results>`; `--markdown <file>` for an MR comment, `--fail-on score-drop=0.05`). `run` exits 1 on any failure or a selected file that failed to load (`--skip-broken` runs the rest), 130 when cancelled with Ctrl+C; cancelled runs are never failures. Datasets' digests are recorded per trial; `compare` lists changed datasets. For baseline/treatment studies, inspect a `--dry-run` first, keep model/task/prompt constant, and identify whether the treatment is injected context or live tool retrieval.
+2. Each `_evalkit-results/<run-uuid>/` has `manifest.json`, `summary.json`, and `trials/<trial-uuid>/{manifest.json,trajectory.jsonl,scoring.json,summary.json,artifacts/candidate/...}`. `trajectory.jsonl` contains one JSON event per line; filter by `source` and `kind`. `scoring.json` records scorer values/explanations/errors; candidate snapshot files are report artifacts.
+3. Read run `summary.json` fields `passed`, `failed`, `trialCount` for success rates. Trial `status: 'completed'` means execution finished, **not** that its scorers passed. `scoring.overall` is an unweighted mean of valid values, not a pass rate. For CI, use `jq -e '.failed == 0 and .trialCount > 0' "_evalkit-results/<run-uuid>/summary.json"` after replacing the placeholder; CLI exits nonzero when a matrix cell fails; the report supplies trial-level detail.
+4. `--json` emits one JSON object per completed matrix cell with its key, eval ID, effective parameters, and run result. Read the saved trial trajectory for adapter-emitted turn latency or token usage; EvalKit does not infer provider cost, latency, or token counts. Compare matrix conditions at both cell and trial level and avoid claiming significance from a small pilot.
+5. The dashboard can start runs and inspect events, scoring, and candidate files. `_evalkit-sandbox/<trial-uuid>/{candidate,evaluator}/` retains **both** local workspaces; evaluator secrets can live there. Treat reports and sandboxes as sensitive, apply Git ignore/retention rules, and don't share raw data without review. They may be deleted after inspection.