@leostera/evalkit 0.7.0 → 0.8.0

package/dist/dashboard/index.html

@@ -4,8 +4,8 @@
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>EvalKit</title>
-    <script type="module" crossorigin src="/assets/index-BEBo_81Y.js"></script>
-    <link rel="stylesheet" crossorigin href="/assets/index-C24IquYZ.css">
+    <script type="module" crossorigin src="/assets/index-xlVsW6ZR.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/index-C12IeG2y.css">
   </head>
   <body>
     <div id="root"></div>

package/dist/index.d.mts

@@ -349,6 +349,7 @@ type EvalkitConfig = {
      * Paths to leave out, as globs relative to the candidate workspace; a
      * pattern without `/` matches that name at any depth. Replaces the default,
      * `DEFAULT_ARTIFACT_EXCLUDE` (dependencies, build output, VCS metadata).
+     * A scenario's `workspace.ignore` patterns come after these.
      */
     exclude?: readonly string[];
   };
@@ -493,6 +494,8 @@ export declare const TrialMetadataSchema: Schema.Struct<{
     version: Schema.optional<typeof Schema.String>;
   }>>;
   startedAt: typeof Schema.String;
+  /** The effective snapshot ignore patterns: the project's, then the scenario's. */
+  workspaceIgnore: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
   datasets: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
     id: typeof Schema.String;
     visibility: Schema.Literal<["candidate", "evaluator"]>;
@@ -556,6 +559,7 @@ export declare const TrialManifestSchema: Schema.Struct<{
     version: Schema.optional<typeof Schema.String>;
   }>>;
   startedAt: typeof Schema.String;
+  workspaceIgnore: Schema.optional<Schema.mutable<Schema.Array$<typeof Schema.String>>>;
   datasets: Schema.optional<Schema.mutable<Schema.Array$<Schema.Struct<{
     id: typeof Schema.String;
     visibility: Schema.Literal<["candidate", "evaluator"]>;
@@ -1232,6 +1236,19 @@ export type EvalDefinition<TAgent extends AutAdapter = AutAdapter> = {
   policy?: EvalPolicy;
   /** Any JSON-convertible object; converted when the eval is defined. */
   metadata?: Record<string, unknown>;
+  workspace?: EvalWorkspace;
+};
+/** How EvalKit treats this scenario's candidate workspace. */
+export type EvalWorkspace = {
+  /**
+   * Paths to leave out of the trial's snapshot, read like `.gitignore`: globs
+   * relative to the workspace, where a pattern without `/` matches that name
+   * at any depth and later patterns win. They come after the project's
+   * `artifacts.exclude` (default `DEFAULT_ARTIFACT_EXCLUDE`), so `!pattern`
+   * can keep a path the project leaves out, even inside an ignored directory:
+   * `['!_build/doc/**']` keeps generated docs while dropping the rest of `_build`.
+   */
+  ignore?: readonly string[];
 };
 export declare function defineEval<const T extends EvalDefinition>(definition: T): T;
 export type EvalSuite<TEvals extends readonly EvalDefinition[] = readonly EvalDefinition[]> = {
@@ -1338,6 +1355,8 @@ export type TrialMetadata = {
   };
   aut?: AutIdentity;
   startedAt: string;
+  /** The effective snapshot ignore patterns: the project's `artifacts.exclude`, then the scenario's `workspace.ignore`. */
+  workspaceIgnore?: string[];
   /** From schema version 5: the datasets this trial used, with content digests. */
   datasets?: TrialDataset[];
 };

package/dist/index.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.mts","names":[],"sources":["../../core/src/identity.ts","../../core/src/dataset.ts","../../core/src/aut-events.ts","../../core/src/catalog.ts","../../core/src/canonical-json.ts","../../core/src/matrix.ts","../../core/src/config.ts","../../core/src/json.ts","../../core/src/definition-site.ts","../../core/src/schema.ts","../../core/src/trajectory.ts","../../core/src/index.ts","../../agents/src/index.ts"],"mappings":";;qBAEa,oBAAkB,OAAA;YACnB,sBAAsB,mBAAmB;;wBAGrC,SAAS;;qBAOZ,YAAU,OAAA,cAAA,OAAA;YAKX,cAAc,WAAW;YAEzB,YAAY,cAAc,eAAe,2BACxC;qBAEA,mBAAiB,OAAA,cAAA,OAAA;wBAMd,kBAAkB,cAAc,cAAc,MAAM,QAAK,OAAA,cAAA,OAAA;wBAWzD,YAAY,cAAc,cACxC,MAAM,OACN,MAAM,OACL,YAAY;wBAIC,iBAAiB,cAAc,cAC7C,eACA,eAAe;EACZ,MAAM;EAAO,MAAM;EAAM,KAAK,YAAY;;wBAY/B,aAAa,eAAe,eAAe,eAAe;;;;YCvD9D;;YAGA,6BAA6B;EAAe;;;;;;YAM5C,wBAAwB,SAAS,eAAe;;YAGhD;EACV,QAAQ;;EAER;;YAGU;;EAEV;EACA,QAAQ;;YAGE;EACV;EACA;EACA;EACA;;EAEA;;EAEA,YAAY;EACZ,QAAQ;;KAGL;EACH;EACA;;EAEA,aAAa;;EAEb;;YAGU,0BAA0B;EACpC,QAAQ;;EAER;;EAEA;EACA;EACA;;YAGU,0BAA0B,gBAAgB,kBACpD;;EAEE,SAAS,MAAM,uBAAuB,UAAU,QAAQ;;EAExD,SACE,QAAQ,SACR,OAAO,wBACJ,iBAAiB,QAAQ;EAC9B;EACA;EACA;;YAGQ;EAEN;EACA,QAAQ;EACR;EACA;;EAGA;EACA,SACE,MAAM,uBACH,iBAAiB,QAAQ;EAC9B,SACE,QAAQ,gBACR,OAAO,wBACJ,iBAAiB,QAAQ;;;YAIxB;WACD;WACA;WACA;WACA,YAAY;;WAEZ;WACA,QAAQ;;;wBAIH,oBAAoB,eAAe;;;;;;;wBAuBnC,cAAc,gBAAgB,gBAC5C,YAAY,0BAA0B,0BAA0B,WAC/D;wBAmFa,UAAU,iBAAiB,SAAS;;;;;;;YC/MxC;EACN;EAAiB;;EAEjB;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB;EAAkB;;EACrC;EAAe,OAAO;EAAe;;YAE/B;EACV;EACA;;EAEA;EACA;;;YAIU;EAGN;EACA,qBAAqB;EACrB,QAAQ;;;wBAIE,kBACd,OAAO,eACP,WAAW,OACV;;;;;wBA0Da,iBACd,QAAQ,YACR,cACA,WAAW,OACV;;;;;;;;;YCtGS;;EAEV;;YAGU;EACV;EACA;EACA;EACA;EACA,UAAU;IAAQ,MAAM;IAAkB;;;EAE1C;;;YAIU;EACN;EAAe;;EACf;EAAgB;;EAChB;;YAEM;EACV;EACA;EACA,YAAY;EACZ;EACA;;EAEA;IAAU;;IAAwB;;;EAElC;IAAU,OAAO;;IAAiB;;;EAElC;IAAU;IAAc;;EACxB;;YAGU;EACV;EACA,MAAM;EACN;EACA;;EAEA;EACA;;YAGU;;;EAEN;EAAc;;EACd;EAAe,aAAa;;EAC5B;EAA0B;EAAc,YAAY;;;;EAEpD;EAAoB,QAAQ;;YAEtB;EACV;EACA;EACA;EACA;EACA,OAAO;;EAEP,aAAa;EACb,UAAU;EACV;EACA;IAAU;IAAgB;IAAoB;;EAC9C,YAAY;EACZ,SAAS;EACT,WAAW;;EAEX;;wBAGc,aACd,YAAY,gBACZ,OAAO,uBACP,UAAS,iBACR;;;;wBC5Fa,oBAAoB,OAAO;;;KCK/B;;EAEV;;EAEA,aAAa,SAAS,wBAAwB;;EAE9C,YAAY;;KAEF;EACV;;EAEA;EACA,MAAM;EACN,YAAY;;KAEF;EACV;EACA;EACA,gBAAgB;EAChB,YAAY,SAAS,wBAAwB;EAC7C,WAAW;;EAEX,iBAAiB;;EAEjB,mBAAmB;;KAET,aAAa;EACvB;EACA,MAAM,YAAY;EAClB,MAAM,YAAY,kBAAkB,iBAAiB;;wBAWvC,uBAAuB,UAAU,sBAC/C,YAAY,IACX,aAAa;;;KCjDJ;;EAEV;EACA;EACA;EACA;;IAEE;;;EAGF,iBAAiB;EACjB,WAAW;;EAEX,SAAS,KAAK;IAA0C;;EACxD;IAAc;IAAsB;IAAiB;;EACrD;;EAEA;;;;;;IAME;;;EAGF;;EAEA,gBAAgB;EAChB,QAAQ;;KAGE;;qBAGC;;KAWD,eAAe;KAEf;;EAEV;;;;;;EAMA,UAAU,sBAAsB,eAAe,QAAQ;;;wBAIzC,mBAAmB,UAAU,eAC3C,eAAe,IACd;;;;qBC9DU,qBAAqB;WAGrB;WACA;WAHO;EAClB,YACW,cACA;;;;;;;;wBAYG,OAAO,gBAAgB,gBAAiB;;wBA4CxC,aACd,gBACA;GACI,cAAc;;;;;;;;;;;KC3DR;EAAmB;EAAc;;wBAqB7B,eAAe,iBAAiB;;;;qBCvBnC,iBAAe,OAAA;qBAOf,qBAAmB,OAAA;;;;;qBAMnB,mBAAiB,OAAA;;;;;;qBAQjB,iBAAiB,OAAO,OAAO;qBAoB/B,iBAAe,OAAA;;;;;;;;;;;;;;qBAiBf,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAejB,qBAAmB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;qBAyBnB,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAIjB,qBAAmB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;qBAKnB,aAAW,OAAA;;;;;qBAMX,oBAAkB,OAAA;;;;;;;;;;;;;;qBAalB,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAYjB,wBAAsB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAgBtB,oBAAkB,OAAA;;;;;;;;;;;;qBAQlB,oBAAkB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAUlB,qBAAmB,OAAA;;;;;qBAKnB,oBAAkB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBASlB,kBAAgB,OAAA;;;;;;;;;;;;;;qBAmJhB,uBAAuB,OAAO,OAAO;qBAGrC,gBAAc,OAAA;;;;;;;wBChVX,qBAAqB,mBAAmB;;;YCgB5C;YACA,YACV,gBAAgB;GAAiB,cAAc;;YACrC;GAAgB,cAAc;;YAE9B;;EAEV;EACA;EACA;EACA;;YAGU;;;;;YAMA;EACV;EACA,gBAAgB;;YAGN,gBAAgB,QAAQ,OAAO,kBAAkB;YAEjD;EACV;EACA;EACA;;YAGU;EACV;EACA;EACA;;YAGU;EACN;EAAiB;;EAEjB;EACA;EACA,SAAS;EACT;;EAGA;EACA;EACA;EACA,WAAW;EACX;;EAGA;EACA;EACA;EACA,QAAQ;EACR;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB,SAAS;EAAW;;EACvC;EAAe,OAAO;EAAe;;YAE/B;EACN;EAAuB;;EAEvB;EACA;;EAEA;EACA;;EAEA;EAAmC;EAAc;;EACjD;EAAiC;EAAc;;EAE/C;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAwB;EAAgB;;EAExC;EACA;;EAEA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAyB;;EACzB;EAAe,OAAO;EAAe;;YAE/B,mBACT;EAAa;MAAoB;EAAgB;;YAExC;;EAEV;;YAGU,qBAAqB;YACrB,qBAAqB;YAErB;EACV;EACA;EACA;EACA;EACA,UAAU;;EAEV,aAAa;EACb,UAAU;;EAEV,MAAM;;EAEN,WAAW;;EAEX,oBAAoB;;YAGV,WAAW;WACZ,WAAW;;WAEX,WAAW;EACpB,MAAM;IACJ,SAAS;IACT,UAAU;;IAEV,UAAU,OAAO,yBAAyB;MACxC,QAAQ,WAAW;;YAGb,WAAW;;;;;EAKrB,KAAK,SAAS,WAAW,eAAe;;EAExC,UAAU;;;qBAIC,wBAAwB;;wBAKrB,cAAc,OAAO,kBAAkB;;;;;wBAUvC,kBAAkB,UAAU,YAAY,OAAO,IAAI;;YAMvD;;EAEV,YAAY;;EAEZ,MAAM;;EAEN;EACA;;YAGU;EACV;;EAEA,oBAAoB,OAAO;;YAGjB;EACV;EACA,aAAa;;YAGH;EACV;EACA,kBAAkB;IAAa;;EAC/B,6BAA6B,QAAQ;IAAY;;IAC/C;IACA;;EAEF;IACE;IACA;IACA;IACA,WAAW;IACX;IACA,SAAS;;EAEX;;YAGU;EACV;EACA;EACA;IAAY;IAAc,YAAY;;;YAE5B,iBACV,WAAW,YAAY,cAAc;;wBAGvB,eACd,UAAU,iCACT;;;;;wBAUa,KAAK,SAAS,sBAAsB;wBAIpC,MAAM,aAAa,aAAa;YAIpC;EACV,QAAQ;EACR,QAAQ;;EAER,SAAS;;;YAIC;EAAiB;EAAe;;YAEhC,sBAER;EAEE;EACA;EACA;;EAEA;;EAEA,QAAQ;;;;;;wBAOE,KAAK,iBAAiB;YAK1B;EACV;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;;EAER;;YAGU;EACV,WAAW;EACX,WAAW;;YAGD;EACV,SAAS;EACT;IAAc,iBAAiB;;EAC/B,WAAW;;EAEX,OAAO;;EAEP,SAAS;;EAET,MAAM;;;;;;EAMN,sBAAsB;;YAGZ;EACV;EACA;EACA;;EAEA;EACA,IACE,SAAS,2BACE,aAAa,kBAAkB;;YAGlC;EACV;EACA;EACA;;EAEA;;YAGU,cAAc,kBAAkB;wBAE5B,UACd,cACA,KAAK,wBACL,UAAS,KAAK,iDACb;;;;;;YAqBS;EACV;EACA;;EAEA;;EAEA;EACA,IACE,SAAS,sCACa;;wBAGV,OACd,cACA,KAAK,eACL,UAAS,KAAK,6BACb;;YAMS;EACV;;EAEA;EACA;EACA;;EAEA;EACA,QAAQ;;wBAGM,MACd,cACA,SAAS,KAAK,2CACb;YAMS;EACV;;EAEA;;EAEA;;YAGU,eAAe,eAAe,aAAa;EACrD;EACA;EACA,OAAO;;EAEP,QAAQ;;EAER,oBAAoB;EACpB,YAAY;EACZ,SAAS;;EAET,mBAAmB;EACnB,SAAS;;EAET,WAAW;;wBAGG,iBAAiB,UAAU,gBAAgB,YAAY,IAAI;YAqD/D,UACV,wBAAwB,4BAA4B;EAEpD;EACA;EACA,OAAO;;wBAGO,kBAAkB,eAAe,WAC/C,OAAO,SACN;YAKS,mBAAmB,iBAAiB,YAAY;wBAE5C,YAAY;EAAS;;YAkBzB;;EAEV,gBAAgB;EAChB,iBAAiB;EACjB,mBAAmB;EACnB,IAAI,cAAc;EAClB,SAAS,cAAc;EACvB,YAAY;EACZ,iBAAiB;;EAEjB,QAAQ,UAAU,iBAAiB;;YAGzB;EACV;EACA;EACA;;YAGU;EACV;EACA;EACA;;;;;;wBAOc,oBACR,gCAAgC,oBACtC,eAAe,iBAAiB;YAuEtB;;;;;YAMA;;EAEV;;EAEA;;EAEA;EACA;EACA;;EAEA;EACA;IAAO;IAAkB;IAAmB;IAAc;;;;YAIhD;EACV,SAAS;EACT,OAAO;EACP,QAAQ;;;YAIE;EAAe;EAAa;;YAE5B;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,SAAS;;EAET,UAAU;;YAGA;EACV;EACA;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,WAAW;;YAGD;EACV;EACA,YAAY;EACZ;;EAEA;;YAGU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;EACR;IAAoB;IAAoB;;;;qBAI7B;YAMD;EACV,SAAS;EACT,UAAU;EACV,cAAc;EACd;EACA;EACA;;YAGU;;EAEV;EACA;EACA;;YAGU;EACV,QAAQ;EACR;EACA;EACA,UAAU;EACV,YAAY;EACZ,QAAQ;;YAGE;EACV,QAAQ;EACR;EACA;EACA;EACA;EACA;;EAEA;EACA,QAAQ;;YAGE;EACV,YAAY,OAAO,kBAAkB;EACrC,YAAY,SAAS,eAAe;;EAEpC,cAAc,cAAc,MAAM,aAAa;EAC/C,SAAS,SAAS,eAAe;;YAGvB;WACD;EACT,WAAW,UAAU,gBAAgB,QAAQ;EAC7C,SAAS,SAAS,aAAa;;YAGrB;EACV,SACE,UAAU,aACV;IAAY,UAAU;MACrB,QAAQ;;YAGD;EACV;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,UAAU;EACV,QAAQ;;YAGE;EACV;EACA;EACA;EACA;EACA;;YAGU,YAAY;EACtB;EACA;EACA;EACA;EACA,mBAAmB;;EAEnB,SAAS;;wBAYK,YAAY,iBAAiB;;;KC5zBjC;EACV;;EAEA;;;wBAIc,QAAQ,UAAS,eAAoB"}
+{"version":3,"file":"index.d.mts","names":[],"sources":["../../core/src/identity.ts","../../core/src/dataset.ts","../../core/src/aut-events.ts","../../core/src/catalog.ts","../../core/src/canonical-json.ts","../../core/src/matrix.ts","../../core/src/config.ts","../../core/src/json.ts","../../core/src/definition-site.ts","../../core/src/schema.ts","../../core/src/trajectory.ts","../../core/src/index.ts","../../agents/src/index.ts"],"mappings":";;qBAEa,oBAAkB,OAAA;YACnB,sBAAsB,mBAAmB;;wBAGrC,SAAS;;qBAOZ,YAAU,OAAA,cAAA,OAAA;YAKX,cAAc,WAAW;YAEzB,YAAY,cAAc,eAAe,2BACxC;qBAEA,mBAAiB,OAAA,cAAA,OAAA;wBAMd,kBAAkB,cAAc,cAAc,MAAM,QAAK,OAAA,cAAA,OAAA;wBAWzD,YAAY,cAAc,cACxC,MAAM,OACN,MAAM,OACL,YAAY;wBAIC,iBAAiB,cAAc,cAC7C,eACA,eAAe;EACZ,MAAM;EAAO,MAAM;EAAM,KAAK,YAAY;;wBAY/B,aAAa,eAAe,eAAe,eAAe;;;;YCvD9D;;YAGA,6BAA6B;EAAe;;;;;;YAM5C,wBAAwB,SAAS,eAAe;;YAGhD;EACV,QAAQ;;EAER;;YAGU;;EAEV;EACA,QAAQ;;YAGE;EACV;EACA;EACA;EACA;;EAEA;;EAEA,YAAY;EACZ,QAAQ;;KAGL;EACH;EACA;;EAEA,aAAa;;EAEb;;YAGU,0BAA0B;EACpC,QAAQ;;EAER;;EAEA;EACA;EACA;;YAGU,0BAA0B,gBAAgB,kBACpD;;EAEE,SAAS,MAAM,uBAAuB,UAAU,QAAQ;;EAExD,SACE,QAAQ,SACR,OAAO,wBACJ,iBAAiB,QAAQ;EAC9B;EACA;EACA;;YAGQ;EAEN;EACA,QAAQ;EACR;EACA;;EAGA;EACA,SACE,MAAM,uBACH,iBAAiB,QAAQ;EAC9B,SACE,QAAQ,gBACR,OAAO,wBACJ,iBAAiB,QAAQ;;;YAIxB;WACD;WACA;WACA;WACA,YAAY;;WAEZ;WACA,QAAQ;;;wBAIH,oBAAoB,eAAe;;;;;;;wBAuBnC,cAAc,gBAAgB,gBAC5C,YAAY,0BAA0B,0BAA0B,WAC/D;wBAmFa,UAAU,iBAAiB,SAAS;;;;;;;YC/MxC;EACN;EAAiB;;EAEjB;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB;EAAkB;;EACrC;EAAe,OAAO;EAAe;;YAE/B;EACV;EACA;;EAEA;EACA;;;YAIU;EAGN;EACA,qBAAqB;EACrB,QAAQ;;;wBAIE,kBACd,OAAO,eACP,WAAW,OACV;;;;;wBA0Da,iBACd,QAAQ,YACR,cACA,WAAW,OACV;;;;;;;;;YCtGS;;EAEV;;YAGU;EACV;EACA;EACA;EACA;EACA,UAAU;IAAQ,MAAM;IAAkB;;;EAE1C;;;YAIU;EACN;EAAe;;EACf;EAAgB;;EAChB;;YAEM;EACV;EACA;EACA,YAAY;EACZ;EACA;;EAEA;IAAU;;IAAwB;;;EAElC;IAAU,OAAO;;IAAiB;;;EAElC;IAAU;IAAc;;EACxB;;YAGU;EACV;EACA,MAAM;EACN;EACA;;EAEA;EACA;;YAGU;;;EAEN;EAAc;;EACd;EAAe,aAAa;;EAC5B;EAA0B;EAAc,YAAY;;;;EAEpD;EAAoB,QAAQ;;YAEtB;EACV;EACA;EACA;EACA;EACA,OAAO;;EAEP,aAAa;EACb,UAAU;EACV;EACA;IAAU;IAAgB;IAAoB;;EAC9C,YAAY;EACZ,SAAS;EACT,WAAW;;EAEX;;wBAGc,aACd,YAAY,gBACZ,OAAO,uBACP,UAAS,iBACR;;;;wBC5Fa,oBAAoB,OAAO;;;KCK/B;;EAEV;;EAEA,aAAa,SAAS,wBAAwB;;EAE9C,YAAY;;KAEF;EACV;;EAEA;EACA,MAAM;EACN,YAAY;;KAEF;EACV;EACA;EACA,gBAAgB;EAChB,YAAY,SAAS,wBAAwB;EAC7C,WAAW;;EAEX,iBAAiB;;EAEjB,mBAAmB;;KAET,aAAa;EACvB;EACA,MAAM,YAAY;EAClB,MAAM,YAAY,kBAAkB,iBAAiB;;wBAWvC,uBAAuB,UAAU,sBAC/C,YAAY,IACX,aAAa;;;KCjDJ;;EAEV;EACA;EACA;EACA;;IAEE;;;EAGF,iBAAiB;EACjB,WAAW;;EAEX,SAAS,KAAK;IAA0C;;EACxD;IAAc;IAAsB;IAAiB;;EACrD;;EAEA;;;;;;;IAOE;;;EAGF;;EAEA,gBAAgB;EAChB,QAAQ;;KAGE;;qBAGC;;KAWD,eAAe;KAEf;;EAEV;;;;;;EAMA,UAAU,sBAAsB,eAAe,QAAQ;;;wBAIzC,mBAAmB,UAAU,eAC3C,eAAe,IACd;;;;qBC/DU,qBAAqB;WAGrB;WACA;WAHO;EAClB,YACW,cACA;;;;;;;;wBAYG,OAAO,gBAAgB,gBAAiB;;wBA4CxC,aACd,gBACA;GACI,cAAc;;;;;;;;;;;KC3DR;EAAmB;EAAc;;wBAqB7B,eAAe,iBAAiB;;;;qBCvBnC,iBAAe,OAAA;qBAOf,qBAAmB,OAAA;;;;;qBAMnB,mBAAiB,OAAA;;;;;;qBAQjB,iBAAiB,OAAO,OAAO;qBAoB/B,iBAAe,OAAA;;;;;;;;;;;;;;qBAiBf,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAejB,qBAAmB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBA2BnB,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAIjB,qBAAmB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAKnB,aAAW,OAAA;;;;;qBAMX,oBAAkB,OAAA;;;;;;;;;;;;;;qBAalB,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAYjB,wBAAsB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAgBtB,oBAAkB,OAAA;;;;;;;;;;;;qBAQlB,oBAAkB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAUlB,qBAAmB,OAAA;;;;;qBAKnB,oBAAkB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBASlB,kBAAgB,OAAA;;;;;;;;;;;;;;qBAmJhB,uBAAuB,OAAO,OAAO;qBAGrC,gBAAc,OAAA;;;;;;;wBClVX,qBAAqB,mBAAmB;;;YCgB5C;YACA,YACV,gBAAgB;GAAiB,cAAc;;YACrC;GAAgB,cAAc;;YAE9B;;EAEV;EACA;EACA;EACA;;YAGU;;;;;YAMA;EACV;EACA,gBAAgB;;YAGN,gBAAgB,QAAQ,OAAO,kBAAkB;YAEjD;EACV;EACA;EACA;;YAGU;EACV;EACA;EACA;;YAGU;EACN;EAAiB;;EAEjB;EACA;EACA,SAAS;EACT;;EAGA;EACA;EACA;EACA,WAAW;EACX;;EAGA;EACA;EACA;EACA,QAAQ;EACR;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB,SAAS;EAAW;;EACvC;EAAe,OAAO;EAAe;;YAE/B;EACN;EAAuB;;EAEvB;EACA;;EAEA;EACA;;EAEA;EAAmC;EAAc;;EACjD;EAAiC;EAAc;;EAE/C;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAwB;EAAgB;;EAExC;EACA;;EAEA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAyB;;EACzB;EAAe,OAAO;EAAe;;YAE/B,mBACT;EAAa;MAAoB;EAAgB;;YAExC;;EAEV;;YAGU,qBAAqB;YACrB,qBAAqB;YAErB;EACV;EACA;EACA;EACA;EACA,UAAU;;EAEV,aAAa;EACb,UAAU;;EAEV,MAAM;;EAEN,WAAW;;EAEX,oBAAoB;;YAGV,WAAW;WACZ,WAAW;;WAEX,WAAW;EACpB,MAAM;IACJ,SAAS;IACT,UAAU;;IAEV,UAAU,OAAO,yBAAyB;MACxC,QAAQ,WAAW;;YAGb,WAAW;;;;;EAKrB,KAAK,SAAS,WAAW,eAAe;;EAExC,UAAU;;;qBAIC,wBAAwB;;wBAKrB,cAAc,OAAO,kBAAkB;;;;;wBAUvC,kBAAkB,UAAU,YAAY,OAAO,IAAI;;YAMvD;;EAEV,YAAY;;EAEZ,MAAM;;EAEN;EACA;;YAGU;EACV;;EAEA,oBAAoB,OAAO;;YAGjB;EACV;EACA,aAAa;;YAGH;EACV;EACA,kBAAkB;IAAa;;EAC/B,6BAA6B,QAAQ;IAAY;;IAC/C;IACA;;EAEF;IACE;IACA;IACA;IACA,WAAW;IACX;IACA,SAAS;;EAEX;;YAGU;EACV;EACA;EACA;IAAY;IAAc,YAAY;;;YAE5B,iBACV,WAAW,YAAY,cAAc;;wBAGvB,eACd,UAAU,iCACT;;;;;wBAUa,KAAK,SAAS,sBAAsB;wBAIpC,MAAM,aAAa,aAAa;YAIpC;EACV,QAAQ;EACR,QAAQ;;EAER,SAAS;;;YAIC;EAAiB;EAAe;;YAEhC,sBAER;EAEE;EACA;EACA;;EAEA;;EAEA,QAAQ;;;;;;wBAOE,KAAK,iBAAiB;YAK1B;EACV;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;;EAER;;YAGU;EACV,WAAW;EACX,WAAW;;YAGD;EACV,SAAS;EACT;IAAc,iBAAiB;;EAC/B,WAAW;;EAEX,OAAO;;EAEP,SAAS;;EAET,MAAM;;;;;;EAMN,sBAAsB;;YAGZ;EACV;EACA;EACA;;EAEA;EACA,IACE,SAAS,2BACE,aAAa,kBAAkB;;YAGlC;EACV;EACA;EACA;;EAEA;;YAGU,cAAc,kBAAkB;wBAE5B,UACd,cACA,KAAK,wBACL,UAAS,KAAK,iDACb;;;;;;YAqBS;EACV;EACA;;EAEA;;EAEA;EACA,IACE,SAAS,sCACa;;wBAGV,OACd,cACA,KAAK,eACL,UAAS,KAAK,6BACb;;YAMS;EACV;;EAEA;EACA;EACA;;EAEA;EACA,QAAQ;;wBAGM,MACd,cACA,SAAS,KAAK,2CACb;YAMS;EACV;;EAEA;;EAEA;;YAGU,eAAe,eAAe,aAAa;EACrD;EACA;EACA,OAAO;;EAEP,QAAQ;;EAER,oBAAoB;EACpB,YAAY;EACZ,SAAS;;EAET,mBAAmB;EACnB,SAAS;;EAET,WAAW;EACX,YAAY;;;YAIF;;;;;;;;;EASV;;wBAGc,iBAAiB,UAAU,gBAAgB,YAAY,IAAI;YA0D/D,UACV,wBAAwB,4BAA4B;EAEpD;EACA;EACA,OAAO;;wBAGO,kBAAkB,eAAe,WAC/C,OAAO,SACN;YAKS,mBAAmB,iBAAiB,YAAY;wBAE5C,YAAY;EAAS;;YAkBzB;;EAEV,gBAAgB;EAChB,iBAAiB;EACjB,mBAAmB;EACnB,IAAI,cAAc;EAClB,SAAS,cAAc;EACvB,YAAY;EACZ,iBAAiB;;EAEjB,QAAQ,UAAU,iBAAiB;;YAGzB;EACV;EACA;EACA;;YAGU;EACV;EACA;EACA;;;;;;wBAOc,oBACR,gCAAgC,oBACtC,eAAe,iBAAiB;YAuEtB;;;;;YAMA;;EAEV;;EAEA;;EAEA;EACA;EACA;;EAEA;EACA;IAAO;IAAkB;IAAmB;IAAc;;;;YAIhD;EACV,SAAS;EACT,OAAO;EACP,QAAQ;;;YAIE;EAAe;EAAa;;YAE5B;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,SAAS;;EAET,UAAU;;YAGA;EACV;EACA;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA;;EAEA,WAAW;;YAGD;EACV;EACA,YAAY;EACZ;;EAEA;;YAGU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;EACR;IAAoB;IAAoB;;;;qBAI7B;YAMD;EACV,SAAS;EACT,UAAU;EACV,cAAc;EACd;EACA;EACA;;YAGU;;EAEV;EACA;EACA;;YAGU;EACV,QAAQ;EACR;EACA;EACA,UAAU;EACV,YAAY;EACZ,QAAQ;;YAGE;EACV,QAAQ;EACR;EACA;EACA;EACA;EACA;;EAEA;EACA,QAAQ;;YAGE;EACV,YAAY,OAAO,kBAAkB;EACrC,YAAY,SAAS,eAAe;;EAEpC,cAAc,cAAc,MAAM,aAAa;EAC/C,SAAS,SAAS,eAAe;;YAGvB;WACD;EACT,WAAW,UAAU,gBAAgB,QAAQ;EAC7C,SAAS,SAAS,aAAa;;YAGrB;EACV,SACE,UAAU,aACV;IAAY,UAAU;MACrB,QAAQ;;YAGD;EACV;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,UAAU;EACV,QAAQ;;YAGE;EACV;EACA;EACA;EACA;EACA;;YAGU,YAAY;EACtB;EACA;EACA;EACA;EACA,mBAAmB;;EAEnB,SAAS;;wBAYK,YAAY,iBAAiB;;;KCj1BjC;EACV;;EAEA;;;wBAIc,QAAQ,UAAS,eAAoB"}

package/dist/index.mjs

@@ -1,4 +1,4 @@
-import { $ as ResourceUriSchema, A as RunSummarySchema, B as defineEvalMatrix, C as JudgeRunInfoSchema, D as RunMetadataSchema, E as RunManifestSchema, F as TrialScoringSchema, G as NotJsonError, H as DEFAULT_ARTIFACT_EXCLUDE, I as TrialSummarySchema, J as defineDataset, K as toJson, L as UsageSchema, M as TrajectoryEventSchema, N as TrialManifestSchema, O as RunSourceSchema, P as TrialMetadataSchema, Q as ResourceKindSchema, R as normalizeAutEvent, S as JsonValueSchema, T as RecordedErrorSchema, U as defineConfig, V as canonicalParameters, W as catalogEntry, X as relativeDatasetPath, Y as isDataset, Z as definitionSite, _ as parseTrajectoryJsonl, a as authoringId, at as resourceUuid, b as AutIdentitySchema, c as defineSuite, d as metric, et as UuidSchema, f as predicate, g as user, h as skip, i as agentRuntimes, it as resourceUriSchema, j as ScoreResultSchema, k as RunStatusSchema, l as expectToolCall, m as registerEvals, n as DEFAULT_AGENT_RUNTIMES, nt as parseResourceUri, o as defineAgent, p as recordError, q as toJsonObject, r as agent, rt as resourceUri, s as defineEval, t as BUILT_IN_METRICS, tt as authorId, u as judge, v as ApiErrorSchema, w as MetricResultSchema, x as CheckpointResultSchema, y as ArtifactEntrySchema, z as turnResultEvents } from "./src-b2Wnmzbb.mjs";
+import { $ as ResourceUriSchema, A as RunSummarySchema, B as defineEvalMatrix, C as JudgeRunInfoSchema, D as RunMetadataSchema, E as RunManifestSchema, F as TrialScoringSchema, G as NotJsonError, H as DEFAULT_ARTIFACT_EXCLUDE, I as TrialSummarySchema, J as defineDataset, K as toJson, L as UsageSchema, M as TrajectoryEventSchema, N as TrialManifestSchema, O as RunSourceSchema, P as TrialMetadataSchema, Q as ResourceKindSchema, R as normalizeAutEvent, S as JsonValueSchema, T as RecordedErrorSchema, U as defineConfig, V as canonicalParameters, W as catalogEntry, X as relativeDatasetPath, Y as isDataset, Z as definitionSite, _ as parseTrajectoryJsonl, a as authoringId, at as resourceUuid, b as AutIdentitySchema, c as defineSuite, d as metric, et as UuidSchema, f as predicate, g as user, h as skip, i as agentRuntimes, it as resourceUriSchema, j as ScoreResultSchema, k as RunStatusSchema, l as expectToolCall, m as registerEvals, n as DEFAULT_AGENT_RUNTIMES, nt as parseResourceUri, o as defineAgent, p as recordError, q as toJsonObject, r as agent, rt as resourceUri, s as defineEval, t as BUILT_IN_METRICS, tt as authorId, u as judge, v as ApiErrorSchema, w as MetricResultSchema, x as CheckpointResultSchema, y as ArtifactEntrySchema, z as turnResultEvents } from "./src-LeGSyDD2.mjs";
 //#region ../agents/src/index.ts
 /** A local, one-prompt-per-turn AUT backed by Pi's non-interactive print mode. */
 function piAgent(options = {}) {

package/dist/runner.d.mts

@@ -642,6 +642,19 @@ type EvalDefinition<TAgent extends AutAdapter = AutAdapter> = {
   policy?: EvalPolicy;
   /** Any JSON-convertible object; converted when the eval is defined. */
   metadata?: Record<string, unknown>;
+  workspace?: EvalWorkspace;
+};
+/** How EvalKit treats this scenario's candidate workspace. */
+type EvalWorkspace = {
+  /**
+   * Paths to leave out of the trial's snapshot, read like `.gitignore`: globs
+   * relative to the workspace, where a pattern without `/` matches that name
+   * at any depth and later patterns win. They come after the project's
+   * `artifacts.exclude` (default `DEFAULT_ARTIFACT_EXCLUDE`), so `!pattern`
+   * can keep a path the project leaves out, even inside an ignored directory:
+   * `['!_build/doc/**']` keeps generated docs while dropping the rest of `_build`.
+   */
+  ignore?: readonly string[];
 };
 type EvalRegistryMetadata = {
   id: string;
@@ -720,6 +733,8 @@ type TrialMetadata = {
   };
   aut?: AutIdentity;
   startedAt: string;
+  /** The effective snapshot ignore patterns: the project's `artifacts.exclude`, then the scenario's `workspace.ignore`. */
+  workspaceIgnore?: string[];
   /** From schema version 5: the datasets this trial used, with content digests. */
   datasets?: TrialDataset[];
 };
@@ -920,6 +935,7 @@ export declare const readTrialManifest: (root: string, runId: string, trialId: s
     readonly version?: string | undefined;
   } | undefined;
   readonly startedAt: string;
+  readonly workspaceIgnore?: string[] | undefined;
   readonly datasets?: {
     readonly id: string;
     readonly visibility: "candidate" | "evaluator";
@@ -1152,7 +1168,10 @@ export type RunEvalOptions = {
   workspaceRoot?: string;
   /** With `workspaceRoot`: which trial workspaces to keep afterwards. Default `'failed'`. */
   keepWorkspaces?: KeepSandboxes;
-  /** Candidate paths left out of the artifact snapshot. Default `DEFAULT_ARTIFACT_EXCLUDE`. */
+  /**
+   * Candidate paths left out of the artifact snapshot, before the scenario's
+   * own `workspace.ignore`. Default `DEFAULT_ARTIFACT_EXCLUDE`.
+   */
   artifactExclude?: readonly string[];
   /** Repository and commit the eval code came from, recorded in the manifest. */
   source?: RunSource;

package/dist/runner.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"runner.d.mts","names":["_$_evalkit_core0.JsonValue","_$_evalkit_core0.TrajectoryEvent"],"sources":["../../runner/src/errors.ts","../../core/src/identity.ts","../../core/src/dataset.ts","../../core/src/aut-events.ts","../../core/src/catalog.ts","../../core/src/matrix.ts","../../core/src/config.ts","../../core/src/index.ts","../../runner/src/local-report-store.ts","../../runner/src/datasets.ts","../../runner/src/local-report-reader.ts","../../runner/src/matrix.ts","../../runner/src/workspace.ts","../../runner/src/index.ts"],"mappings":";;;;;;qBAEa,0BAA0B;EACrC;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBAGW,oBAAoB;EAC/B;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBASW,iCAAiC;EAG5C;EACA;EACA;EACA;;;;;;qBAIW,uBAAuB;EAClC;;;;cCrCW,oBAAkB,OAAA;KACnB,sBAAsB,mBAAmB;KAiBzC,YAAY,cAAc,eAAe,2BACxC;;;;KCdD;;KAGA,6BAA6B;EAAe;;;;;;KAM5C,wBAAwB,SAAS,eAAe;;KAGhD;EACV,QAAQ;;EAER;;KAGU;;EAEV;EACA,QAAQ;;KAGE;EACV;EACA;EACA;EACA;;EAEA;;EAEA,YAAY;EACZ,QAAQ;;KAoCE;EAEN;EACA,QAAQ;EACR;EACA;;EAGA;EACA,SACE,MAAM,uBACH,iBAAiB,QAAQ;EAC9B,SACE,QAAQ,gBACR,OAAO,wBACJ,iBAAiB,QAAQ;;;KAIxB;WACD;WACA;WACA;WACA,YAAY;;WAEZ;WACA,QAAQ;;;;;;;;KC/FP;EACN;EAAiB;;EAEjB;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB;EAAkB;;EACrC;EAAe,OAAO;EAAe;;KAE/B;EACV;EACA;;EAEA;EACA;;;KAIU;EAGN;EACA,qBAAqB;EACrB,QAAQ;;;;KC5BF;EACV;EACA;EACA;EACA;EACA,UAAU;IAAQ,MAAM;IAAkB;;;EAE1C;;;KAIU;EACN;EAAe;;EACf;EAAgB;;EAChB;;KAEM;EACV;EACA;EACA,YAAY;EACZ;EACA;;EAEA;IAAU;;IAAwB;;;EAElC;IAAU,OAAO;;IAAiB;;;EAElC;IAAU;IAAc;;EACxB;;KAGU;EACV;EACA,MAAM;EACN;EACA;;EAEA;EACA;;KAGU;;;EAEN;EAAc;;EACd;EAAe,aAAa;;EAC5B;EAA0B;EAAc,YAAY;;;;EAEpD;EAAoB,QAAQ;;KAEtB;EACV;EACA;EACA;EACA;EACA,OAAO;;EAEP,aAAa;EACb,UAAU;EACV;EACA;IAAU;IAAgB;IAAoB;;EAC9C,YAAY;EACZ,SAAS;EACT,WAAW;;EAEX;;;;KChFU;;EAEV;;EAEA,aAAa,SAAS,wBAAwB;;EAE9C,YAAY;;KAEF;EACV;;EAEA;EACA,MAAM;EACN,YAAY;;KAEF;EACV;EACA;EACA,gBAAgB;EAChB,YAAY,SAAS,wBAAwB;EAC7C,WAAW;;EAEX,iBAAiB;;EAEjB,mBAAmB;;KAET,aAAa;EACvB;EACA,MAAM,YAAY;EAClB,MAAM,YAAY,kBAAkB,iBAAiB;;;;KCJ3C;;;KCdA;KACA,YACV,gBAAgB;GAAiB,cAAc;;KACrC;GAAgB,cAAc;;KAE9B;;EAEV;EACA;EACA;EACA;;KAGU;;;;;KAMA;EACV;EACA,gBAAgB;;KAGN,gBAAgB,QAAQ,OAAO,kBAAkB;KAEjD;EACV;EACA;EACA;;KAGU;EACV;EACA;EACA;;KAGU;EACN;EAAiB;;EAEjB;EACA;EACA,SAAS;EACT;;EAGA;EACA;EACA;EACA,WAAW;EACX;;EAGA;EACA;EACA;EACA,QAAQ;EACR;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB,SAAS;EAAW;;EACvC;EAAe,OAAO;EAAe;;KAE/B;EACN;EAAuB;;EAEvB;EACA;;EAEA;EACA;;EAEA;EAAmC;EAAc;;EACjD;EAAiC;EAAc;;EAE/C;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAwB;EAAgB;;EAExC;EACA;;EAEA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAyB;;EACzB;EAAe,OAAO;EAAe;;KAE/B,mBACT;EAAa;MAAoB;EAAgB;;KAExC;;EAEV;;KAGU,qBAAqB;KACrB,qBAAqB;KAErB;EACV;EACA;EACA;EACA;EACA,UAAU;;EAEV,aAAa;EACb,UAAU;;EAEV,MAAM;;EAEN,WAAW;;EAEX,oBAAoB;;KAGV,WAAW;WACZ,WAAW;;WAEX,WAAW;EACpB,MAAM;IACJ,SAAS;IACT,UAAU;;IAEV,UAAU,OAAO,yBAAyB;MACxC,QAAQ,WAAW;;KAGb,WAAW;;;;;EAKrB,KAAK,SAAS,WAAW,eAAe;;EAExC,UAAU;;;KAyBA;;EAEV,YAAY;;EAEZ,MAAM;;EAEN;EACA;;KAGU;EACV;;EAEA,oBAAoB,OAAO;;KAGjB;EACV;EACA,aAAa;;KAGH;EACV;EACA,kBAAkB;IAAa;;EAC/B,6BAA6B,QAAQ;IAAY;;IAC/C;IACA;;EAEF;IACE;IACA;IACA;IACA,WAAW;IACX;IACA,SAAS;;EAEX;;KAGU;EACV;EACA;EACA;IAAY;IAAc,YAAY;;;KAE5B,iBACV,WAAW,YAAY,cAAc;KAuB3B;EACV,QAAQ;EACR,QAAQ;;EAER,SAAS;;;KAIC;EAAiB;EAAe;;KAEhC,sBAER;EAEE;EACA;EACA;;EAEA;;EAEA,QAAQ;;KAYF;EACV;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;;EAER;;KAGU;EACV,WAAW;EACX,WAAW;;KAGD;EACV,SAAS;EACT;IAAc,iBAAiB;;EAC/B,WAAW;;EAEX,OAAO;;EAEP,SAAS;;EAET,MAAM;;;;;;EAMN,sBAAsB;;KAGZ;EACV;EACA;EACA;;EAEA;EACA,IACE,SAAS,2BACE,aAAa,kBAAkB;;KAGlC;EACV;EACA;EACA;;EAEA;;KAGU,cAAc,kBAAkB;;;;;;KA2BhC;EACV;EACA;;EAEA;;EAEA;EACA,IACE,SAAS,sCACa;;;KAad;EACV;;EAEA;EACA;EACA;;EAEA;EACA,QAAQ;;KAYE;EACV;;EAEA;;EAEA;;KAGU,eAAe,eAAe,aAAa;EACrD;EACA;EACA,OAAO;;EAEP,QAAQ;;EAER,oBAAoB;EACpB,YAAY;EACZ,SAAS;;EAET,mBAAmB;EACnB,SAAS;;EAET,WAAW;;KAwGD;EACV;EACA;EACA;;KAGU;EACV;EACA;EACA;;KAgFU;;;;;KAMA;;EAEV;;EAEA;;EAEA;EACA;EACA;;EAEA;EACA;IAAO;IAAkB;IAAmB;IAAc;;;;KAIhD;EACV,SAAS;EACT,OAAO;EACP,QAAQ;;;KAIE;EAAe;EAAa;;KAE5B;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,SAAS;;EAET,UAAU;;KAGA;EACV;EACA;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,WAAW;;KAGD;EACV;EACA,YAAY;EACZ;;EAEA;;KAGU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;EACR;IAAoB;IAAoB;;;KAU9B;EACV,SAAS;EACT,UAAU;EACV,cAAc;EACd;EACA;EACA;;KAGU;;EAEV;EACA;EACA;;KAGU;EACV,QAAQ;EACR;EACA;EACA,UAAU;EACV,YAAY;EACZ,QAAQ;;KAGE;EACV,QAAQ;EACR;EACA;EACA;EACA;EACA;;EAEA;EACA,QAAQ;;KAGE;EACV,YAAY,OAAO,kBAAkB;EACrC,YAAY,SAAS,eAAe;;EAEpC,cAAc,cAAc,MAAM,aAAa;EAC/C,SAAS,SAAS,eAAe;;KAGvB;WACD;EACT,WAAW,UAAU,gBAAgB,QAAQ;EAC7C,SAAS,SAAS,aAAa;;KAGrB;EACV,SACE,UAAU,aACV;IAAY,UAAU;MACrB,QAAQ;;KAGD;EACV;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,UAAU;EACV,QAAQ;;KAGE;EACV;EACA;EACA;EACA;EACA;;KAGU,YAAY;EACtB;EACA;EACA;EACA;EACA,mBAAmB;;EAEnB,SAAS;;;;;wBCvrBK,iBAAiB,wBAAwB;;;;KCiI7C;EAAqB;EAAc;;;;;;;wBAOzB,mBACpB,SAAS,SACT,eACC,QAAQ;;wBA6BW,kBACpB,SAAS,SACT,cACA,eACC,QAAQ,WAAW;;;qBC3RT,kBAAe,cAAgB,kBAAe;;;;;;0BAA9CA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAEA,iBAAc,cAAgB,kBAAe;;;;;;;;;;;;;;qBAE7C,oBAAiB,cAChB,eACC,oBACE;;;;;;;0BAHJA;;;;;;;;;;;;;;;;;;;;;qBASA,mBAAgB,cACf,eACC,oBACE;;;;;;;;;;;0BAHJA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBASA,mBAAgB,cACf,eACC,oBACE;;;;;;;wBAHJA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;wBASS,gBACpB,cACA,eACA,kBAAe,QAHKC;;;;;wBAiBN,YAAY,OAAO;;;;KC7DvB;EACV;EACA;EACA;EACA;;;wBAIc,UACd,QAAQ,YACR,SAAS,KACP;EAGA,YAAY;;EAEZ,YAAY,MAAM;EAClB,YACE,MAAM,gBACN,QAAQ,qBACE;IAEb,OAAO,OAAO;;;;wBChBD,mBAAmB,cAAc;;;YCyDrC;EACV,QAAQ;EACR;EACA;EACA;;EAEA;;EAEA,aAAa;;EAEb,SAAS;;EAET,UAAU;;EAEV;;EAEA,YAAY,OAAO;;EAEnB;;EAEA,iBAAiB;;EAEjB;;EAEA,SAAS;;EAET,UAAU;;EAEV;;;;;EAKA,SAAS;;;;;EAKT,WAAW,OAAO,UAAU,OAAO;EACnC,YAAY;;;YAIF;EACV;EACA;EACA;EACA;EACA;EACA,aAAa;;;wBASC,QACd,YAAY,gBACZ,SAAS,iBACR,OAAO,OAAO"}
+{"version":3,"file":"runner.d.mts","names":["_$_evalkit_core0.JsonValue","_$_evalkit_core0.TrajectoryEvent"],"sources":["../../runner/src/errors.ts","../../core/src/identity.ts","../../core/src/dataset.ts","../../core/src/aut-events.ts","../../core/src/catalog.ts","../../core/src/matrix.ts","../../core/src/config.ts","../../core/src/index.ts","../../runner/src/local-report-store.ts","../../runner/src/datasets.ts","../../runner/src/local-report-reader.ts","../../runner/src/matrix.ts","../../runner/src/workspace.ts","../../runner/src/index.ts"],"mappings":";;;;;;qBAEa,0BAA0B;EACrC;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBAGW,oBAAoB;EAC/B;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBASW,iCAAiC;EAG5C;EACA;EACA;EACA;;;;;;qBAIW,uBAAuB;EAClC;;;;cCrCW,oBAAkB,OAAA;KACnB,sBAAsB,mBAAmB;KAiBzC,YAAY,cAAc,eAAe,2BACxC;;;;KCdD;;KAGA,6BAA6B;EAAe;;;;;;KAM5C,wBAAwB,SAAS,eAAe;;KAGhD;EACV,QAAQ;;EAER;;KAGU;;EAEV;EACA,QAAQ;;KAGE;EACV;EACA;EACA;EACA;;EAEA;;EAEA,YAAY;EACZ,QAAQ;;KAoCE;EAEN;EACA,QAAQ;EACR;EACA;;EAGA;EACA,SACE,MAAM,uBACH,iBAAiB,QAAQ;EAC9B,SACE,QAAQ,gBACR,OAAO,wBACJ,iBAAiB,QAAQ;;;KAIxB;WACD;WACA;WACA;WACA,YAAY;;WAEZ;WACA,QAAQ;;;;;;;;KC/FP;EACN;EAAiB;;EAEjB;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB;EAAkB;;EACrC;EAAe,OAAO;EAAe;;KAE/B;EACV;EACA;;EAEA;EACA;;;KAIU;EAGN;EACA,qBAAqB;EACrB,QAAQ;;;;KC5BF;EACV;EACA;EACA;EACA;EACA,UAAU;IAAQ,MAAM;IAAkB;;;EAE1C;;;KAIU;EACN;EAAe;;EACf;EAAgB;;EAChB;;KAEM;EACV;EACA;EACA,YAAY;EACZ;EACA;;EAEA;IAAU;;IAAwB;;;EAElC;IAAU,OAAO;;IAAiB;;;EAElC;IAAU;IAAc;;EACxB;;KAGU;EACV;EACA,MAAM;EACN;EACA;;EAEA;EACA;;KAGU;;;EAEN;EAAc;;EACd;EAAe,aAAa;;EAC5B;EAA0B;EAAc,YAAY;;;;EAEpD;EAAoB,QAAQ;;KAEtB;EACV;EACA;EACA;EACA;EACA,OAAO;;EAEP,aAAa;EACb,UAAU;EACV;EACA;IAAU;IAAgB;IAAoB;;EAC9C,YAAY;EACZ,SAAS;EACT,WAAW;;EAEX;;;;KChFU;;EAEV;;EAEA,aAAa,SAAS,wBAAwB;;EAE9C,YAAY;;KAEF;EACV;;EAEA;EACA,MAAM;EACN,YAAY;;KAEF;EACV;EACA;EACA,gBAAgB;EAChB,YAAY,SAAS,wBAAwB;EAC7C,WAAW;;EAEX,iBAAiB;;EAEjB,mBAAmB;;KAET,aAAa;EACvB;EACA,MAAM,YAAY;EAClB,MAAM,YAAY,kBAAkB,iBAAiB;;;;KCH3C;;;KCfA;KACA,YACV,gBAAgB;GAAiB,cAAc;;KACrC;GAAgB,cAAc;;KAE9B;;EAEV;EACA;EACA;EACA;;KAGU;;;;;KAMA;EACV;EACA,gBAAgB;;KAGN,gBAAgB,QAAQ,OAAO,kBAAkB;KAEjD;EACV;EACA;EACA;;KAGU;EACV;EACA;EACA;;KAGU;EACN;EAAiB;;EAEjB;EACA;EACA,SAAS;EACT;;EAGA;EACA;EACA;EACA,WAAW;EACX;;EAGA;EACA;EACA;EACA,QAAQ;EACR;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB,SAAS;EAAW;;EACvC;EAAe,OAAO;EAAe;;KAE/B;EACN;EAAuB;;EAEvB;EACA;;EAEA;EACA;;EAEA;EAAmC;EAAc;;EACjD;EAAiC;EAAc;;EAE/C;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAwB;EAAgB;;EAExC;EACA;;EAEA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAyB;;EACzB;EAAe,OAAO;EAAe;;KAE/B,mBACT;EAAa;MAAoB;EAAgB;;KAExC;;EAEV;;KAGU,qBAAqB;KACrB,qBAAqB;KAErB;EACV;EACA;EACA;EACA;EACA,UAAU;;EAEV,aAAa;EACb,UAAU;;EAEV,MAAM;;EAEN,WAAW;;EAEX,oBAAoB;;KAGV,WAAW;WACZ,WAAW;;WAEX,WAAW;EACpB,MAAM;IACJ,SAAS;IACT,UAAU;;IAEV,UAAU,OAAO,yBAAyB;MACxC,QAAQ,WAAW;;KAGb,WAAW;;;;;EAKrB,KAAK,SAAS,WAAW,eAAe;;EAExC,UAAU;;;KAyBA;;EAEV,YAAY;;EAEZ,MAAM;;EAEN;EACA;;KAGU;EACV;;EAEA,oBAAoB,OAAO;;KAGjB;EACV;EACA,aAAa;;KAGH;EACV;EACA,kBAAkB;IAAa;;EAC/B,6BAA6B,QAAQ;IAAY;;IAC/C;IACA;;EAEF;IACE;IACA;IACA;IACA,WAAW;IACX;IACA,SAAS;;EAEX;;KAGU;EACV;EACA;EACA;IAAY;IAAc,YAAY;;;KAE5B,iBACV,WAAW,YAAY,cAAc;KAuB3B;EACV,QAAQ;EACR,QAAQ;;EAER,SAAS;;;KAIC;EAAiB;EAAe;;KAEhC,sBAER;EAEE;EACA;EACA;;EAEA;;EAEA,QAAQ;;KAYF;EACV;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;;EAER;;KAGU;EACV,WAAW;EACX,WAAW;;KAGD;EACV,SAAS;EACT;IAAc,iBAAiB;;EAC/B,WAAW;;EAEX,OAAO;;EAEP,SAAS;;EAET,MAAM;;;;;;EAMN,sBAAsB;;KAGZ;EACV;EACA;EACA;;EAEA;EACA,IACE,SAAS,2BACE,aAAa,kBAAkB;;KAGlC;EACV;EACA;EACA;;EAEA;;KAGU,cAAc,kBAAkB;;;;;;KA2BhC;EACV;EACA;;EAEA;;EAEA;EACA,IACE,SAAS,sCACa;;;KAad;EACV;;EAEA;EACA;EACA;;EAEA;EACA,QAAQ;;KAYE;EACV;;EAEA;;EAEA;;KAGU,eAAe,eAAe,aAAa;EACrD;EACA;EACA,OAAO;;EAEP,QAAQ;;EAER,oBAAoB;EACpB,YAAY;EACZ,SAAS;;EAET,mBAAmB;EACnB,SAAS;;EAET,WAAW;EACX,YAAY;;;KAIF;;;;;;;;;EASV;;KA6GU;EACV;EACA;EACA;;KAGU;EACV;EACA;EACA;;KAgFU;;;;;KAMA;;EAEV;;EAEA;;EAEA;EACA;EACA;;EAEA;EACA;IAAO;IAAkB;IAAmB;IAAc;;;;KAIhD;EACV,SAAS;EACT,OAAO;EACP,QAAQ;;;KAIE;EAAe;EAAa;;KAE5B;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA,SAAS;;EAET,UAAU;;KAGA;EACV;EACA;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;EAEA;;EAEA,WAAW;;KAGD;EACV;EACA,YAAY;EACZ;;EAEA;;KAGU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;EACR;IAAoB;IAAoB;;;KAU9B;EACV,SAAS;EACT,UAAU;EACV,cAAc;EACd;EACA;EACA;;KAGU;;EAEV;EACA;EACA;;KAGU;EACV,QAAQ;EACR;EACA;EACA,UAAU;EACV,YAAY;EACZ,QAAQ;;KAGE;EACV,QAAQ;EACR;EACA;EACA;EACA;EACA;;EAEA;EACA,QAAQ;;KAGE;EACV,YAAY,OAAO,kBAAkB;EACrC,YAAY,SAAS,eAAe;;EAEpC,cAAc,cAAc,MAAM,aAAa;EAC/C,SAAS,SAAS,eAAe;;KAGvB;WACD;EACT,WAAW,UAAU,gBAAgB,QAAQ;EAC7C,SAAS,SAAS,aAAa;;KAGrB;EACV,SACE,UAAU,aACV;IAAY,UAAU;MACrB,QAAQ;;KAGD;EACV;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,UAAU;EACV,QAAQ;;KAGE;EACV;EACA;EACA;EACA;EACA;;KAGU,YAAY;EACtB;EACA;EACA;EACA;EACA,mBAAmB;;EAEnB,SAAS;;;;;wBC5sBK,iBAAiB,wBAAwB;;;;KCiI7C;EAAqB;EAAc;;;;;;;wBAOzB,mBACpB,SAAS,SACT,eACC,QAAQ;;wBA6BW,kBACpB,SAAS,SACT,cACA,eACC,QAAQ,WAAW;;;qBC3RT,kBAAe,cAAgB,kBAAe;;;;;;0BAA9CA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAEA,iBAAc,cAAgB,kBAAe;;;;;;;;;;;;;;qBAE7C,oBAAiB,cAChB,eACC,oBACE;;;;;;;0BAHJA;;;;;;;;;;;;;;;;;;;;;;qBASA,mBAAgB,cACf,eACC,oBACE;;;;;;;;;;;0BAHJA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBASA,mBAAgB,cACf,eACC,oBACE;;;;;;;wBAHJA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;wBASS,gBACpB,cACA,eACA,kBAAe,QAHKC;;;;;wBAiBN,YAAY,OAAO;;;;KC7DvB;EACV;EACA;EACA;EACA;;;wBAIc,UACd,QAAQ,YACR,SAAS,KACP;EAGA,YAAY;;EAEZ,YAAY,MAAM;EAClB,YACE,MAAM,gBACN,QAAQ,qBACE;IAEb,OAAO,OAAO;;;;wBChBD,mBAAmB,cAAc;;;YC0DrC;EACV,QAAQ;EACR;EACA;EACA;;EAEA;;EAEA,aAAa;;EAEb,SAAS;;EAET,UAAU;;EAEV;;EAEA,YAAY,OAAO;;EAEnB;;EAEA,iBAAiB;;;;;EAKjB;;EAEA,SAAS;;EAET,UAAU;;EAEV;;;;;EAKA,SAAS;;;;;EAKT,WAAW,OAAO,UAAU,OAAO;EACnC,YAAY;;;YAIF;EACV;EACA;EACA;EACA;EACA;EACA,aAAa;;;wBASC,QACd,YAAY,gBACZ,SAAS,iBACR,OAAO,OAAO"}

package/dist/runner.mjs

@@ -1,2 +1,2 @@
-import { _ as CheckpointExecutionError, a as readRunSummary, b as ScoringError, c as readTrialScoring, d as trialWorkspacePath, f as staticDatasetFile, g as CancelledError, h as AutExecutionError, i as readRunManifest, l as readTrialSummary, n as runMatrix, o as readTrialEvents, p as staticDatasetFiles, r as ownerIsGone, s as readTrialManifest, t as runEval, u as localReportStore, v as DatasetError, y as ReportError } from "./src-DhromKNu.mjs";
+import { _ as CheckpointExecutionError, a as readRunSummary, b as ScoringError, c as readTrialScoring, d as trialWorkspacePath, f as staticDatasetFile, g as CancelledError, h as AutExecutionError, i as readRunManifest, l as readTrialSummary, n as runMatrix, o as readTrialEvents, p as staticDatasetFiles, r as ownerIsGone, s as readTrialManifest, t as runEval, u as localReportStore, v as DatasetError, y as ReportError } from "./src-mm8L_IxM.mjs";
 export { AutExecutionError, CancelledError, CheckpointExecutionError, DatasetError, ReportError, ScoringError, localReportStore, ownerIsGone, readRunManifest, readRunSummary, readTrialEvents, readTrialManifest, readTrialScoring, readTrialSummary, runEval, runMatrix, staticDatasetFile, staticDatasetFiles, trialWorkspacePath };

package/dist/{src-b2Wnmzbb.mjs → src-LeGSyDD2.mjs}

@@ -619,6 +619,8 @@ const TrialMetadataSchema = Schema.Struct({
 	matrix: Schema.optional(MatrixCellSchema),
 	aut: Schema.optional(AutIdentitySchema),
 	startedAt: Schema.String,
+	/** The effective snapshot ignore patterns: the project's, then the scenario's. */
+	workspaceIgnore: Schema.optional(Schema.mutable(Schema.Array(Schema.String))),
 	datasets: Schema.optional(Schema.mutable(Schema.Array(Schema.Struct({
 		id: Schema.String,
 		visibility: Schema.Literal("candidate", "evaluator"),
@@ -957,6 +959,7 @@ function defineEval(definition) {
 	authoringId(definition);
 	if ("fixtures" in definition) throw new Error(`Eval ${definition.id} uses fixtures, which were replaced by datasets: [defineDataset(...)]`);
 	if (definition.metadata !== void 0) toJsonObject(definition.metadata, `${definition.id}.metadata`);
+	for (const [index, pattern] of (definition.workspace?.ignore ?? []).entries()) if (typeof pattern !== "string" || pattern.replace(/^!/, "").trim() === "") throw new Error(`Eval ${definition.id} workspace.ignore[${index}] must be a non-empty glob, got ${JSON.stringify(pattern)}`);
 	const datasetIds = /* @__PURE__ */ new Set();
 	for (const [index, dataset] of (definition.datasets ?? []).entries()) {
 		if (!isDataset(dataset)) throw new Error(`Eval ${definition.id} datasets[${index}] is not a dataset; create it with defineDataset(...)`);

package/dist/{src-DhromKNu.mjs → src-mm8L_IxM.mjs}

@@ -1,4 +1,4 @@
-import { A as RunSummarySchema, D as RunMetadataSchema, E as RunManifestSchema, F as TrialScoringSchema, H as DEFAULT_ARTIFACT_EXCLUDE, I as TrialSummarySchema, K as toJson, M as TrajectoryEventSchema, N as TrialManifestSchema, P as TrialMetadataSchema, R as normalizeAutEvent, S as JsonValueSchema, V as canonicalParameters, X as relativeDatasetPath, _ as parseTrajectoryJsonl, a as authoringId, i as agentRuntimes, j as ScoreResultSchema, p as recordError, q as toJsonObject, rt as resourceUri, x as CheckpointResultSchema, z as turnResultEvents } from "./src-b2Wnmzbb.mjs";
+import { A as RunSummarySchema, D as RunMetadataSchema, E as RunManifestSchema, F as TrialScoringSchema, H as DEFAULT_ARTIFACT_EXCLUDE, I as TrialSummarySchema, K as toJson, M as TrajectoryEventSchema, N as TrialManifestSchema, P as TrialMetadataSchema, R as normalizeAutEvent, S as JsonValueSchema, V as canonicalParameters, X as relativeDatasetPath, _ as parseTrajectoryJsonl, a as authoringId, i as agentRuntimes, j as ScoreResultSchema, p as recordError, q as toJsonObject, rt as resourceUri, x as CheckpointResultSchema, z as turnResultEvents } from "./src-LeGSyDD2.mjs";
 import * as Schema from "effect/Schema";
 import { Data, Effect, Either, Exit, Scope } from "effect";
 import os from "node:os";
@@ -63,41 +63,82 @@ function globMatcher(pattern) {
 }
 //#endregion
 //#region ../runner/src/snapshot.ts
-/** A pattern without `/` matches a name at any depth, like `.gitignore`. */
-function excludeMatcher(patterns) {
-	const matchers = patterns.map((pattern) => {
-		const trimmed = pattern.replace(/^\.\//, "").replace(/\/$/, "");
-		const match = globMatcher(trimmed);
-		return trimmed.includes("/") ? match : (relativePath) => match(path.posix.basename(relativePath));
-	});
-	return (relativePath) => matchers.some((match) => match(relativePath));
+const GLOB_CHARS = /[*?{]/;
+function ignoreRule(pattern) {
+	const negated = pattern.startsWith("!");
+	const trimmed = (negated ? pattern.slice(1) : pattern).replace(/^\.\//, "").replace(/\/$/, "");
+	const match = globMatcher(trimmed);
+	if (!trimmed.includes("/")) return {
+		negated,
+		match: (relativePath) => match(path.posix.basename(relativePath)),
+		mayMatchBelow: () => true
+	};
+	const segments = trimmed.split("/");
+	const firstGlob = segments.findIndex((segment) => GLOB_CHARS.test(segment));
+	const prefix = segments.slice(0, firstGlob === -1 ? segments.length : firstGlob).join("/");
+	return {
+		negated,
+		match,
+		mayMatchBelow: (directory) => prefix === "" || prefix.startsWith(`${directory}/`) || prefix === directory || directory.startsWith(`${prefix}/`)
+	};
+}
+/**
+* Ignore rules for candidate snapshots, read like `.gitignore`: globs relative
+* to the workspace root, where a pattern without `/` matches that name at any
+* depth, and later patterns win. `!pattern` re-includes a path; unlike Git, it
+* also works below an ignored directory (`_build` then `!_build/doc/**`).
+*/
+function ignoreRules(patterns) {
+	const rules = patterns.map(ignoreRule);
+	return {
+		/** Whether a path is ignored, given whether its directory was. */
+		ignored(relativePath, parentIgnored = false) {
+			let ignored = parentIgnored;
+			for (const rule of rules) if (rule.match(relativePath)) ignored = !rule.negated;
+			return ignored;
+		},
+		/** Whether an ignored directory must still be walked for re-included paths. */
+		mayReinclude(directory) {
+			return rules.some((rule) => rule.negated && rule.mayMatchBelow(directory));
+		}
+	};
 }
 /**
 * Copies the candidate workspace's regular files into the trial report,
-* leaving out excluded paths. Symbolic links and special files are skipped,
-* never followed, so a snapshot cannot read outside the trial.
+* leaving out ignored paths. Symbolic links and special files are skipped,
+* never followed, so a snapshot cannot read outside the trial. An ignored
+* directory is walked only when a `!` pattern could re-include something in
+* it, and is listed only if it ends up containing a snapshotted file.
 */
-async function snapshotCandidateWorkspace(candidateRoot, writer, exclude = DEFAULT_ARTIFACT_EXCLUDE) {
+async function snapshotCandidateWorkspace(candidateRoot, writer, ignore = DEFAULT_ARTIFACT_EXCLUDE) {
 	const artifacts = [];
-	const excluded = excludeMatcher(exclude);
-	async function visit(directory, relativeDirectory) {
+	const rules = ignoreRules(ignore);
+	/** Returns whether anything below `directory` was snapshotted. */
+	async function visit(directory, relativeDirectory, directoryIgnored) {
 		const entries = await readdir(directory, { withFileTypes: true });
 		entries.sort((left, right) => left.name.localeCompare(right.name));
+		let kept = false;
 		for (const entry of entries) {
 			const absolutePath = path.join(directory, entry.name);
 			const relativePath = path.join(relativeDirectory, entry.name);
-			if (excluded(relativePath.split(path.sep).join("/"))) continue;
+			const posixPath = relativePath.split(path.sep).join("/");
+			const ignored = rules.ignored(posixPath, directoryIgnored);
 			const stat = await lstat(absolutePath);
 			if (stat.isSymbolicLink()) continue;
 			if (stat.isDirectory()) {
-				artifacts.push({
-					path: path.join("candidate", relativePath),
-					kind: "directory"
-				});
-				await visit(absolutePath, relativePath);
+				if (ignored && !rules.mayReinclude(posixPath)) continue;
+				const entryIndex = artifacts.length;
+				const keptBelow = await visit(absolutePath, relativePath, ignored);
+				if (!ignored || keptBelow) {
+					artifacts.splice(entryIndex, 0, {
+						path: path.join("candidate", relativePath),
+						kind: "directory"
+					});
+					kept = true;
+				}
 				continue;
 			}
-			if (!stat.isFile()) continue;
+			if (ignored || !stat.isFile()) continue;
 			const data = await readFile(absolutePath);
 			const artifactPath = path.join("candidate", relativePath);
 			await writer.writeArtifact(artifactPath, data);
@@ -106,9 +147,11 @@ async function snapshotCandidateWorkspace(candidateRoot, writer, exclude = DEFAU
 				kind: "file",
 				size: data.byteLength
 			});
+			kept = true;
 		}
+		return kept;
 	}
-	await visit(candidateRoot, "");
+	await visit(candidateRoot, "", false);
 	return artifacts;
 }
 //#endregion
@@ -1202,6 +1245,7 @@ async function runTrial(definition, options) {
 	}
 	const candidateData = datasetValues(prepared, "candidate");
 	const scoringData = datasetValues(prepared, "all");
+	const workspaceIgnore = [...options.artifactExclude ?? DEFAULT_ARTIFACT_EXCLUDE, ...definition.workspace?.ignore ?? []];
 	const events = [];
 	const runWriter = options.runWriter ?? await options.report.startRun(runMetadata(definition, options, runId, startedAt), { catalog: options.catalog });
 	const trialWriter = await runWriter.startTrial({
@@ -1216,6 +1260,7 @@ async function runTrial(definition, options) {
 		parameters: options.parameters,
 		matrix: options.matrix,
 		startedAt,
+		workspaceIgnore,
 		...prepared.length ? { datasets: trialDatasets(prepared) } : {}
 	});
 	let writeChain = Promise.resolve();
@@ -1381,7 +1426,7 @@ async function runTrial(definition, options) {
 		scoring = scoreSummary(scoreResults, checkpoints, skippedScorers, metrics);
 		await trialWriter.writeScores(scoring);
 		if (workspace) try {
-			artifacts = await snapshotCandidateWorkspace(workspace.artifacts.candidate.root, trialWriter, options.artifactExclude);
+			artifacts = await snapshotCandidateWorkspace(workspace.artifacts.candidate.root, trialWriter, workspaceIgnore);
 		} catch (captureError) {
 			if (!primaryError) primaryError = captureError;
 			await tryEmitError(captureError, emitRunner, now);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leostera/evalkit",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "Author and run agent evals as code",
   "type": "module",
   "license": "MIT",