@prismatic-io/lux 0.0.2-preview.3 → 0.0.2-preview.32

package/lib/core/assertion.d.ts

@@ -6,6 +6,7 @@ export declare const AssertionResultSchema: z.ZodObject<{
     score: z.ZodNumber;
     comment: z.ZodExactOptional<z.ZodString>;
     detail: z.ZodOptional<z.ZodUnknown>;
+    abstention: z.ZodExactOptional<z.ZodLiteral<"insufficient-evidence">>;
     errorMessage: z.ZodExactOptional<z.ZodString>;
     usage: z.ZodExactOptional<z.ZodObject<{
         input: z.ZodNumber;
@@ -64,19 +65,28 @@ export declare const assertionPluginIssue: (definition: {
  * Define a schema-validated assertion plugin while inferring the config passed
  * to `evaluate` from the schema.
  */
+export declare const baseAssertionFields: {
+    readonly id: z.ZodOptional<z.ZodString>;
+    readonly weight: z.ZodOptional<z.ZodNumber>;
+    readonly threshold: z.ZodOptional<z.ZodNumber>;
+    readonly name: z.ZodOptional<z.ZodString>;
+    readonly informational: z.ZodOptional<z.ZodBoolean>;
+};
 export declare const defineAssertion: <S extends z.ZodObject>(definition: AssertionPluginDefinition<S>) => AssertionPlugin<SchemaAssertion<S>, SchemaAssertionInput<S>>;
 export declare const defineAssertionRegistry: <const Plugins extends readonly AssertionPlugin[]>(...plugins: Plugins) => Plugins;
 export declare const GradingAssertionEntrySchema: z.ZodObject<{
     id: z.ZodExactOptional<z.ZodString>;
     type: z.ZodString;
     name: z.ZodString;
-    passed: z.ZodBoolean;
+    passed: z.ZodNullable<z.ZodBoolean>;
+    informational: z.ZodExactOptional<z.ZodBoolean>;
     score: z.ZodNumber;
     weight: z.ZodNumber;
     threshold: z.ZodNumber;
     durationMs: z.ZodNumber;
     comment: z.ZodExactOptional<z.ZodString>;
     detail: z.ZodOptional<z.ZodUnknown>;
+    abstention: z.ZodExactOptional<z.ZodLiteral<"insufficient-evidence">>;
     errorMessage: z.ZodExactOptional<z.ZodString>;
     usage: z.ZodExactOptional<z.ZodObject<{
         input: z.ZodNumber;
@@ -94,13 +104,15 @@ export declare const GradingReportSchema: z.ZodObject<{
         id: z.ZodExactOptional<z.ZodString>;
         type: z.ZodString;
         name: z.ZodString;
-        passed: z.ZodBoolean;
+        passed: z.ZodNullable<z.ZodBoolean>;
+        informational: z.ZodExactOptional<z.ZodBoolean>;
         score: z.ZodNumber;
         weight: z.ZodNumber;
         threshold: z.ZodNumber;
         durationMs: z.ZodNumber;
         comment: z.ZodExactOptional<z.ZodString>;
         detail: z.ZodOptional<z.ZodUnknown>;
+        abstention: z.ZodExactOptional<z.ZodLiteral<"insufficient-evidence">>;
         errorMessage: z.ZodExactOptional<z.ZodString>;
         usage: z.ZodExactOptional<z.ZodObject<{
             input: z.ZodNumber;
@@ -109,6 +121,12 @@ export declare const GradingReportSchema: z.ZodObject<{
             cacheCreation: z.ZodNumber;
         }, z.core.$strip>>;
     }, z.core.$strip>>;
+    skippedAssertions: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        type: z.ZodString;
+        name: z.ZodString;
+        reason: z.ZodLiteral<"judge-disabled">;
+    }, z.core.$strip>>>;
     incomplete: z.ZodExactOptional<z.ZodObject<{
         exitReason: z.ZodEnum<{
             error: "error";

package/lib/core/assertion.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"assertion.d.ts","sourceRoot":"","sources":["../../src/core/assertion.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAE3C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAGpC,eAAO,MAAM,qBAAqB;;;;;;;;;;;;iBAShC,CAAC;AACH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAEpE,MAAM,MAAM,eAAe,GAAG;IAC5B;;;;OAIG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B;;;OAGG;IACH,uBAAuB,CAAC,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5D,CAAC;AAGF,MAAM,WAAW,aAAa,CAAC,CAAC,SAAS,SAAS,GAAG,SAAS;IAC5D,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IAChB,YAAY,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;CAChF;AAED;;;;GAIG;AACH,MAAM,MAAM,eAAe,CACzB,CAAC,SAAS,SAAS,GAAG,SAAS,EAC/B,KAAK,SAAS,SAAS,GAAG,CAAC,IACzB,aAAa,CAAC,CAAC,CAAC,GAAG;IACrB,YAAY,EAAE,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACzC,6EAA6E;IAC7E,QAAQ,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC;IACzB,gEAAgE;IAChE,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,KAAK,eAAe,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;AAClG,KAAK,oBAAoB,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS,IAC7C,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;AAEpD,KAAK,oBAAoB,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS,IAC7C,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC,MAAM,OAAO,SAAS,MAAM,CAAC,GAAG,OAAO,GAAG,KAAK,CAAC;AAE1F,MAAM,MAAM,yBAAyB,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS,IAAI;IAC7D,IAAI,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAAC;IAC9B,YAAY,EAAE,CAAC,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;CACjG,CAAC;AAEF,eAAO,MAAM,oBAAoB,eAAgB;IAC/C,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB,KAAG,MAAM,GAAG,SAsBZ,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,eAAe,GAAI,CAAC,SAAS,CAAC,CAAC,SAAS,cACvC,yBAAyB,CAAC,CAAC,CAAC,KACvC,eAAe,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAI7D,CAAC;AAEF,eAAO,MAAM,uBAAuB,GAAI,KAAK,CAAC,OAAO,SAAS,SAAS,eAAe,EAAE,cAC1E,OAAO,KAClB,OAWF,CAAC;AAEF,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;iBAetC,CAAC;AACH,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC;AAEhF,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAe9B,CAAC;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC"}
+{"version":3,"file":"assertion.d.ts","sourceRoot":"","sources":["../../src/core/assertion.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAE3C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAGpC,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;iBAWhC,CAAC;AACH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAEpE,MAAM,MAAM,eAAe,GAAG;IAC5B;;;;OAIG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B;;;OAGG;IACH,uBAAuB,CAAC,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5D,CAAC;AAGF,MAAM,WAAW,aAAa,CAAC,CAAC,SAAS,SAAS,GAAG,SAAS;IAC5D,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IAChB,YAAY,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;CAChF;AAED;;;;GAIG;AACH,MAAM,MAAM,eAAe,CACzB,CAAC,SAAS,SAAS,GAAG,SAAS,EAC/B,KAAK,SAAS,SAAS,GAAG,CAAC,IACzB,aAAa,CAAC,CAAC,CAAC,GAAG;IACrB,YAAY,EAAE,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACzC,6EAA6E;IAC7E,QAAQ,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC;IACzB,gEAAgE;IAChE,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,KAAK,eAAe,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;AAClG,KAAK,oBAAoB,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS,IAC7C,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;AAEpD,KAAK,oBAAoB,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS,IAC7C,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC,MAAM,OAAO,SAAS,MAAM,CAAC,GAAG,OAAO,GAAG,KAAK,CAAC;AAE1F,MAAM,MAAM,yBAAyB,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS,IAAI;IAC7D,IAAI,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAAC;IAC9B,YAAY,EAAE,CAAC,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;CACjG,CAAC;AAEF,eAAO,MAAM,oBAAoB,eAAgB;IAC/C,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB,KAAG,MAAM,GAAG,SAsBZ,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,mBAAmB;aAC9B,EAAE;aACF,MAAM;aACN,SAAS;aACT,IAAI;aACJ,aAAa;CACL,CAAC;AAEX,eAAO,MAAM,eAAe,GAAI,CAAC,SAAS,CAAC,CAAC,SAAS,cACvC,yBAAyB,CAAC,CAAC,CAAC,KACvC,eAAe,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAI7D,CAAC;AAEF,eAAO,MAAM,uBAAuB,GAAI,KAAK,CAAC,OAAO,SAAS,SAAS,eAAe,EAAE,cAC1E,OAAO,KAClB,OAWF,CAAC;AAEF,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;iBAiBtC,CAAC;AACH,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC;AAShF,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAgB9B,CAAC;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC"}

package/lib/core/assertion.js

@@ -6,6 +6,8 @@ export const AssertionResultSchema = z.object({
     score: z.number().min(0).max(1),
     comment: z.exactOptional(z.string()),
     detail: z.unknown().optional(),
+    /** The evaluator ran successfully but lacked evidence for a verdict. */
+    abstention: z.exactOptional(z.literal("insufficient-evidence")),
     /** Present when the assertion could not produce a judgment. */
     errorMessage: z.exactOptional(z.string()),
     /** Tokens the assertion itself spent. Only LLM-judge assertions report this. */
@@ -36,6 +38,13 @@ export const assertionPluginIssue = (definition) => {
  * Define a schema-validated assertion plugin while inferring the config passed
  * to `evaluate` from the schema.
  */
+export const baseAssertionFields = {
+    id: z.string().min(1).optional(),
+    weight: z.number().positive().optional(),
+    threshold: z.number().min(0).max(1).optional(),
+    name: z.string().optional(),
+    informational: z.boolean().optional(),
+};
 export const defineAssertion = (definition) => {
     const issue = assertionPluginIssue(definition);
     if (issue)
@@ -60,7 +69,8 @@ export const GradingAssertionEntrySchema = z.object({
     id: z.exactOptional(z.string()),
     type: z.string(),
     name: z.string(),
-    passed: z.boolean(),
+    passed: z.boolean().nullable(),
+    informational: z.exactOptional(z.boolean()),
     score: z.number(),
     weight: z.number(),
     threshold: z.number(),
@@ -68,14 +78,22 @@ export const GradingAssertionEntrySchema = z.object({
     durationMs: z.number(),
     comment: z.exactOptional(z.string()),
     detail: z.unknown().optional(),
+    abstention: z.exactOptional(z.literal("insufficient-evidence")),
     errorMessage: z.exactOptional(z.string()),
     usage: z.exactOptional(TokenUsageSchema),
 });
+const SkippedGradingAssertionSchema = z.object({
+    id: z.string(),
+    type: z.string(),
+    name: z.string(),
+    reason: z.literal("judge-disabled"),
+});
 export const GradingReportSchema = z.object({
     schemaVersion: z.literal(1),
     casePassed: z.boolean().nullable(),
     caseScore: z.number().nullable(),
     assertions: z.array(GradingAssertionEntrySchema),
+    skippedAssertions: z.array(SkippedGradingAssertionSchema).optional(),
     /**
      * Present when the subject run or grading did not complete. Assertions may
      * still contain partial diagnostics, but the report is not a clean verdict.

package/lib/core/assertion.js.map

@@ -1 +1 @@
-{"version":3,"file":"assertion.js","sourceRoot":"","sources":["../../src/core/assertion.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AAErD,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE9C,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/B,OAAO,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACpC,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC9B,+DAA+D;IAC/D,YAAY,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACzC,gFAAgF;IAChF,KAAK,EAAE,CAAC,CAAC,aAAa,CAAC,gBAAgB,CAAC;CACzC,CAAC,CAAC;AAsDH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,UAKpC,EAAsB,EAAE;IACvB,IAAI,OAAO,UAAU,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACxC,OAAO,wCAAwC,CAAC;IAClD,CAAC;IACD,IAAI,CAAC,CAAC,UAAU,CAAC,YAAY,YAAY,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;QACtD,OAAO,2DAA2D,CAAC;IACrE,CAAC;IACD,MAAM,aAAa,GAAG,UAAU,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC;IACzD,IACE,CAAC,CAAC,aAAa,YAAY,CAAC,CAAC,UAAU,CAAC;QACxC,aAAa,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC;QAC/B,aAAa,CAAC,KAAK,KAAK,UAAU,CAAC,IAAI,EACvC,CAAC;QACD,OAAO,wEAAwE,CAAC;IAClF,CAAC;IACD,IAAI,OAAO,UAAU,CAAC,aAAa,KAAK,QAAQ,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,EAAE,EAAE,CAAC;QACrF,OAAO,kDAAkD,CAAC;IAC5D,CAAC;IACD,IAAI,OAAO,UAAU,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;QAC9C,OAAO,8CAA8C,CAAC;IACxD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAC7B,UAAwC,EACsB,EAAE;IAChE,MAAM,KAAK,GAAG,oBAAoB,CAAC,UAAU,CAAC,CAAC;IAC/C,IAAI,KAAK;QAAE,MAAM,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC;IACtC,OAAO,UAAqF,CAAC;AAC/F,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CACrC,GAAG,OAAgB,EACV,EAAE;IACX,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,KAAK;YAAE,MAAM,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,SAAS,CAAC,0CAA0C,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;QAChF,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC,MAAM,CAAC;IAClD,yEAAyE;IACzE,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,+FAA+F;IAC/F,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;IACtB,OAAO,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACpC,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC9B,YAAY,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACzC,KAAK,EAAE,CAAC,CAAC,aAAa,CAAC,gBAAgB,CAAC;CACzC,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1C,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3B,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,2BAA2B,CAAC;IAChD;;;OAGG;IACH,UAAU,EAAE,CAAC,CAAC,aAAa,CACzB,CAAC,CAAC,MAAM,CAAC;QACP,UAAU,EAAE,sBAAsB;QAClC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,YAAY,EAAE,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE;KAC3D,CAAC,CACH;CACF,CAAC,CAAC"}
+{"version":3,"file":"assertion.js","sourceRoot":"","sources":["../../src/core/assertion.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AAErD,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE9C,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/B,OAAO,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACpC,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC9B,wEAAwE;IACxE,UAAU,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;IAC/D,+DAA+D;IAC/D,YAAY,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACzC,gFAAgF;IAChF,KAAK,EAAE,CAAC,CAAC,aAAa,CAAC,gBAAgB,CAAC;CACzC,CAAC,CAAC;AAsDH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,UAKpC,EAAsB,EAAE;IACvB,IAAI,OAAO,UAAU,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACxC,OAAO,wCAAwC,CAAC;IAClD,CAAC;IACD,IAAI,CAAC,CAAC,UAAU,CAAC,YAAY,YAAY,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;QACtD,OAAO,2DAA2D,CAAC;IACrE,CAAC;IACD,MAAM,aAAa,GAAG,UAAU,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC;IACzD,IACE,CAAC,CAAC,aAAa,YAAY,CAAC,CAAC,UAAU,CAAC;QACxC,aAAa,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC;QAC/B,aAAa,CAAC,KAAK,KAAK,UAAU,CAAC,IAAI,EACvC,CAAC;QACD,OAAO,wEAAwE,CAAC;IAClF,CAAC;IACD,IAAI,OAAO,UAAU,CAAC,aAAa,KAAK,QAAQ,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,EAAE,EAAE,CAAC;QACrF,OAAO,kDAAkD,CAAC;IAC5D,CAAC;IACD,IAAI,OAAO,UAAU,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;QAC9C,OAAO,8CAA8C,CAAC;IACxD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG;IACjC,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAChC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACxC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC9C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CAC7B,CAAC;AAEX,MAAM,CAAC,MAAM,eAAe,GAAG,CAC7B,UAAwC,EACsB,EAAE;IAChE,MAAM,KAAK,GAAG,oBAAoB,CAAC,UAAU,CAAC,CAAC;IAC/C,IAAI,KAAK;QAAE,MAAM,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC;IACtC,OAAO,UAAqF,CAAC;AAC/F,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CACrC,GAAG,OAAgB,EACV,EAAE;IACX,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,KAAK;YAAE,MAAM,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,SAAS,CAAC,0CAA0C,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;QAChF,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC,MAAM,CAAC;IAClD,yEAAyE;IACzE,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC9B,aAAa,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;IAC3C,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,+FAA+F;IAC/F,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;IACtB,OAAO,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACpC,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC9B,UAAU,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;IAC/D,YAAY,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACzC,KAAK,EAAE,CAAC,CAAC,aAAa,CAAC,gBAAgB,CAAC;CACzC,CAAC,CAAC;AAGH,MAAM,6BAA6B,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7C,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;IACd,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC;CACpC,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1C,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3B,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,2BAA2B,CAAC;IAChD,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC,QAAQ,EAAE;IACpE;;;OAGG;IACH,UAAU,EAAE,CAAC,CAAC,aAAa,CACzB,CAAC,CAAC,MAAM,CAAC;QACP,UAAU,EAAE,sBAAsB;QAClC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,YAAY,EAAE,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE;KAC3D,CAAC,CACH;CACF,CAAC,CAAC"}

package/lib/core/case.d.ts

@@ -9,6 +9,46 @@ export type FixturesRef = z.infer<typeof FixturesRefSchema>;
 export declare const DriverRefSchema: z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
     name: z.ZodString;
     config: z.ZodOptional<z.ZodUnknown>;
+    execution: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"host">;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"container">;
+        image: z.ZodString;
+        command: z.ZodOptional<z.ZodString>;
+        worker: z.ZodOptional<z.ZodString>;
+        driverModule: z.ZodOptional<z.ZodString>;
+        mounts: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            source: z.ZodString;
+            target: z.ZodOptional<z.ZodString>;
+            readOnly: z.ZodOptional<z.ZodBoolean>;
+        }, z.core.$strict>>>;
+        passEnv: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        homeFiles: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            destination: z.ZodString;
+            source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                kind: z.ZodLiteral<"file">;
+                path: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"keychain">;
+                service: z.ZodString;
+                account: z.ZodString;
+                encoding: z.ZodOptional<z.ZodEnum<{
+                    "go-keyring-base64": "go-keyring-base64";
+                    utf8: "utf8";
+                }>>;
+            }, z.core.$strict>], "kind">;
+        }, z.core.$strict>>>;
+        network: z.ZodOptional<z.ZodEnum<{
+            bridge: "bridge";
+            none: "none";
+        }>>;
+        user: z.ZodOptional<z.ZodString>;
+        memory: z.ZodOptional<z.ZodString>;
+        cpus: z.ZodOptional<z.ZodNumber>;
+        pidsLimit: z.ZodOptional<z.ZodNumber>;
+        startupTimeoutMs: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strict>], "kind">>;
 }, z.core.$strict>]>;
 export type DriverRef = z.infer<typeof DriverRefSchema>;
 export declare const AnswererRefSchema: z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
@@ -22,6 +62,7 @@ export declare const AssertionSchema: z.ZodObject<{
     name: z.ZodOptional<z.ZodString>;
     weight: z.ZodOptional<z.ZodNumber>;
     threshold: z.ZodOptional<z.ZodNumber>;
+    informational: z.ZodOptional<z.ZodBoolean>;
 }, z.core.$loose>;
 export type Assertion = z.infer<typeof AssertionSchema>;
 export declare const EvalCaseSchema: z.ZodObject<{
@@ -31,7 +72,48 @@ export declare const EvalCaseSchema: z.ZodObject<{
     driver: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
         name: z.ZodString;
         config: z.ZodOptional<z.ZodUnknown>;
+        execution: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"host">;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"container">;
+            image: z.ZodString;
+            command: z.ZodOptional<z.ZodString>;
+            worker: z.ZodOptional<z.ZodString>;
+            driverModule: z.ZodOptional<z.ZodString>;
+            mounts: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                source: z.ZodString;
+                target: z.ZodOptional<z.ZodString>;
+                readOnly: z.ZodOptional<z.ZodBoolean>;
+            }, z.core.$strict>>>;
+            passEnv: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+            homeFiles: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                destination: z.ZodString;
+                source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                    kind: z.ZodLiteral<"file">;
+                    path: z.ZodString;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"keychain">;
+                    service: z.ZodString;
+                    account: z.ZodString;
+                    encoding: z.ZodOptional<z.ZodEnum<{
+                        "go-keyring-base64": "go-keyring-base64";
+                        utf8: "utf8";
+                    }>>;
+                }, z.core.$strict>], "kind">;
+            }, z.core.$strict>>>;
+            network: z.ZodOptional<z.ZodEnum<{
+                bridge: "bridge";
+                none: "none";
+            }>>;
+            user: z.ZodOptional<z.ZodString>;
+            memory: z.ZodOptional<z.ZodString>;
+            cpus: z.ZodOptional<z.ZodNumber>;
+            pidsLimit: z.ZodOptional<z.ZodNumber>;
+            startupTimeoutMs: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strict>], "kind">>;
     }, z.core.$strict>]>>;
+    matrix: z.ZodOptional<z.ZodArray<z.ZodCustom<import("./harness-matrix.js").HarnessMatrixEntry, import("./harness-matrix.js").HarnessMatrixEntry>>>;
     answerer: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
         name: z.ZodString;
         config: z.ZodOptional<z.ZodUnknown>;
@@ -40,17 +122,30 @@ export declare const EvalCaseSchema: z.ZodObject<{
         kind: z.ZodLiteral<"dir">;
         path: z.ZodString;
     }, z.core.$strict>]>>;
+    requires: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+        artifacts: "artifacts";
+        cost: "cost";
+        interactive: "interactive";
+        isolation: "isolation";
+        subagents: "subagents";
+        "tool-calls": "tool-calls";
+        "tool-results": "tool-results";
+        usage: "usage";
+    }>>>;
     assertions: z.ZodOptional<z.ZodArray<z.ZodObject<{
         id: z.ZodOptional<z.ZodString>;
         type: z.ZodString;
         name: z.ZodOptional<z.ZodString>;
         weight: z.ZodOptional<z.ZodNumber>;
         threshold: z.ZodOptional<z.ZodNumber>;
+        informational: z.ZodOptional<z.ZodBoolean>;
     }, z.core.$loose>>>;
-    meta: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    meta: z.ZodOptional<z.ZodObject<{
+        cluster: z.ZodOptional<z.ZodString>;
+    }, z.core.$catchall<z.ZodUnknown>>>;
 }, z.core.$strict>;
 export type EvalCase = z.infer<typeof EvalCaseSchema>;
-type AssertionPlanFields = Pick<Assertion, "id" | "name" | "threshold" | "weight">;
+type AssertionPlanFields = Pick<Assertion, "id" | "name" | "threshold" | "weight" | "informational">;
 type AssertionInputOf<Plugin> = Plugin extends AssertionPlugin<Assertion, infer Input> ? Input & AssertionPlanFields : never;
 export type EvalCaseInput<AllowedAssertion extends Assertion = Assertion> = Omit<EvalCase, "assertions"> & {
     assertions?: AllowedAssertion[];
@@ -75,19 +170,31 @@ export declare const defineEvalCase: EvalCaseDefinition<{
     name?: string | undefined;
     weight?: number | undefined;
     threshold?: number | undefined;
+    informational?: boolean | undefined;
 }>;
 export declare const PersistedCaseSchema: z.ZodObject<{
     id: z.ZodOptional<z.ZodOptional<z.ZodString>>;
     prompt: z.ZodOptional<z.ZodString>;
     persona: z.ZodOptional<z.ZodOptional<z.ZodString>>;
+    matrix: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodCustom<import("./harness-matrix.js").HarnessMatrixEntry, import("./harness-matrix.js").HarnessMatrixEntry>>>>;
+    requires: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodEnum<{
+        artifacts: "artifacts";
+        cost: "cost";
+        interactive: "interactive";
+        isolation: "isolation";
+        subagents: "subagents";
+        "tool-calls": "tool-calls";
+        "tool-results": "tool-results";
+        usage: "usage";
+    }>>>>;
     assertions: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
         id: z.ZodOptional<z.ZodString>;
         type: z.ZodString;
         name: z.ZodOptional<z.ZodString>;
         weight: z.ZodOptional<z.ZodNumber>;
         threshold: z.ZodOptional<z.ZodNumber>;
+        informational: z.ZodOptional<z.ZodBoolean>;
     }, z.core.$loose>>>>;
-    meta: z.ZodOptional<z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
     casePath: z.ZodExactOptional<z.ZodString>;
     driver: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
         name: z.ZodString;
@@ -101,6 +208,7 @@ export declare const PersistedCaseSchema: z.ZodObject<{
         kind: z.ZodLiteral<"dir">;
         path: z.ZodString;
     }, z.core.$loose>]>>;
+    meta: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
 }, z.core.$loose>;
 export type PersistedCase = z.infer<typeof PersistedCaseSchema>;
 export {};

package/lib/core/case.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"case.d.ts","sourceRoot":"","sources":["../../src/core/case.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAEtD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAGjE,eAAO,MAAM,iBAAiB;;;oBAG5B,CAAC;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAO5D,eAAO,MAAM,eAAe;;;oBAG1B,CAAC;AACH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAExD,eAAO,MAAM,iBAAiB;;;oBAG5B,CAAC;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAE5D,eAAO,MAAM,eAAe;;;;;;iBASlB,CAAC;AACX,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAaxD,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;kBAiBvB,CAAC;AACL,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAEtD,KAAK,mBAAmB,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,GAAG,MAAM,GAAG,WAAW,GAAG,QAAQ,CAAC,CAAC;AACnF,KAAK,gBAAgB,CAAC,MAAM,IAC1B,MAAM,SAAS,eAAe,CAAC,SAAS,EAAE,MAAM,KAAK,CAAC,GAAG,KAAK,GAAG,mBAAmB,GAAG,KAAK,CAAC;AAE/F,MAAM,MAAM,aAAa,CAAC,gBAAgB,SAAS,SAAS,GAAG,SAAS,IAAI,IAAI,CAC9E,QAAQ,EACR,YAAY,CACb,GAAG;IACF,UAAU,CAAC,EAAE,gBAAgB,EAAE,CAAC;CACjC,CAAC;AAEF,MAAM,MAAM,wBAAwB,CAClC,OAAO,SAAS,SAAS,eAAe,EAAE,GAAG,SAAS,eAAe,EAAE,IACrE;IACF,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE,SAAS,oBAAoB,EAAE,CAAC;CAC5C,CAAC;AAEF,MAAM,MAAM,kBAAkB,CAAC,gBAAgB,SAAS,SAAS,GAAG,SAAS,IAAI;IAC/E,CAAC,KAAK,EAAE,aAAa,CAAC,gBAAgB,CAAC,GAAG,QAAQ,CAAC;IACnD;;;OAGG;IACH,MAAM,CAAC,KAAK,CAAC,OAAO,SAAS,SAAS,eAAe,EAAE,GAAG,SAAS,EAAE,EACnE,OAAO,EAAE,wBAAwB,CAAC,OAAO,CAAC,GACzC,kBAAkB,CAAC,gBAAgB,GAAG,gBAAgB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;CAC7E,CAAC;AAEF,eAAO,MAAM,wBAAwB,GAAI,gBAAgB,SAAS,SAAS,GAAG,SAAS,wBAChE,SAAS,eAAe,EAAE,sBAC5B,SAAS,oBAAoB,EAAE,KACjD,kBAAkB,CAAC,gBAAgB,CAoBrC,CAAC;AAEF,eAAO,MAAM,cAAc;;;;;;;EAA6B,CAAC;AAmBzD,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;iBAOtB,CAAC;AACX,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC"}
+{"version":3,"file":"case.d.ts","sourceRoot":"","sources":["../../src/core/case.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAMtD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAGjE,eAAO,MAAM,iBAAiB;;;oBAG5B,CAAC;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAO5D,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oBAS1B,CAAC;AACH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAExD,eAAO,MAAM,iBAAiB;;;oBAG5B,CAAC;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAE5D,eAAO,MAAM,eAAe;;;;;;;iBAUlB,CAAC;AACX,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAgBxD,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAiBvB,CAAC;AACL,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAEtD,KAAK,mBAAmB,GAAG,IAAI,CAC7B,SAAS,EACT,IAAI,GAAG,MAAM,GAAG,WAAW,GAAG,QAAQ,GAAG,eAAe,CACzD,CAAC;AACF,KAAK,gBAAgB,CAAC,MAAM,IAC1B,MAAM,SAAS,eAAe,CAAC,SAAS,EAAE,MAAM,KAAK,CAAC,GAAG,KAAK,GAAG,mBAAmB,GAAG,KAAK,CAAC;AAE/F,MAAM,MAAM,aAAa,CAAC,gBAAgB,SAAS,SAAS,GAAG,SAAS,IAAI,IAAI,CAC9E,QAAQ,EACR,YAAY,CACb,GAAG;IACF,UAAU,CAAC,EAAE,gBAAgB,EAAE,CAAC;CACjC,CAAC;AAEF,MAAM,MAAM,wBAAwB,CAClC,OAAO,SAAS,SAAS,eAAe,EAAE,GAAG,SAAS,eAAe,EAAE,IACrE;IACF,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE,SAAS,oBAAoB,EAAE,CAAC;CAC5C,CAAC;AAEF,MAAM,MAAM,kBAAkB,CAAC,gBAAgB,SAAS,SAAS,GAAG,SAAS,IAAI;IAC/E,CAAC,KAAK,EAAE,aAAa,CAAC,gBAAgB,CAAC,GAAG,QAAQ,CAAC;IACnD;;;OAGG;IACH,MAAM,CAAC,KAAK,CAAC,OAAO,SAAS,SAAS,eAAe,EAAE,GAAG,SAAS,EAAE,EACnE,OAAO,EAAE,wBAAwB,CAAC,OAAO,CAAC,GACzC,kBAAkB,CAAC,gBAAgB,GAAG,gBAAgB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;CAC7E,CAAC;AAEF,eAAO,MAAM,wBAAwB,GAAI,gBAAgB,SAAS,SAAS,GAAG,SAAS,wBAChE,SAAS,eAAe,EAAE,sBAC5B,SAAS,oBAAoB,EAAE,KACjD,kBAAkB,CAAC,gBAAgB,CAoBrC,CAAC;AAEF,eAAO,MAAM,cAAc;;;;;;;;EAA6B,CAAC;AAmBzD,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAQtB,CAAC;AACX,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC"}

package/lib/core/case.js

@@ -1,5 +1,9 @@
 import { z } from "zod";
 import { collectEvalCaseExtensions, setEvalCaseExtensions } from "./case-authoring-state.js";
+import { DriverCapabilitySchema } from "./driver.js";
+import { DriverExecutionSchema } from "./driver-execution.js";
+import { EvaluationMetadataSchema } from "./evaluation-clusters.js";
+import { HarnessMatrixSchema } from "./harness-matrix.js";
 import { formatZodIssues } from "./zod.js";
 export const FixturesRefSchema = z.union([
     z.string(),
@@ -12,7 +16,13 @@ export const FixturesRefSchema = z.union([
 // `defineEvalCase` discriminate the config type on `name`.
 export const DriverRefSchema = z.union([
     z.string(),
-    z.object({ name: z.string(), config: z.unknown().optional() }).strict(),
+    z
+        .object({
+        name: z.string(),
+        config: z.unknown().optional(),
+        execution: DriverExecutionSchema.optional(),
+    })
+        .strict(),
 ]);
 export const AnswererRefSchema = z.union([
     z.string(),
@@ -26,6 +36,7 @@ export const AssertionSchema = z
     name: z.string().optional(),
     weight: z.number().positive().optional(),
     threshold: z.number().min(0).max(1).optional(),
+    informational: z.boolean().optional(),
 })
     .loose();
 const EvalCaseObjectSchema = z.object({
@@ -33,10 +44,13 @@ const EvalCaseObjectSchema = z.object({
     prompt: z.string().min(1, "prompt is required"),
     persona: z.string().min(1).optional(),
     driver: DriverRefSchema.optional(),
+    /** Harness permutations this case runs through instead of the project matrix. */
+    matrix: HarnessMatrixSchema.optional(),
     answerer: AnswererRefSchema.optional(),
     fixtures: FixturesRefSchema.optional(),
+    requires: z.array(DriverCapabilitySchema).optional(),
     assertions: z.array(AssertionSchema).optional(),
-    meta: z.record(z.string(), z.unknown()).optional(),
+    meta: EvaluationMetadataSchema.optional(),
 });
 export const EvalCaseSchema = EvalCaseObjectSchema
     // strict so a stray `driverConfig`/`answererConfig` (the old shape) or a
@@ -93,6 +107,7 @@ export const PersistedCaseSchema = EvalCaseObjectSchema.partial()
     driver: PersistedDriverRefSchema.optional(),
     answerer: PersistedAnswererRefSchema.optional(),
     fixtures: PersistedFixturesRefSchema.optional(),
+    meta: z.record(z.string(), z.unknown()).optional(),
 })
     .loose();
 //# sourceMappingURL=case.js.map

package/lib/core/case.js.map

@@ -1 +1 @@
-{"version":3,"file":"case.js","sourceRoot":"","sources":["../../src/core/case.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,yBAAyB,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAE7F,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAE3C,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC;IACvC,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;CAChE,CAAC,CAAC;AAGH,6EAA6E;AAC7E,6EAA6E;AAC7E,8EAA8E;AAC9E,8EAA8E;AAC9E,2DAA2D;AAC3D,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC;IACrC,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;CACxE,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC;IACvC,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;CACxE,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,sFAAsF;IACtF,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAChC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACxC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CAC/C,CAAC;KACD,KAAK,EAAE,CAAC;AAGX,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACzB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,oBAAoB,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACrC,MAAM,EAAE,eAAe,CAAC,QAAQ,EAAE;IAClC,QAAQ,EAAE,iBAAiB,CAAC,QAAQ,EAAE;IACtC,QAAQ,EAAE,iBAAiB,CAAC,QAAQ,EAAE;IACtC,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,QAAQ,EAAE;IAC/C,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACnD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,cAAc,GAAG,oBAAoB;IAChD,yEAAyE;IACzE,+DAA+D;KAC9D,MAAM,EAAE;KACR,WAAW,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE;IAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QACvE,IAAI,CAAC,SAAS,CAAC,EAAE;YAAE,SAAS;QAC5B,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC;YAC3B,GAAG,CAAC,QAAQ,CAAC;gBACX,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,CAAC,YAAY,EAAE,KAAK,EAAE,IAAI,CAAC;gBACjC,OAAO,EAAE,2BAA2B,SAAS,CAAC,EAAE,GAAG;aACpD,CAAC,CAAC;QACL,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACzB,CAAC;AACH,CAAC,CAAC,CAAC;AAgCL,MAAM,CAAC,MAAM,wBAAwB,GAAG,CACtC,mBAAmB,GAA+B,EAAE,EACpD,iBAAiB,GAAoC,EAAE,EACjB,EAAE;IACxC,MAAM,MAAM,GAAG,CAAC,CAAC,KAAsC,EAAY,EAAE;QACnE,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC/C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,sBAAsB,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,UAAU,GAAG,yBAAyB,CAC1C,MAAM,CAAC,IAAI,EACX,mBAAmB,EACnB,iBAAiB,CAClB,CAAC;QACF,qBAAqB,CAAC,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAC/C,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC,CAAyC,CAAC;IAC3C,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,UAAU,GAAG,EAAE,EAAE,QAAQ,GAAG,EAAE,EAA4B,EAAE,EAAE,CAChF,wBAAwB,CACtB,CAAC,GAAG,mBAAmB,EAAE,GAAG,UAAU,CAAC,EACvC,CAAC,GAAG,iBAAiB,EAAE,GAAG,QAAQ,CAAC,CACpC,CAAmD,CAAC;IACvD,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,wBAAwB,EAAE,CAAC;AAEzD,MAAM,0BAA0B,GAAG,CAAC,CAAC,KAAK,CAAC;IACzC,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;CAC5D,CAAC,CAAC;AACH,MAAM,wBAAwB,GAAG,CAAC,CAAC,KAAK,CAAC;IACvC,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC;CACpE,CAAC,CAAC;AACH,MAAM,0BAA0B,GAAG,CAAC,CAAC,KAAK,CAAC;IACzC,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC;CACpE,CAAC,CAAC;AAEH,8EAA8E;AAC9E,uEAAuE;AACvE,2EAA2E;AAC3E,0DAA0D;AAC1D,MAAM,CAAC,MAAM,mBAAmB,GAAG,oBAAoB,CAAC,OAAO,EAAE;KAC9D,MAAM,CAAC;IACN,QAAQ,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACrC,MAAM,EAAE,wBAAwB,CAAC,QAAQ,EAAE;IAC3C,QAAQ,EAAE,0BAA0B,CAAC,QAAQ,EAAE;IAC/C,QAAQ,EAAE,0BAA0B,CAAC,QAAQ,EAAE;CAChD,CAAC;KACD,KAAK,EAAE,CAAC"}
+{"version":3,"file":"case.js","sourceRoot":"","sources":["../../src/core/case.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,yBAAyB,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAC7F,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1D,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAE3C,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC;IACvC,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;CAChE,CAAC,CAAC;AAGH,6EAA6E;AAC7E,6EAA6E;AAC7E,8EAA8E;AAC9E,8EAA8E;AAC9E,2DAA2D;AAC3D,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC;IACrC,CAAC,CAAC,MAAM,EAAE;IACV,CAAC;SACE,MAAM,CAAC;QACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;QAChB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;QAC9B,SAAS,EAAE,qBAAqB,CAAC,QAAQ,EAAE;KAC5C,CAAC;SACD,MAAM,EAAE;CACZ,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC;IACvC,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;CACxE,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,sFAAsF;IACtF,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAChC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACxC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC9C,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CACtC,CAAC;KACD,KAAK,EAAE,CAAC;AAGX,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACzB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,oBAAoB,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACrC,MAAM,EAAE,eAAe,CAAC,QAAQ,EAAE;IAClC,iFAAiF;IACjF,MAAM,EAAE,mBAAmB,CAAC,QAAQ,EAAE;IACtC,QAAQ,EAAE,iBAAiB,CAAC,QAAQ,EAAE;IACtC,QAAQ,EAAE,iBAAiB,CAAC,QAAQ,EAAE;IACtC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE;IACpD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,QAAQ,EAAE;IAC/C,IAAI,EAAE,wBAAwB,CAAC,QAAQ,EAAE;CAC1C,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,cAAc,GAAG,oBAAoB;IAChD,yEAAyE;IACzE,+DAA+D;KAC9D,MAAM,EAAE;KACR,WAAW,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE;IAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QACvE,IAAI,CAAC,SAAS,CAAC,EAAE;YAAE,SAAS;QAC5B,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC;YAC3B,GAAG,CAAC,QAAQ,CAAC;gBACX,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,CAAC,YAAY,EAAE,KAAK,EAAE,IAAI,CAAC;gBACjC,OAAO,EAAE,2BAA2B,SAAS,CAAC,EAAE,GAAG;aACpD,CAAC,CAAC;QACL,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACzB,CAAC;AACH,CAAC,CAAC,CAAC;AAmCL,MAAM,CAAC,MAAM,wBAAwB,GAAG,CACtC,mBAAmB,GAA+B,EAAE,EACpD,iBAAiB,GAAoC,EAAE,EACjB,EAAE;IACxC,MAAM,MAAM,GAAG,CAAC,CAAC,KAAsC,EAAY,EAAE;QACnE,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC/C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,sBAAsB,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,UAAU,GAAG,yBAAyB,CAC1C,MAAM,CAAC,IAAI,EACX,mBAAmB,EACnB,iBAAiB,CAClB,CAAC;QACF,qBAAqB,CAAC,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAC/C,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC,CAAyC,CAAC;IAC3C,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,UAAU,GAAG,EAAE,EAAE,QAAQ,GAAG,EAAE,EAA4B,EAAE,EAAE,CAChF,wBAAwB,CACtB,CAAC,GAAG,mBAAmB,EAAE,GAAG,UAAU,CAAC,EACvC,CAAC,GAAG,iBAAiB,EAAE,GAAG,QAAQ,CAAC,CACpC,CAAmD,CAAC;IACvD,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,wBAAwB,EAAE,CAAC;AAEzD,MAAM,0BAA0B,GAAG,CAAC,CAAC,KAAK,CAAC;IACzC,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;CAC5D,CAAC,CAAC;AACH,MAAM,wBAAwB,GAAG,CAAC,CAAC,KAAK,CAAC;IACvC,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC;CACpE,CAAC,CAAC;AACH,MAAM,0BAA0B,GAAG,CAAC,CAAC,KAAK,CAAC;IACzC,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC;CACpE,CAAC,CAAC;AAEH,8EAA8E;AAC9E,uEAAuE;AACvE,2EAA2E;AAC3E,0DAA0D;AAC1D,MAAM,CAAC,MAAM,mBAAmB,GAAG,oBAAoB,CAAC,OAAO,EAAE;KAC9D,MAAM,CAAC;IACN,QAAQ,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACrC,MAAM,EAAE,wBAAwB,CAAC,QAAQ,EAAE;IAC3C,QAAQ,EAAE,0BAA0B,CAAC,QAAQ,EAAE;IAC/C,QAAQ,EAAE,0BAA0B,CAAC,QAAQ,EAAE;IAC/C,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACnD,CAAC;KACD,KAAK,EAAE,CAAC"}

package/lib/core/command-verification.d.ts

@@ -0,0 +1,10 @@
+import type { Run } from "./run.js";
+/** Internal: live execution state is deliberately absent from persisted run evidence. */
+export declare const bindRunExecutionWorkspace: (run: Run, root: string) => void;
+export declare const runExecutionWorkspace: (run: Run) => string | undefined;
+/** A disposable command tree preserves dependencies without exposing host symlink targets. */
+export declare const copyCommandWorkspace: (source: string, signal?: AbortSignal) => Promise<{
+    root: string;
+    close: () => Promise<void>;
+}>;
+//# sourceMappingURL=command-verification.d.ts.map

package/lib/core/command-verification.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"command-verification.d.ts","sourceRoot":"","sources":["../../src/core/command-verification.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAKpC,yFAAyF;AACzF,eAAO,MAAM,yBAAyB,QAAS,GAAG,QAAQ,MAAM,KAAG,IAElE,CAAC;AAEF,eAAO,MAAM,qBAAqB,QAAS,GAAG,KAAG,MAAM,GAAG,SAAyC,CAAC;AAOpG,8FAA8F;AAC9F,eAAO,MAAM,oBAAoB,WACvB,MAAM,WACL,WAAW,KACnB,OAAO,CAAC;IACT,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5B,CA8CA,CAAC"}

package/lib/core/command-verification.js

@@ -0,0 +1,66 @@
+import { chmod, lstat, mkdir, mkdtemp, readdir, realpath, rm, symlink } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+import { copyRegularFileNoFollow } from "./secure-copy.js";
+const executionWorkspaces = new WeakMap();
+/** Internal: live execution state is deliberately absent from persisted run evidence. */
+export const bindRunExecutionWorkspace = (run, root) => {
+    executionWorkspaces.set(run, resolve(root));
+};
+export const runExecutionWorkspace = (run) => executionWorkspaces.get(run);
+const within = (root, path) => {
+    const rel = relative(root, path);
+    return rel !== ".." && !rel.startsWith(`..${sep}`) && !isAbsolute(rel);
+};
+/** A disposable command tree preserves dependencies without exposing host symlink targets. */
+export const copyCommandWorkspace = async (source, signal) => {
+    const sourceRoot = await realpath(source);
+    if (!(await lstat(source)).isDirectory() || (await lstat(source)).isSymbolicLink()) {
+        throw new Error(`verification workspace must be a real directory: ${source}`);
+    }
+    const root = await mkdtemp(join(tmpdir(), "lux-command-verification-"));
+    const close = () => rm(root, { recursive: true, force: true });
+    let entries = 0;
+    const copy = async (directory) => {
+        if (!within(sourceRoot, await realpath(directory))) {
+            throw new Error(`verification directory escapes its workspace: ${directory}`);
+        }
+        for (const entry of await readdir(directory, { withFileTypes: true })) {
+            signal?.throwIfAborted();
+            if (++entries > 100_000)
+                throw new Error("verification workspace exceeds 100000 entries");
+            if (directory === sourceRoot && entry.name === ".lux-claude")
+                continue;
+            const path = join(directory, entry.name);
+            const target = join(root, relative(sourceRoot, path));
+            const metadata = await lstat(path);
+            if (metadata.isSymbolicLink()) {
+                const resolved = await realpath(path);
+                if (!within(sourceRoot, resolved)) {
+                    throw new Error(`verification symlink escapes its workspace: ${path}`);
+                }
+                await symlink(relative(dirname(target), join(root, relative(sourceRoot, resolved))), target);
+            }
+            else if (metadata.isDirectory()) {
+                await mkdir(target, { mode: (metadata.mode & 0o777) | 0o700 });
+                await copy(path);
+                await chmod(target, metadata.mode & 0o777);
+            }
+            else if (metadata.isFile()) {
+                await copyRegularFileNoFollow(sourceRoot, path, target);
+            }
+            else {
+                throw new Error(`verification workspace contains an unsupported entry: ${path}`);
+            }
+        }
+    };
+    try {
+        await copy(sourceRoot);
+        return { root, close };
+    }
+    catch (error) {
+        await close();
+        throw error;
+    }
+};
+//# sourceMappingURL=command-verification.js.map

package/lib/core/command-verification.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"command-verification.js","sourceRoot":"","sources":["../../src/core/command-verification.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAChG,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAE9E,OAAO,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAC;AAE3D,MAAM,mBAAmB,GAAG,IAAI,OAAO,EAAe,CAAC;AAEvD,yFAAyF;AACzF,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,GAAQ,EAAE,IAAY,EAAQ,EAAE;IACxE,mBAAmB,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;AAC9C,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,GAAQ,EAAsB,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAEpG,MAAM,MAAM,GAAG,CAAC,IAAY,EAAE,IAAY,EAAW,EAAE;IACrD,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACjC,OAAO,GAAG,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;AACzE,CAAC,CAAC;AAEF,8FAA8F;AAC9F,MAAM,CAAC,MAAM,oBAAoB,GAAG,KAAK,EACvC,MAAc,EACd,MAAoB,EAInB,EAAE;IACH,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,EAAE,EAAE,CAAC;QACnF,MAAM,IAAI,KAAK,CAAC,oDAAoD,MAAM,EAAE,CAAC,CAAC;IAChF,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,2BAA2B,CAAC,CAAC,CAAC;IACxE,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/D,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,MAAM,IAAI,GAAG,KAAK,EAAE,SAAiB,EAAiB,EAAE;QACtD,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,iDAAiD,SAAS,EAAE,CAAC,CAAC;QAChF,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YACtE,MAAM,EAAE,cAAc,EAAE,CAAC;YACzB,IAAI,EAAE,OAAO,GAAG,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;YAC1F,IAAI,SAAS,KAAK,UAAU,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa;gBAAE,SAAS;YACvE,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACzC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC;YACtD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;YACnC,IAAI,QAAQ,CAAC,cAAc,EAAE,EAAE,CAAC;gBAC9B,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACtC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,CAAC;oBAClC,MAAM,IAAI,KAAK,CAAC,+CAA+C,IAAI,EAAE,CAAC,CAAC;gBACzE,CAAC;gBACD,MAAM,OAAO,CACX,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC,EACrE,MAAM,CACP,CAAC;YACJ,CAAC;iBAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;gBAClC,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,IAAI,GAAG,KAAK,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;gBAC/D,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;gBACjB,MAAM,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC;YAC7C,CAAC;iBAAM,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC7B,MAAM,uBAAuB,CAAC,UAAU,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;YAC1D,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CAAC,yDAAyD,IAAI,EAAE,CAAC,CAAC;YACnF,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IACF,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;QACvB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACzB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,KAAK,EAAE,CAAC;QACd,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC,CAAC"}

package/lib/core/driver-execution.d.ts

@@ -0,0 +1,83 @@
+import { z } from "zod";
+export declare const ContainerExecutionSchema: z.ZodObject<{
+    kind: z.ZodLiteral<"container">;
+    image: z.ZodString;
+    command: z.ZodOptional<z.ZodString>;
+    worker: z.ZodOptional<z.ZodString>;
+    driverModule: z.ZodOptional<z.ZodString>;
+    mounts: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        source: z.ZodString;
+        target: z.ZodOptional<z.ZodString>;
+        readOnly: z.ZodOptional<z.ZodBoolean>;
+    }, z.core.$strict>>>;
+    passEnv: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    homeFiles: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        destination: z.ZodString;
+        source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"file">;
+            path: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"keychain">;
+            service: z.ZodString;
+            account: z.ZodString;
+            encoding: z.ZodOptional<z.ZodEnum<{
+                "go-keyring-base64": "go-keyring-base64";
+                utf8: "utf8";
+            }>>;
+        }, z.core.$strict>], "kind">;
+    }, z.core.$strict>>>;
+    network: z.ZodOptional<z.ZodEnum<{
+        bridge: "bridge";
+        none: "none";
+    }>>;
+    user: z.ZodOptional<z.ZodString>;
+    memory: z.ZodOptional<z.ZodString>;
+    cpus: z.ZodOptional<z.ZodNumber>;
+    pidsLimit: z.ZodOptional<z.ZodNumber>;
+    startupTimeoutMs: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strict>;
+export declare const DriverExecutionSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    kind: z.ZodLiteral<"host">;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"container">;
+    image: z.ZodString;
+    command: z.ZodOptional<z.ZodString>;
+    worker: z.ZodOptional<z.ZodString>;
+    driverModule: z.ZodOptional<z.ZodString>;
+    mounts: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        source: z.ZodString;
+        target: z.ZodOptional<z.ZodString>;
+        readOnly: z.ZodOptional<z.ZodBoolean>;
+    }, z.core.$strict>>>;
+    passEnv: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    homeFiles: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        destination: z.ZodString;
+        source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"file">;
+            path: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"keychain">;
+            service: z.ZodString;
+            account: z.ZodString;
+            encoding: z.ZodOptional<z.ZodEnum<{
+                "go-keyring-base64": "go-keyring-base64";
+                utf8: "utf8";
+            }>>;
+        }, z.core.$strict>], "kind">;
+    }, z.core.$strict>>>;
+    network: z.ZodOptional<z.ZodEnum<{
+        bridge: "bridge";
+        none: "none";
+    }>>;
+    user: z.ZodOptional<z.ZodString>;
+    memory: z.ZodOptional<z.ZodString>;
+    cpus: z.ZodOptional<z.ZodNumber>;
+    pidsLimit: z.ZodOptional<z.ZodNumber>;
+    startupTimeoutMs: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strict>], "kind">;
+export type DriverExecution = z.infer<typeof DriverExecutionSchema>;
+export type ContainerExecution = z.infer<typeof ContainerExecutionSchema>;
+export declare const driverExecutionOf: (reference: unknown) => DriverExecution | undefined;
+//# sourceMappingURL=driver-execution.d.ts.map

package/lib/core/driver-execution.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"driver-execution.d.ts","sourceRoot":"","sources":["../../src/core/driver-execution.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAaxB,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAwD1B,CAAC;AAEZ,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4BAGhC,CAAC;AACH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACpE,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAE1E,eAAO,MAAM,iBAAiB,cAAe,OAAO,KAAG,eAAe,GAAG,SAKxE,CAAC"}