@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.41

package/lib/orchestrator/suite-compare.js

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { loadSuiteSummary, MetricSummarySchema, summarizeNumbers, } from "./suite-summary.js";
+import { loadSuiteSummary, MetricSummarySchema, suiteGroupKey, summarizeNumbers, } from "./suite-summary.js";
 const ComparedOutcomeSchema = z
     .object({
     attempts: z.number().int().nonnegative(),
@@ -9,6 +9,7 @@ const ComparedOutcomeSchema = z
 export const SuiteCaseComparisonSchema = z
     .object({
     canonicalCaseId: z.string(),
+    permutationId: z.exactOptional(z.string()),
     before: ComparedOutcomeSchema.nullable(),
     after: ComparedOutcomeSchema.nullable(),
     passRateDelta: z.number().nullable(),
@@ -28,6 +29,7 @@ export const SuiteComparisonSchema = z
     identityMismatches: z.array(z
         .object({
         canonicalCaseId: z.string(),
+        permutationId: z.exactOptional(z.string()),
         repetition: z.number().int().positive(),
         before: z.string().nullable(),
         after: z.string().nullable(),
@@ -39,9 +41,10 @@ export const SuiteComparisonSchema = z
 const samplesByCase = (summary) => {
     const out = new Map();
     for (const sample of summary.samples) {
-        const entries = out.get(sample.canonicalCaseId) ?? [];
+        const key = suiteGroupKey(sample);
+        const entries = out.get(key) ?? [];
         entries.push(sample);
-        out.set(sample.canonicalCaseId, entries);
+        out.set(key, entries);
     }
     return out;
 };
@@ -52,10 +55,20 @@ const pairedMetricDeltas = (before, after) => {
         const b = after[index];
         if (!a || !b)
             continue;
-        const names = new Set([...Object.keys(a.metrics), ...Object.keys(b.metrics)]);
+        const am = {
+            ...a.metrics,
+            ...(a.score !== undefined ? { score: a.score } : {}),
+        };
+        const bm = {
+            ...b.metrics,
+            ...(b.score !== undefined ? { score: b.score } : {}),
+        };
+        const names = new Set([...Object.keys(am), ...Object.keys(bm)]);
         for (const name of names) {
-            const av = a.metrics[name];
-            const bv = b.metrics[name];
+            if (name === "score" && (!a.scoreBasis || a.scoreBasis !== b.scoreBasis))
+                continue;
+            const av = am[name];
+            const bv = bm[name];
             if (av === undefined || bv === undefined)
                 continue;
             const values = deltas.get(name) ?? [];
@@ -69,14 +82,21 @@ const pairedMetricDeltas = (before, after) => {
 };
 export const compareSuiteSummaries = (pathA, a, pathB, b, options = {}) => {
     const passRateTolerance = options.passRateTolerance ?? 0;
-    const aCases = new Map(a.cases.map((entry) => [entry.canonicalCaseId, entry]));
-    const bCases = new Map(b.cases.map((entry) => [entry.canonicalCaseId, entry]));
+    const aCases = new Map(a.cases.map((entry) => [suiteGroupKey(entry), entry]));
+    const bCases = new Map(b.cases.map((entry) => [suiteGroupKey(entry), entry]));
     const aSamples = samplesByCase(a);
     const bSamples = samplesByCase(b);
-    const ids = new Set([...aCases.keys(), ...bCases.keys()]);
-    const identityMismatches = [...ids].sort().flatMap((canonicalCaseId) => {
-        const before = aSamples.get(canonicalCaseId) ?? [];
-        const after = bSamples.get(canonicalCaseId) ?? [];
+    const groups = new Map([...a.cases, ...b.cases].map((entry) => [
+        suiteGroupKey(entry),
+        {
+            canonicalCaseId: entry.canonicalCaseId,
+            ...(entry.permutationId ? { permutationId: entry.permutationId } : {}),
+        },
+    ]));
+    const keys = [...groups.keys()].sort();
+    const identityMismatches = keys.flatMap((key) => {
+        const before = aSamples.get(key) ?? [];
+        const after = bSamples.get(key) ?? [];
         return Array.from({ length: Math.max(before.length, after.length) }, (_, index) => {
             const beforeHash = before[index]?.identityHash ?? null;
             const afterHash = after[index]?.identityHash ?? null;
@@ -84,7 +104,7 @@ export const compareSuiteSummaries = (pathA, a, pathB, b, options = {}) => {
                 ? []
                 : [
                     {
-                        canonicalCaseId,
+                        ...groups.get(key),
                         repetition: index + 1,
                         before: beforeHash,
                         after: afterHash,
@@ -92,9 +112,9 @@ export const compareSuiteSummaries = (pathA, a, pathB, b, options = {}) => {
                 ];
         }).flat();
     });
-    const cases = [...ids].sort().map((canonicalCaseId) => {
-        const before = aCases.get(canonicalCaseId);
-        const after = bCases.get(canonicalCaseId);
+    const cases = keys.map((key) => {
+        const before = aCases.get(key);
+        const after = bCases.get(key);
         let passRateDelta = null;
         if (before && after) {
             passRateDelta = after.passRate - before.passRate;
@@ -103,12 +123,12 @@ export const compareSuiteSummaries = (pathA, a, pathB, b, options = {}) => {
             passRateDelta = -before.passRate;
         }
         return {
-            canonicalCaseId,
+            ...groups.get(key),
             before: before ? { attempts: before.attempts, passRate: before.passRate } : null,
             after: after ? { attempts: after.attempts, passRate: after.passRate } : null,
             passRateDelta,
             regressed: Boolean(before) && (!after || after.passRate + passRateTolerance < (before?.passRate ?? 0)),
-            pairedMetricDeltas: pairedMetricDeltas(aSamples.get(canonicalCaseId) ?? [], bSamples.get(canonicalCaseId) ?? []),
+            pairedMetricDeltas: pairedMetricDeltas(aSamples.get(key) ?? [], bSamples.get(key) ?? []),
         };
     });
     return SuiteComparisonSchema.parse({
@@ -146,13 +166,15 @@ export const formatSuiteComparisonText = (comparison) => {
     if (comparison.identityMismatches.length > 0) {
         lines.push(`  identity mismatches: ${comparison.identityMismatches.length}`);
         for (const mismatch of comparison.identityMismatches) {
-            lines.push(`    ! ${mismatch.canonicalCaseId} repetition ${mismatch.repetition}`);
+            lines.push(`    ! ${suiteGroupKey(mismatch)} repetition ${mismatch.repetition}`);
         }
     }
     for (const entry of comparison.cases) {
-        if (!entry.regressed)
-            continue;
-        lines.push(`    ↓ ${entry.canonicalCaseId}: ${entry.before ? percent(entry.before.passRate) : "—"} → ${entry.after ? percent(entry.after.passRate) : "missing"}`);
+        if (entry.regressed) {
+            lines.push(`    ↓ ${suiteGroupKey(entry)}: ${entry.before ? percent(entry.before.passRate) : "—"} → ${entry.after ? percent(entry.after.passRate) : "missing"}`);
+        }
+        const score = entry.pairedMetricDeltas.score;
+        lines.push(`    ${suiteGroupKey(entry)} score change: ${score ? `${score.mean >= 0 ? "+" : ""}${score.mean} (${score.n} paired)` : "unavailable"}`);
     }
     return lines.join("\n");
 };

package/lib/orchestrator/suite-compare.js.map

@@ -1 +1 @@
-{"version":3,"file":"suite-compare.js","sourceRoot":"","sources":["../../src/orchestrator/suite-compare.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EACL,gBAAgB,EAEhB,mBAAmB,EAGnB,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,qBAAqB,GAAG,CAAC;KAC5B,MAAM,CAAC;IACN,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IACxC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CACnC,CAAC;KACD,MAAM,EAAE,CAAC;AAEZ,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC;KACvC,MAAM,CAAC;IACN,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;IAC3B,MAAM,EAAE,qBAAqB,CAAC,QAAQ,EAAE;IACxC,KAAK,EAAE,qBAAqB,CAAC,QAAQ,EAAE;IACvC,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE;IACtB,kBAAkB,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC;CAC9D,CAAC;KACD,MAAM,EAAE,CAAC;AAGZ,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC;KACnC,MAAM,CAAC;IACN,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE;IAC3C,MAAM,EAAE,qBAAqB;IAC7B,KAAK,EAAE,qBAAqB;IAC5B,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;IACzB,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,yBAAyB,CAAC;IACzC,kBAAkB,EAAE,CAAC,CAAC,KAAK,CACzB,CAAC;SACE,MAAM,CAAC;QACN,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;QAC3B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;QACvC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC7B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAC7B,CAAC;SACD,MAAM,EAAE,CACZ;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;CAC5C,CAAC;KACD,MAAM,EAAE,CAAC;AAGZ,MAAM,aAAa,GAAG,CAAC,OAAqB,EAA8B,EAAE;IAC1E,MAAM,GAAG,GAAG,IAAI,GAAG,EAAyB,CAAC;IAC7C,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACrC,MAAM,OAAO,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QACtD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CACzB,MAA8B,EAC9B,KAA6B,EACE,EAAE;IACjC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC3C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;QAC3E,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QACxB,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QACvB,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;YAAE,SAAS;QACvB,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC9E,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC3B,MAAM,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC3B,IAAI,EAAE,KAAK,SAAS,IAAI,EAAE,KAAK,SAAS;gBAAE,SAAS;YACnD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;YACrB,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC,WAAW,CACvB,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;SAClB,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;SACtC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,CAC7D,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,qBAAqB,GAAG,CACnC,KAAa,EACb,CAAe,EACf,KAAa,EACb,CAAe,EACf,OAAO,GAAmC,EAAE,EAC3B,EAAE;IACnB,MAAM,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,IAAI,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IAC/E,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IAC/E,MAAM,QAAQ,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;IAClC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,EAAE,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC1D,MAAM,kBAAkB,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,CAAC,eAAe,EAAE,EAAE;QACrE,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClD,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE;YAChF,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,YAAY,IAAI,IAAI,CAAC;YACvD,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,EAAE,YAAY,IAAI,IAAI,CAAC;YACrD,OAAO,UAAU,KAAK,SAAS,IAAI,UAAU,KAAK,IAAI;gBACpD,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC;oBACE;wBACE,eAAe;wBACf,UAAU,EAAE,KAAK,GAAG,CAAC;wBACrB,MAAM,EAAE,UAAU;wBAClB,KAAK,EAAE,SAAS;qBACjB;iBACF,CAAC;QACR,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,eAAe,EAAE,EAAE;QACpD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAC1C,IAAI,aAAa,GAAkB,IAAI,CAAC;QACxC,IAAI,MAAM,IAAI,KAAK,EAAE,CAAC;YACpB,aAAa,GAAG,KAAK,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QACnD,CAAC;aAAM,IAAI,MAAM,EAAE,CAAC;YAClB,aAAa,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;QACnC,CAAC;QACD,OAAO;YACL,eAAe;YACf,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI;YAChF,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI;YAC5E,aAAa;YACb,SAAS,EACP,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,QAAQ,GAAG,iBAAiB,GAAG,CAAC,MAAM,EAAE,QAAQ,IAAI,CAAC,CAAC,CAAC;YAC7F,kBAAkB,EAAE,kBAAkB,CACpC,QAAQ,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,EACnC,QAAQ,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,CACpC;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO,qBAAqB,CAAC,KAAK,CAAC;QACjC,KAAK;QACL,KAAK;QACL,iBAAiB;QACjB,MAAM,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;QACpE,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;QACnE,aAAa,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,QAAQ;QACpD,KAAK;QACL,kBAAkB;QAClB,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM;KAC7D,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,EACnC,KAAa,EACb,KAAa,EACb,OAAO,GAAmC,EAAE,EAClB,EAAE;IAC5B,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACrF,IAAI,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,EAAE,CAAC,CAAC;IAC/D,IAAI,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,EAAE,CAAC,CAAC;IAC/D,OAAO,qBAAqB,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC;AAC5D,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,UAA2B,EAAW,EAAE,CACzE,UAAU,CAAC,WAAW,GAAG,CAAC,IAAI,UAAU,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC;AAEzE,MAAM,OAAO,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AAE1E,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,UAA2B,EAAU,EAAE;IAC/E,MAAM,KAAK,GAAG;QACZ,MAAM,UAAU,CAAC,KAAK,EAAE;QACxB,MAAM,UAAU,CAAC,KAAK,EAAE;QACxB,EAAE;QACF,QAAQ;QACR,gBAAgB,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,UAAU,CAAC,aAAa,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,GAAG;QAC/K,gBAAgB,UAAU,CAAC,MAAM,CAAC,QAAQ,MAAM,UAAU,CAAC,KAAK,CAAC,QAAQ,EAAE;QAC3E,kBAAkB,UAAU,CAAC,WAAW,EAAE;KAC3C,CAAC;IACF,IAAI,UAAU,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7C,KAAK,CAAC,IAAI,CAAC,0BAA0B,UAAU,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7E,KAAK,MAAM,QAAQ,IAAI,UAAU,CAAC,kBAAkB,EAAE,CAAC;YACrD,KAAK,CAAC,IAAI,CAAC,SAAS,QAAQ,CAAC,eAAe,eAAe,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;QACrC,IAAI,CAAC,KAAK,CAAC,SAAS;YAAE,SAAS;QAC/B,KAAK,CAAC,IAAI,CACR,SAAS,KAAK,CAAC,eAAe,KAAK,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CACtJ,CAAC;IACJ,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC,CAAC"}
+{"version":3,"file":"suite-compare.js","sourceRoot":"","sources":["../../src/orchestrator/suite-compare.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EACL,gBAAgB,EAEhB,mBAAmB,EAGnB,aAAa,EACb,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,qBAAqB,GAAG,CAAC;KAC5B,MAAM,CAAC;IACN,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IACxC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CACnC,CAAC;KACD,MAAM,EAAE,CAAC;AAEZ,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC;KACvC,MAAM,CAAC;IACN,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;IAC3B,aAAa,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IAC1C,MAAM,EAAE,qBAAqB,CAAC,QAAQ,EAAE;IACxC,KAAK,EAAE,qBAAqB,CAAC,QAAQ,EAAE;IACvC,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE;IACtB,kBAAkB,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC;CAC9D,CAAC;KACD,MAAM,EAAE,CAAC;AAGZ,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC;KACnC,MAAM,CAAC;IACN,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE;IAC3C,MAAM,EAAE,qBAAqB;IAC7B,KAAK,EAAE,qBAAqB;IAC5B,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;IACzB,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,yBAAyB,CAAC;IACzC,kBAAkB,EAAE,CAAC,CAAC,KAAK,CACzB,CAAC;SACE,MAAM,CAAC;QACN,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;QAC3B,aAAa,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1C,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;QACvC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC7B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAC7B,CAAC;SACD,MAAM,EAAE,CACZ;IACD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;CAC5C,CAAC;KACD,MAAM,EAAE,CAAC;AAGZ,MAAM,aAAa,GAAG,CAAC,OAAqB,EAA8B,EAAE;IAC1E,MAAM,GAAG,GAAG,IAAI,GAAG,EAAyB,CAAC;IAC7C,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;QAClC,MAAM,OAAO,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACxB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CACzB,MAA8B,EAC9B,KAA6B,EACE,EAAE;IACjC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC3C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;QAC3E,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QACxB,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QACvB,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;YAAE,SAAS;QACvB,MAAM,EAAE,GAA2B;YACjC,GAAG,CAAC,CAAC,OAAO;YACZ,GAAG,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrD,CAAC;QACF,MAAM,EAAE,GAA2B;YACjC,GAAG,CAAC,CAAC,OAAO;YACZ,GAAG,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrD,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,KAAK,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU,CAAC;gBAAE,SAAS;YACnF,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;YACpB,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;YACpB,IAAI,EAAE,KAAK,SAAS,IAAI,EAAE,KAAK,SAAS;gBAAE,SAAS;YACnD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;YACrB,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC,WAAW,CACvB,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;SAClB,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;SACtC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,CAC7D,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,qBAAqB,GAAG,CACnC,KAAa,EACb,CAAe,EACf,KAAa,EACb,CAAe,EACf,OAAO,GAAmC,EAAE,EAC3B,EAAE;IACnB,MAAM,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,IAAI,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IAC9E,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IAC9E,MAAM,QAAQ,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;IAClC,MAAM,MAAM,GAAG,IAAI,GAAG,CACpB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;QACtC,aAAa,CAAC,KAAK,CAAC;QACpB;YACE,eAAe,EAAE,KAAK,CAAC,eAAe;YACtC,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACvE;KACF,CAAC,CACH,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACvC,MAAM,kBAAkB,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;QAC9C,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACtC,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE;YAChF,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,YAAY,IAAI,IAAI,CAAC;YACvD,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,EAAE,YAAY,IAAI,IAAI,CAAC;YACrD,OAAO,UAAU,KAAK,SAAS,IAAI,UAAU,KAAK,IAAI;gBACpD,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC;oBACE;wBACE,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;wBAClB,UAAU,EAAE,KAAK,GAAG,CAAC;wBACrB,MAAM,EAAE,UAAU;wBAClB,KAAK,EAAE,SAAS;qBACjB;iBACF,CAAC;QACR,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QAC7B,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,aAAa,GAAkB,IAAI,CAAC;QACxC,IAAI,MAAM,IAAI,KAAK,EAAE,CAAC;YACpB,aAAa,GAAG,KAAK,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QACnD,CAAC;aAAM,IAAI,MAAM,EAAE,CAAC;YAClB,aAAa,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;QACnC,CAAC;QACD,OAAO;YACL,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;YAClB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI;YAChF,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI;YAC5E,aAAa;YACb,SAAS,EACP,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,QAAQ,GAAG,iBAAiB,GAAG,CAAC,MAAM,EAAE,QAAQ,IAAI,CAAC,CAAC,CAAC;YAC7F,kBAAkB,EAAE,kBAAkB,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;SACzF,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO,qBAAqB,CAAC,KAAK,CAAC;QACjC,KAAK;QACL,KAAK;QACL,iBAAiB;QACjB,MAAM,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;QACpE,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;QACnE,aAAa,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,QAAQ;QACpD,KAAK;QACL,kBAAkB;QAClB,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM;KAC7D,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,EACnC,KAAa,EACb,KAAa,EACb,OAAO,GAAmC,EAAE,EAClB,EAAE;IAC5B,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACrF,IAAI,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,EAAE,CAAC,CAAC;IAC/D,IAAI,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,EAAE,CAAC,CAAC;IAC/D,OAAO,qBAAqB,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC;AAC5D,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,UAA2B,EAAW,EAAE,CACzE,UAAU,CAAC,WAAW,GAAG,CAAC,IAAI,UAAU,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC;AAEzE,MAAM,OAAO,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AAE1E,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,UAA2B,EAAU,EAAE;IAC/E,MAAM,KAAK,GAAG;QACZ,MAAM,UAAU,CAAC,KAAK,EAAE;QACxB,MAAM,UAAU,CAAC,KAAK,EAAE;QACxB,EAAE;QACF,QAAQ;QACR,gBAAgB,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,UAAU,CAAC,aAAa,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,GAAG;QAC/K,gBAAgB,UAAU,CAAC,MAAM,CAAC,QAAQ,MAAM,UAAU,CAAC,KAAK,CAAC,QAAQ,EAAE;QAC3E,kBAAkB,UAAU,CAAC,WAAW,EAAE;KAC3C,CAAC;IACF,IAAI,UAAU,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7C,KAAK,CAAC,IAAI,CAAC,0BAA0B,UAAU,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7E,KAAK,MAAM,QAAQ,IAAI,UAAU,CAAC,kBAAkB,EAAE,CAAC;YACrD,KAAK,CAAC,IAAI,CAAC,SAAS,aAAa,CAAC,QAAQ,CAAC,eAAe,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACnF,CAAC;IACH,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;QACrC,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACpB,KAAK,CAAC,IAAI,CACR,SAAS,aAAa,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CACrJ,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GAAG,KAAK,CAAC,kBAAkB,CAAC,KAAK,CAAC;QAC7C,KAAK,CAAC,IAAI,CACR,OAAO,aAAa,CAAC,KAAK,CAAC,kBAAkB,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,aAAa,EAAE,CACxI,CAAC;IACJ,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC,CAAC"}

package/lib/orchestrator/suite-runs.d.ts

@@ -2,6 +2,7 @@ import { z } from "zod";
 declare const SuiteItemSchema: z.ZodObject<{
     caseId: z.ZodString;
     canonicalCaseId: z.ZodString;
+    permutationId: z.ZodExactOptional<z.ZodString>;
     status: z.ZodEnum<{
         completed: "completed";
         pending: "pending";
@@ -40,6 +41,7 @@ export declare const SuiteRunSchema: z.ZodObject<{
     items: z.ZodArray<z.ZodObject<{
         caseId: z.ZodString;
         canonicalCaseId: z.ZodString;
+        permutationId: z.ZodExactOptional<z.ZodString>;
         status: z.ZodEnum<{
             completed: "completed";
             pending: "pending";
@@ -65,6 +67,7 @@ export declare const openSuiteRun: (options: {
     items: {
         id: string;
         caseId?: string | undefined;
+        permutationId?: string | undefined;
     }[];
     startedAt?: Date | undefined;
     profileId?: string | undefined;
@@ -80,7 +83,9 @@ export declare const openSuiteRun: (options: {
     }> | undefined;
 }) => Promise<OpenedSuiteRun>;
 export declare const loadSuiteRun: (path: string) => Promise<SuiteRun | null>;
-export declare const listSuiteRuns: (runsRoot: string) => Promise<Array<SuiteRun & {
+export declare const listSuiteRuns: (runsRoot: string, options?: {
+    onReadError?: (path: string, error: unknown) => void;
+}) => Promise<Array<SuiteRun & {
     suiteDir: string;
 }>>;
 export declare const formatSuiteRunsTable: (suites: Array<SuiteRun & {

package/lib/orchestrator/suite-runs.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"suite-runs.d.ts","sourceRoot":"","sources":["../../src/orchestrator/suite-runs.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAMxB,QAAA,MAAM,eAAe;;;;;;;;;;;;iBAQnB,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAyBzB,CAAC;AACH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AACtD,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAU3D,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,cAAc,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;CAC5C,CAAC;AAEF,eAAO,MAAM,YAAY,YAAmB;IAC1C,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,EAAE,CAAC;IACrD,SAAS,CAAC,EAAE,IAAI,GAAG,SAAS,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,gBAAgB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,mBAAmB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,QAAQ,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,SAAS,CAAC;CACvE,KAAG,OAAO,CAAC,cAAc,CA6CzB,CAAC;AAEF,eAAO,MAAM,YAAY,SAAgB,MAAM,KAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CACnB,CAAC;AAEvD,eAAO,MAAM,aAAa,aACd,MAAM,KACf,OAAO,CAAC,KAAK,CAAC,QAAQ,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,CAYhD,CAAC;AAIF,eAAO,MAAM,oBAAoB,WAAY,KAAK,CAAC,QAAQ,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,KAAG,MA+BrF,CAAC;AAEF,eAAO,MAAM,oBAAoB,aAAc,MAAM,SAAS,QAAQ,KAAG,MAwBxE,CAAC"}
+{"version":3,"file":"suite-runs.d.ts","sourceRoot":"","sources":["../../src/orchestrator/suite-runs.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAMxB,QAAA,MAAM,eAAe;;;;;;;;;;;;;iBASnB,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAyBzB,CAAC;AACH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AACtD,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAU3D,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,cAAc,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;CAC5C,CAAC;AAEF,eAAO,MAAM,YAAY,YAAmB;IAC1C,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,EAAE,CAAC;IACzF,SAAS,CAAC,EAAE,IAAI,GAAG,SAAS,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,gBAAgB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,mBAAmB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,QAAQ,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,SAAS,CAAC;CACvE,KAAG,OAAO,CAAC,cAAc,CA8CzB,CAAC;AAEF,eAAO,MAAM,YAAY,SAAgB,MAAM,KAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CACnB,CAAC;AAEvD,eAAO,MAAM,aAAa,aACd,MAAM,YACP;IAAE,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,KAAK,IAAI,CAAA;CAAE,KAChE,OAAO,CAAC,KAAK,CAAC,QAAQ,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,CAoBhD,CAAC;AAIF,eAAO,MAAM,oBAAoB,WAAY,KAAK,CAAC,QAAQ,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,KAAG,MA+BrF,CAAC;AAEF,eAAO,MAAM,oBAAoB,aAAc,MAAM,SAAS,QAAQ,KAAG,MAwBxE,CAAC"}

package/lib/orchestrator/suite-runs.js

@@ -8,6 +8,7 @@ const SUITE_FILE = "suite.json";
 const SuiteItemSchema = z.object({
     caseId: z.string(),
     canonicalCaseId: z.string(),
+    permutationId: z.exactOptional(z.string()),
     status: z.enum(["pending", "running", "completed"]),
     runDir: z.string().nullable(),
     exitReason: z.union([z.literal("done"), z.string()]).nullable(),
@@ -82,6 +83,7 @@ export const openSuiteRun = async (options) => {
         items: options.items.map((item) => ({
             caseId: item.id,
             canonicalCaseId: item.caseId ?? item.id,
+            ...(item.permutationId ? { permutationId: item.permutationId } : {}),
             status: "pending",
             runDir: null,
             exitReason: null,
@@ -93,14 +95,24 @@ export const openSuiteRun = async (options) => {
     return { path, manifest, write, relativeRunDir: (runDir) => relative(path, runDir) };
 };
 export const loadSuiteRun = async (path) => safeReadJson(join(path, SUITE_FILE), SuiteRunSchema);
-export const listSuiteRuns = async (runsRoot) => {
+export const listSuiteRuns = async (runsRoot, options = {}) => {
     const root = join(runsRoot, SUITES_DIR);
     if (!existsSync(root))
         return [];
     const dirs = (await readdir(root, { withFileTypes: true }))
         .filter((entry) => entry.isDirectory())
         .map((entry) => join(root, entry.name));
-    const manifests = await Promise.all(dirs.map(async (suiteDir) => ({ suiteDir, suite: await loadSuiteRun(suiteDir) })));
+    const manifests = await Promise.all(dirs.map(async (suiteDir) => {
+        try {
+            return { suiteDir, suite: await loadSuiteRun(suiteDir) };
+        }
+        catch (error) {
+            if (!options.onReadError)
+                throw error;
+            options.onReadError(suiteDir, error);
+            return { suiteDir, suite: null };
+        }
+    }));
     return manifests
         .flatMap(({ suiteDir, suite }) => (suite ? [{ ...suite, suiteDir }] : []))
         .sort((a, b) => b.suiteId.localeCompare(a.suiteId));

package/lib/orchestrator/suite-runs.js.map

@@ -1 +1 @@
-{"version":3,"file":"suite-runs.js","sourceRoot":"","sources":["../../src/orchestrator/suite-runs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC9D,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,UAAU,GAAG,SAAS,CAAC;AAC7B,MAAM,UAAU,GAAG,YAAY,CAAC;AAEhC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;IAC3B,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;IACnD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC/D,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAClC,YAAY,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;CAC1C,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC1D,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE;IACpB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACxC,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACtC,YAAY,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACzC,gBAAgB,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IAC7C,mBAAmB,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IAChD,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACtC,WAAW,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACxC,WAAW,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACxC,YAAY,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACzC,QAAQ,EAAE,CAAC;SACR,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;QACP,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;QAChB,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;KAC1B,CAAC,CACH;SACA,QAAQ,EAAE;IACb,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC;CAChC,CAAC,CAAC;AAIH,MAAM,WAAW,GAAG,CAAC,IAAU,EAAU,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AAErF,MAAM,eAAe,GAAG,KAAK,EAAE,IAAY,EAAE,KAAc,EAAiB,EAAE;IAC5E,MAAM,SAAS,GAAG,GAAG,IAAI,IAAI,OAAO,CAAC,GAAG,MAAM,CAAC;IAC/C,MAAM,SAAS,CAAC,SAAS,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IAClE,MAAM,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;AAChC,CAAC,CAAC;AASF,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,EAAE,OAalC,EAA2B,EAAE;IAC5B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;IAClD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,UAAU,CAAC,CAAC;IAC/D,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;IACtC,IAAI,OAAO,GAAG,MAAM,CAAC;IACrB,IAAI,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACrC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAI,CAAC,EAAE,EAAE,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;YAClB,MAAM;QACR,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ;gBAAE,MAAM,GAAG,CAAC;YAChE,OAAO,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC;YAC3B,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IACD,MAAM,QAAQ,GAAa;QACzB,aAAa,EAAE,CAAC;QAChB,OAAO;QACP,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;QAClC,OAAO,EAAE,IAAI;QACb,MAAM,EAAE,SAAS;QACjB,OAAO,EAAE,KAAK;QACd,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnF,GAAG,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5F,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpE,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpE,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAClC,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,eAAe,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,EAAE;YACvC,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,IAAI;YACZ,UAAU,EAAE,IAAI;YAChB,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;KACJ,CAAC;IACF,MAAM,KAAK,GAAG,GAAkB,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,QAAQ,CAAC,CAAC;IACrF,MAAM,KAAK,EAAE,CAAC;IACd,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC;AACvF,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,EAAE,IAAY,EAA4B,EAAE,CAC3E,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,cAAc,CAAC,CAAC;AAEvD,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,EAChC,QAAgB,EACiC,EAAE;IACnD,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACxC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,MAAM,IAAI,GAAG,CAAC,MAAM,OAAO,CAAC,IAAI,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;SACxD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;SACtC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CACjC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAClF,CAAC;IACF,OAAO,SAAS;SACb,OAAO,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;SACzE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;AACxD,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CAAC,QAAgB,EAAE,MAAc,EAAU,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;AAEnG,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,MAA8C,EAAU,EAAE;IAC7F,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,uBAAuB,CAAC;IACxD,MAAM,IAAI,GAAG;QACX,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,CAAC;QACvE,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;YACvB,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE;YAC1B,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC;YACxB,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;YAC1B,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;YACrE,MAAM,CACJ,KAAK,CAAC,KAAK,CAAC,MAAM,CAChB,CAAC,IAAI,EAAE,EAAE,CACP,IAAI,CAAC,MAAM,KAAK,WAAW;gBAC3B,CAAC,IAAI,CAAC,UAAU,KAAK,MAAM,IAAI,IAAI,CAAC,UAAU,KAAK,KAAK,CAAC,CAC5D,CAAC,MAAM,CACT;YACD,MAAM,CACJ,KAAK,CAAC,KAAK,CAAC,MAAM,CAChB,CAAC,IAAI,EAAE,EAAE,CACP,IAAI,CAAC,MAAM,KAAK,WAAW,IAAI,IAAI,CAAC,UAAU,KAAK,MAAM,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,CACxF,CAAC,MAAM,CACT;YACD,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,MAAM,CAAC;SACzE,CAAC;KACH,CAAC;IACF,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAC/C,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC,CACzD,CAAC;IACF,OAAO,IAAI;SACR,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACpF,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,QAAgB,EAAE,KAAe,EAAU,EAAE;IAChF,MAAM,KAAK,GAAG;QACZ,aAAa,QAAQ,EAAE;QACvB,aAAa,KAAK,CAAC,OAAO,EAAE;QAC5B,aAAa,KAAK,CAAC,MAAM,EAAE;QAC3B,aAAa,KAAK,CAAC,SAAS,EAAE;QAC9B,aAAa,KAAK,CAAC,OAAO,IAAI,GAAG,EAAE;QACnC,aAAa,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE;QACjC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,aAAa,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,EAAE;KACH,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE;YACnB,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW;gBAAE,OAAO,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAClE,IAAI,IAAI,CAAC,UAAU,KAAK,MAAM,IAAI,IAAI,CAAC,UAAU,KAAK,KAAK;gBAAE,OAAO,MAAM,CAAC;YAC3E,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI;gBAAE,OAAO,MAAM,CAAC;YAC5C,OAAO,UAAU,CAAC;QACpB,CAAC,CAAC,EAAE,CAAC;QACL,KAAK,CAAC,IAAI,CACR,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,kBAAkB,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAC3G,CAAC;QACF,IAAI,IAAI,CAAC,YAAY;YAAE,KAAK,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC,CAAC"}
+{"version":3,"file":"suite-runs.js","sourceRoot":"","sources":["../../src/orchestrator/suite-runs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC9D,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,UAAU,GAAG,SAAS,CAAC;AAC7B,MAAM,UAAU,GAAG,YAAY,CAAC;AAEhC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;IAC3B,aAAa,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IAC1C,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;IACnD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC/D,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAClC,YAAY,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;CAC1C,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC1D,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE;IACpB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACxC,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACtC,YAAY,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACzC,gBAAgB,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IAC7C,mBAAmB,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IAChD,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACtC,WAAW,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACxC,WAAW,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACxC,YAAY,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACzC,QAAQ,EAAE,CAAC;SACR,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;QACP,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;QAChB,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;KAC1B,CAAC,CACH;SACA,QAAQ,EAAE;IACb,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC;CAChC,CAAC,CAAC;AAIH,MAAM,WAAW,GAAG,CAAC,IAAU,EAAU,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AAErF,MAAM,eAAe,GAAG,KAAK,EAAE,IAAY,EAAE,KAAc,EAAiB,EAAE;IAC5E,MAAM,SAAS,GAAG,GAAG,IAAI,IAAI,OAAO,CAAC,GAAG,MAAM,CAAC;IAC/C,MAAM,SAAS,CAAC,SAAS,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IAClE,MAAM,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;AAChC,CAAC,CAAC;AASF,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,EAAE,OAalC,EAA2B,EAAE;IAC5B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;IAClD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,UAAU,CAAC,CAAC;IAC/D,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;IACtC,IAAI,OAAO,GAAG,MAAM,CAAC;IACrB,IAAI,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACrC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAI,CAAC,EAAE,EAAE,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;YAClB,MAAM;QACR,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ;gBAAE,MAAM,GAAG,CAAC;YAChE,OAAO,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC;YAC3B,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IACD,MAAM,QAAQ,GAAa;QACzB,aAAa,EAAE,CAAC;QAChB,OAAO;QACP,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;QAClC,OAAO,EAAE,IAAI;QACb,MAAM,EAAE,SAAS;QACjB,OAAO,EAAE,KAAK;QACd,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnF,GAAG,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5F,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpE,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpE,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAClC,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,eAAe,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,EAAE;YACvC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpE,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,IAAI;YACZ,UAAU,EAAE,IAAI;YAChB,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;KACJ,CAAC;IACF,MAAM,KAAK,GAAG,GAAkB,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,QAAQ,CAAC,CAAC;IACrF,MAAM,KAAK,EAAE,CAAC;IACd,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC;AACvF,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,EAAE,IAAY,EAA4B,EAAE,CAC3E,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,cAAc,CAAC,CAAC;AAEvD,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,EAChC,QAAgB,EAChB,OAAO,GAA6D,EAAE,EACrB,EAAE;IACnD,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACxC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,MAAM,IAAI,GAAG,CAAC,MAAM,OAAO,CAAC,IAAI,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;SACxD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;SACtC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CACjC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;QAC1B,IAAI,CAAC;YACH,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC3D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,OAAO,CAAC,WAAW;gBAAE,MAAM,KAAK,CAAC;YACtC,OAAO,CAAC,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACrC,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;QACnC,CAAC;IACH,CAAC,CAAC,CACH,CAAC;IACF,OAAO,SAAS;SACb,OAAO,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;SACzE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;AACxD,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CAAC,QAAgB,EAAE,MAAc,EAAU,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;AAEnG,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,MAA8C,EAAU,EAAE;IAC7F,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,uBAAuB,CAAC;IACxD,MAAM,IAAI,GAAG;QACX,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,CAAC;QACvE,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;YACvB,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE;YAC1B,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC;YACxB,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;YAC1B,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;YACrE,MAAM,CACJ,KAAK,CAAC,KAAK,CAAC,MAAM,CAChB,CAAC,IAAI,EAAE,EAAE,CACP,IAAI,CAAC,MAAM,KAAK,WAAW;gBAC3B,CAAC,IAAI,CAAC,UAAU,KAAK,MAAM,IAAI,IAAI,CAAC,UAAU,KAAK,KAAK,CAAC,CAC5D,CAAC,MAAM,CACT;YACD,MAAM,CACJ,KAAK,CAAC,KAAK,CAAC,MAAM,CAChB,CAAC,IAAI,EAAE,EAAE,CACP,IAAI,CAAC,MAAM,KAAK,WAAW,IAAI,IAAI,CAAC,UAAU,KAAK,MAAM,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,CACxF,CAAC,MAAM,CACT;YACD,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,MAAM,CAAC;SACzE,CAAC;KACH,CAAC;IACF,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAC/C,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC,CACzD,CAAC;IACF,OAAO,IAAI;SACR,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACpF,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,QAAgB,EAAE,KAAe,EAAU,EAAE;IAChF,MAAM,KAAK,GAAG;QACZ,aAAa,QAAQ,EAAE;QACvB,aAAa,KAAK,CAAC,OAAO,EAAE;QAC5B,aAAa,KAAK,CAAC,MAAM,EAAE;QAC3B,aAAa,KAAK,CAAC,SAAS,EAAE;QAC9B,aAAa,KAAK,CAAC,OAAO,IAAI,GAAG,EAAE;QACnC,aAAa,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE;QACjC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,aAAa,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,EAAE;KACH,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE;YACnB,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW;gBAAE,OAAO,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAClE,IAAI,IAAI,CAAC,UAAU,KAAK,MAAM,IAAI,IAAI,CAAC,UAAU,KAAK,KAAK;gBAAE,OAAO,MAAM,CAAC;YAC3E,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI;gBAAE,OAAO,MAAM,CAAC;YAC5C,OAAO,UAAU,CAAC;QACpB,CAAC,CAAC,EAAE,CAAC;QACL,KAAK,CAAC,IAAI,CACR,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,kBAAkB,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAC3G,CAAC;QACF,IAAI,IAAI,CAAC,YAAY;YAAE,KAAK,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC,CAAC"}

package/lib/orchestrator/suite-summary.d.ts

@@ -8,6 +8,23 @@ export declare const MetricSummarySchema: z.ZodObject<{
     mean: z.ZodNumber;
     median: z.ZodNumber;
     p90: z.ZodNumber;
+    unit: z.ZodExactOptional<z.ZodEnum<{
+        bytes: "bytes";
+        count: "count";
+        milliseconds: "milliseconds";
+        number: "number";
+        percent: "percent";
+        ratio: "ratio";
+        score: "score";
+        tokens: "tokens";
+        usd: "usd";
+    }>>;
+    label: z.ZodExactOptional<z.ZodString>;
+    better: z.ZodExactOptional<z.ZodEnum<{
+        higher: "higher";
+        lower: "lower";
+        neutral: "neutral";
+    }>>;
 }, z.core.$strip>;
 export type MetricSummary = z.infer<typeof MetricSummarySchema>;
 declare const OutcomeSummarySchema: z.ZodObject<{
@@ -29,6 +46,7 @@ declare const CaseSummarySchema: z.ZodObject<{
     pending: z.ZodNumber;
     passRate: z.ZodNumber;
     canonicalCaseId: z.ZodString;
+    permutationId: z.ZodExactOptional<z.ZodString>;
     metrics: z.ZodRecord<z.ZodString, z.ZodObject<{
         n: z.ZodNumber;
         min: z.ZodNumber;
@@ -36,17 +54,41 @@ declare const CaseSummarySchema: z.ZodObject<{
         mean: z.ZodNumber;
         median: z.ZodNumber;
         p90: z.ZodNumber;
+        unit: z.ZodExactOptional<z.ZodEnum<{
+            bytes: "bytes";
+            count: "count";
+            milliseconds: "milliseconds";
+            number: "number";
+            percent: "percent";
+            ratio: "ratio";
+            score: "score";
+            tokens: "tokens";
+            usd: "usd";
+        }>>;
+        label: z.ZodExactOptional<z.ZodString>;
+        better: z.ZodExactOptional<z.ZodEnum<{
+            higher: "higher";
+            lower: "lower";
+            neutral: "neutral";
+        }>>;
     }, z.core.$strip>>;
 }, z.core.$strip>;
+/** Repetitions pool per case and harness permutation, never across permutations. */
+export declare const suiteGroupKey: (entry: {
+    canonicalCaseId: string;
+    permutationId?: string | undefined;
+}) => string;
 export type CaseSummary = z.infer<typeof CaseSummarySchema>;
 export declare const SuiteSampleSchema: z.ZodObject<{
     index: z.ZodNumber;
     caseId: z.ZodString;
     canonicalCaseId: z.ZodString;
+    permutationId: z.ZodExactOptional<z.ZodString>;
     exitReason: z.ZodNullable<z.ZodString>;
     passed: z.ZodNullable<z.ZodBoolean>;
     identityHash: z.ZodNullable<z.ZodString>;
     score: z.ZodExactOptional<z.ZodNumber>;
+    scoreBasis: z.ZodExactOptional<z.ZodNullable<z.ZodString>>;
     metrics: z.ZodRecord<z.ZodString, z.ZodNumber>;
 }, z.core.$strip>;
 export type SuiteSample = z.infer<typeof SuiteSampleSchema>;
@@ -70,6 +112,23 @@ export declare const SuiteSummarySchema: z.ZodObject<{
         mean: z.ZodNumber;
         median: z.ZodNumber;
         p90: z.ZodNumber;
+        unit: z.ZodExactOptional<z.ZodEnum<{
+            bytes: "bytes";
+            count: "count";
+            milliseconds: "milliseconds";
+            number: "number";
+            percent: "percent";
+            ratio: "ratio";
+            score: "score";
+            tokens: "tokens";
+            usd: "usd";
+        }>>;
+        label: z.ZodExactOptional<z.ZodString>;
+        better: z.ZodExactOptional<z.ZodEnum<{
+            higher: "higher";
+            lower: "lower";
+            neutral: "neutral";
+        }>>;
     }, z.core.$strip>>;
     cases: z.ZodArray<z.ZodObject<{
         attempts: z.ZodNumber;
@@ -80,6 +139,7 @@ export declare const SuiteSummarySchema: z.ZodObject<{
         pending: z.ZodNumber;
         passRate: z.ZodNumber;
         canonicalCaseId: z.ZodString;
+        permutationId: z.ZodExactOptional<z.ZodString>;
         metrics: z.ZodRecord<z.ZodString, z.ZodObject<{
             n: z.ZodNumber;
             min: z.ZodNumber;
@@ -87,16 +147,35 @@ export declare const SuiteSummarySchema: z.ZodObject<{
             mean: z.ZodNumber;
             median: z.ZodNumber;
             p90: z.ZodNumber;
+            unit: z.ZodExactOptional<z.ZodEnum<{
+                bytes: "bytes";
+                count: "count";
+                milliseconds: "milliseconds";
+                number: "number";
+                percent: "percent";
+                ratio: "ratio";
+                score: "score";
+                tokens: "tokens";
+                usd: "usd";
+            }>>;
+            label: z.ZodExactOptional<z.ZodString>;
+            better: z.ZodExactOptional<z.ZodEnum<{
+                higher: "higher";
+                lower: "lower";
+                neutral: "neutral";
+            }>>;
         }, z.core.$strip>>;
     }, z.core.$strip>>;
     samples: z.ZodArray<z.ZodObject<{
         index: z.ZodNumber;
         caseId: z.ZodString;
         canonicalCaseId: z.ZodString;
+        permutationId: z.ZodExactOptional<z.ZodString>;
         exitReason: z.ZodNullable<z.ZodString>;
         passed: z.ZodNullable<z.ZodBoolean>;
         identityHash: z.ZodNullable<z.ZodString>;
         score: z.ZodExactOptional<z.ZodNumber>;
+        scoreBasis: z.ZodExactOptional<z.ZodNullable<z.ZodString>>;
         metrics: z.ZodRecord<z.ZodString, z.ZodNumber>;
     }, z.core.$strip>>;
 }, z.core.$strip>;

package/lib/orchestrator/suite-summary.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"suite-summary.d.ts","sourceRoot":"","sources":["../../src/orchestrator/suite-summary.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAIhD,eAAO,MAAM,mBAAmB;;;;;;;iBAO9B,CAAC;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,QAAA,MAAM,oBAAoB;;;;;;;;iBAQxB,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAElE,QAAA,MAAM,iBAAiB;;;;;;;;;;;;;;;;;iBAGrB,CAAC;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAE5D,eAAO,MAAM,iBAAiB;;;;;;;;;iBAS5B,CAAC;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAE5D,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAQ7B,CAAC;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAuF9D,eAAO,MAAM,gBAAgB,WAAY,SAAS,MAAM,EAAE,KAAG,aAe5D,CAAC;AAuCF,eAAO,MAAM,iBAAiB,UACrB,QAAQ,WACN,SAAS,CAAC,eAAe,GAAG,SAAS,CAAC,EAAE,gBACpC,IAAI,KAChB,YA6DF,CAAC;AAEF,eAAO,MAAM,iBAAiB,aAAoB,MAAM,WAAW,YAAY,KAAG,OAAO,CAAC,IAAI,CAK7F,CAAC;AAEF,eAAO,MAAM,gBAAgB,aAAoB,MAAM,KAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CACrB,CAAC"}
+{"version":3,"file":"suite-summary.d.ts","sourceRoot":"","sources":["../../src/orchestrator/suite-summary.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAYxB,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAIhD,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;iBAU9B,CAAC;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,QAAA,MAAM,oBAAoB;;;;;;;;iBAQxB,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAElE,QAAA,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAIrB,CAAC;AAEH,oFAAoF;AACpF,eAAO,MAAM,aAAa,UAAW;IACnC,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,KAAG,MAGqD,CAAC;AAC1D,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAE5D,eAAO,MAAM,iBAAiB;;;;;;;;;;;iBAW5B,CAAC;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAE5D,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAQ7B,CAAC;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAwK9D,eAAO,MAAM,gBAAgB,WAAY,SAAS,MAAM,EAAE,KAAG,aAe5D,CAAC;AA0CF,eAAO,MAAM,iBAAiB,UACrB,QAAQ,WACN,SAAS,CAAC,eAAe,GAAG,SAAS,CAAC,EAAE,gBACpC,IAAI,KAChB,YAoEF,CAAC;AAEF,eAAO,MAAM,iBAAiB,aAAoB,MAAM,WAAW,YAAY,KAAG,OAAO,CAAC,IAAI,CAK7F,CAAC;AAEF,eAAO,MAAM,gBAAgB,aAAoB,MAAM,KAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CACrB,CAAC"}

package/lib/orchestrator/suite-summary.js

@@ -1,9 +1,11 @@
 import { rename, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { z } from "zod";
-import { harnessUsage, isUsageEmpty } from "../core/index.js";
+import { DriverMetricUnitSchema, harnessUsage, isUsageEmpty, } from "../core/index.js";
+import { hasAgentUsage } from "../core/usage.js";
 import { runComparisonIdentity } from "./comparison-identity.js";
 import { safeReadJson } from "./fs-read.js";
+import { gradingScoreBasis } from "./grading.js";
 const SUMMARY_FILE = "suite-summary.json";
 export const MetricSummarySchema = z.object({
     n: z.number().int().nonnegative(),
@@ -12,6 +14,9 @@ export const MetricSummarySchema = z.object({
     mean: z.number(),
     median: z.number(),
     p90: z.number(),
+    unit: z.exactOptional(DriverMetricUnitSchema),
+    label: z.exactOptional(z.string()),
+    better: z.exactOptional(z.enum(["lower", "higher", "neutral"])),
 });
 const OutcomeSummarySchema = z.object({
     attempts: z.number().int().nonnegative(),
@@ -24,16 +29,23 @@ const OutcomeSummarySchema = z.object({
 });
 const CaseSummarySchema = OutcomeSummarySchema.extend({
     canonicalCaseId: z.string(),
+    permutationId: z.exactOptional(z.string()),
     metrics: z.record(z.string(), MetricSummarySchema),
 });
+/** Repetitions pool per case and harness permutation, never across permutations. */
+export const suiteGroupKey = (entry) => entry.permutationId === undefined
+    ? entry.canonicalCaseId
+    : `${entry.canonicalCaseId} [${entry.permutationId}]`;
 export const SuiteSampleSchema = z.object({
     index: z.number().int().nonnegative(),
     caseId: z.string(),
     canonicalCaseId: z.string(),
+    permutationId: z.exactOptional(z.string()),
     exitReason: z.string().nullable(),
     passed: z.boolean().nullable(),
     identityHash: z.string().nullable(),
     score: z.exactOptional(z.number()),
+    scoreBasis: z.exactOptional(z.string().nullable()),
     metrics: z.record(z.string(), z.number()),
 });
 export const SuiteSummarySchema = z.object({
@@ -45,12 +57,14 @@ export const SuiteSummarySchema = z.object({
     cases: z.array(CaseSummarySchema),
     samples: z.array(SuiteSampleSchema),
 });
-const add = (samples, name, value) => {
+const add = (samples, name, value, descriptor) => {
     if (value === undefined || !Number.isFinite(value))
         return;
-    const values = samples.get(name) ?? [];
-    values.push(value);
-    samples.set(name, values);
+    const sample = samples.get(name) ?? { values: [], descriptor };
+    if (!sample.descriptor && descriptor)
+        sample.descriptor = descriptor;
+    sample.values.push(value);
+    samples.set(name, sample);
 };
 const cacheHitRate = (usage) => {
     const promptTokens = usage.input + usage.cacheRead + usage.cacheCreation;
@@ -62,7 +76,7 @@ const collectMetadata = (samples, metadata) => {
     if (!Number.isNaN(started) && !Number.isNaN(ended)) {
         add(samples, "durationMs", Math.max(0, ended - started));
     }
-    if (!isUsageEmpty(metadata.usage.agent)) {
+    if (hasAgentUsage(metadata)) {
         add(samples, "agent.input", metadata.usage.agent.input);
         add(samples, "agent.output", metadata.usage.agent.output);
         add(samples, "agent.cacheRead", metadata.usage.agent.cacheRead);
@@ -102,6 +116,36 @@ const collectMetadata = (samples, metadata) => {
     add(samples, "phase.agentMs", metadata.phaseDurationMs?.agent);
     add(samples, "phase.collectMs", metadata.phaseDurationMs?.collect);
     add(samples, "phase.gradingMs", metadata.phaseDurationMs?.grading);
+    add(samples, "timeToFirstActionMs", metadata.actionTiming?.timeToFirstActionMs);
+    add(samples, "timeToFirstMutationMs", metadata.actionTiming?.timeToFirstMutationMs);
+    add(samples, "timeToFirstDurableMutationMs", metadata.actionTiming?.timeToFirstDurableMutationMs);
+    const subagents = (metadata.agentActivity ?? []).filter((activity) => activity.agent.role === "subagent");
+    add(samples, "subagents.count", subagents.length, {
+        unit: "count",
+        label: "Subagents spawned",
+        better: "neutral",
+    });
+    add(samples, "subagents.events", subagents.reduce((sum, activity) => sum + activity.eventCount, 0), { unit: "count", label: "Subagent events", better: "neutral" });
+    add(samples, "subagents.toolCalls", subagents.reduce((sum, activity) => sum + activity.toolCalls.total, 0), { unit: "count", label: "Subagent tool calls", better: "neutral" });
+    add(samples, "subagents.toolCalls.failed", subagents.reduce((sum, activity) => sum + activity.toolCalls.failed, 0), { unit: "count", label: "Failed subagent tool calls", better: "lower" });
+    add(samples, "subagents.tokens", subagents.reduce((sum, activity) => sum + activity.usage.total, 0), { unit: "tokens", label: "Subagent tokens", better: "lower" });
+    add(samples, "subagents.actions", subagents.reduce((sum, activity) => sum + activity.actions.total, 0), { unit: "count", label: "Subagent actions", better: "neutral" });
+    add(samples, "subagents.mutations", subagents.reduce((sum, activity) => sum + activity.actions.mutations, 0), { unit: "count", label: "Subagent mutations", better: "neutral" });
+    add(samples, "subagents.totalDurationMs", subagents.reduce((sum, activity) => sum + activity.durationMs, 0), { unit: "milliseconds", label: "Total subagent time", better: "neutral" });
+    add(samples, "subagents.maxDurationMs", subagents.length > 0 ? Math.max(...subagents.map((activity) => activity.durationMs)) : 0, { unit: "milliseconds", label: "Longest subagent", better: "lower" });
+    const firstSubagentAction = subagents
+        .map((activity) => activity.actions.timeToFirstActionMs)
+        .filter((value) => value !== undefined)
+        .sort((a, b) => a - b)[0];
+    add(samples, "subagents.timeToFirstActionMs", firstSubagentAction, {
+        unit: "milliseconds",
+        label: "First subagent action",
+        better: "lower",
+    });
+    for (const [name, metric] of Object.entries(metadata.driverMetrics ?? {})) {
+        const { value, ...descriptor } = metric;
+        add(samples, `driver.${name}`, value, descriptor);
+    }
 };
 const percentile = (sorted, fraction) => {
     const index = Math.max(0, Math.ceil(sorted.length * fraction) - 1);
@@ -124,7 +168,10 @@ export const summarizeNumbers = (values) => {
 };
 const summarizeMetrics = (samples) => Object.fromEntries([...samples.entries()]
     .sort(([a], [b]) => a.localeCompare(b))
-    .map(([name, values]) => [name, summarizeNumbers(values)]));
+    .map(([name, { values, descriptor }]) => [
+    name,
+    MetricSummarySchema.parse({ ...summarizeNumbers(values), ...descriptor }),
+]));
 const summarizeOutcomes = (entries) => {
     const completed = entries.filter((entry) => entry.item.status === "completed").length;
     const passed = entries.filter((entry) => entry.result?.casePassed === true).length;
@@ -155,13 +202,14 @@ export const buildSuiteSummary = (suite, results, generatedAt = new Date()) => {
     }
     const byCase = new Map();
     for (const entry of entries) {
-        const group = byCase.get(entry.item.canonicalCaseId) ?? [];
+        const key = suiteGroupKey(entry.item);
+        const group = byCase.get(key) ?? [];
         group.push(entry);
-        byCase.set(entry.item.canonicalCaseId, group);
+        byCase.set(key, group);
     }
     const cases = [...byCase.entries()]
         .sort(([a], [b]) => a.localeCompare(b))
-        .map(([canonicalCaseId, caseEntries]) => {
+        .map(([, caseEntries]) => {
         const metrics = new Map();
         for (const entry of caseEntries) {
             const result = entry.result?.result;
@@ -170,8 +218,10 @@ export const buildSuiteSummary = (suite, results, generatedAt = new Date()) => {
             add(metrics, "score", result.grading.caseScore ?? undefined);
             collectMetadata(metrics, result.metadata);
         }
+        const first = caseEntries[0]?.item;
         return {
-            canonicalCaseId,
+            canonicalCaseId: first?.canonicalCaseId ?? "",
+            ...(first?.permutationId ? { permutationId: first.permutationId } : {}),
             ...summarizeOutcomes(caseEntries),
             metrics: summarizeMetrics(metrics),
         };
@@ -186,19 +236,22 @@ export const buildSuiteSummary = (suite, results, generatedAt = new Date()) => {
         samples: entries.map((entry, index) => {
             const samples = new Map();
             const result = entry.result?.result;
-            if (result)
+            if (result) {
+                add(samples, "score", result.grading.caseScore ?? undefined);
                 collectMetadata(samples, result.metadata);
+            }
             return {
                 index,
                 caseId: entry.item.caseId,
                 canonicalCaseId: entry.item.canonicalCaseId,
+                ...(entry.item.permutationId ? { permutationId: entry.item.permutationId } : {}),
                 exitReason: entry.result?.exitReason ?? null,
                 passed: entry.result?.casePassed ?? null,
                 identityHash: result ? runComparisonIdentity(result.metadata) : null,
                 ...(result?.grading.caseScore !== null && result?.grading.caseScore !== undefined
-                    ? { score: result.grading.caseScore }
+                    ? { score: result.grading.caseScore, scoreBasis: gradingScoreBasis(result.grading) }
                     : {}),
-                metrics: Object.fromEntries([...samples.entries()].map(([name, values]) => [name, values[0] ?? 0])),
+                metrics: Object.fromEntries([...samples.entries()].map(([name, sample]) => [name, sample.values[0] ?? 0])),
             };
         }),
     };