oh-my-knowledge 0.0.1 → 0.18.0

package/dist/src/eval-core/statistics.js

@@ -0,0 +1,210 @@
+/**
+ * Statistical functions for evaluation analysis.
+ *
+ * Assumptions:
+ * - All inputs are independent observations from normally distributed populations
+ * - Welch's t-test is used (does NOT assume equal variances)
+ * - Confidence intervals use the t-distribution for small samples (df ≤ 30)
+ *   and normal approximation (z = 1.96) for larger samples
+ *
+ * Dependencies: simple-statistics (for mean, sampleStandardDeviation, sampleVariance)
+ */
+import * as ss from 'simple-statistics';
+/**
+ * Arithmetic mean. Returns 0 for empty/null input.
+ */
+export function mean(arr) {
+    if (!arr || arr.length === 0)
+        return 0;
+    return ss.mean(arr);
+}
+/**
+ * Sample standard deviation (Bessel's correction: n-1 denominator).
+ * Returns 0 for arrays with fewer than 2 elements.
+ */
+export function stddev(arr) {
+    if (!arr || arr.length < 2)
+        return 0;
+    return ss.sampleStandardDeviation(arr);
+}
+// t-distribution critical values for two-tailed tests at p = 0.05, 0.01, 0.001.
+// Source: standard statistical tables. Used both for the significance gate and
+// for bucketed p-value reporting (see pValueCategory below).
+const T_CRITICAL_95 = {
+    1: 12.706, 2: 4.303, 3: 3.182, 4: 2.776, 5: 2.571,
+    6: 2.447, 7: 2.365, 8: 2.306, 9: 2.262, 10: 2.228,
+    11: 2.201, 12: 2.179, 13: 2.160, 14: 2.145, 15: 2.131,
+    16: 2.120, 17: 2.110, 18: 2.101, 19: 2.093, 20: 2.086,
+    21: 2.080, 22: 2.074, 23: 2.069, 24: 2.064, 25: 2.060,
+    26: 2.056, 27: 2.052, 28: 2.048, 29: 2.045, 30: 2.042,
+};
+const T_CRITICAL_99 = {
+    1: 63.657, 2: 9.925, 3: 5.841, 4: 4.604, 5: 4.032,
+    6: 3.707, 7: 3.499, 8: 3.355, 9: 3.250, 10: 3.169,
+    11: 3.106, 12: 3.055, 13: 3.012, 14: 2.977, 15: 2.947,
+    16: 2.921, 17: 2.898, 18: 2.878, 19: 2.861, 20: 2.845,
+    21: 2.831, 22: 2.819, 23: 2.807, 24: 2.797, 25: 2.787,
+    26: 2.779, 27: 2.771, 28: 2.763, 29: 2.756, 30: 2.750,
+};
+const T_CRITICAL_999 = {
+    1: 636.619, 2: 31.599, 3: 12.924, 4: 8.610, 5: 6.869,
+    6: 5.959, 7: 5.408, 8: 5.041, 9: 4.781, 10: 4.587,
+    11: 4.437, 12: 4.318, 13: 4.221, 14: 4.140, 15: 4.073,
+    16: 4.015, 17: 3.965, 18: 3.922, 19: 3.883, 20: 3.850,
+    21: 3.819, 22: 3.792, 23: 3.768, 24: 3.745, 25: 3.725,
+    26: 3.707, 27: 3.690, 28: 3.674, 29: 3.659, 30: 3.646,
+};
+/**
+ * Bucket a t-statistic into a p-value category by comparing |t| to standard
+ * critical-value tables at α = 0.001 / 0.01 / 0.05 (two-tailed).
+ *
+ * Returns one of four buckets instead of a continuous p value. This avoids
+ * implementing the regularized incomplete beta function for an exact CDF
+ * while still giving readers enough resolution to distinguish
+ * "barely significant" from "strongly significant".
+ *
+ * For df > 30 falls back to the normal-approximation critical values.
+ */
+export function pValueCategory(tStatistic, df) {
+    if (!Number.isFinite(tStatistic) || df <= 0)
+        return '≥0.05';
+    const absT = Math.abs(tStatistic);
+    const dfKey = Math.min(Math.max(1, Math.floor(df)), 30);
+    const c999 = df > 30 ? 3.291 : (T_CRITICAL_999[dfKey] ?? 3.291);
+    const c99 = df > 30 ? 2.576 : (T_CRITICAL_99[dfKey] ?? 2.576);
+    const c95 = df > 30 ? 1.96 : (T_CRITICAL_95[dfKey] ?? 1.96);
+    if (absT >= c999)
+        return '<0.001';
+    if (absT >= c99)
+        return '<0.01';
+    if (absT >= c95)
+        return '<0.05';
+    return '≥0.05';
+}
+/**
+ * Get t-distribution critical value for given degrees of freedom (95% CI, two-tailed).
+ * Uses lookup table for df ≤ 30, normal approximation (1.96) for df > 30.
+ */
+function tCritical(df) {
+    if (df <= 0)
+        return Infinity;
+    if (df <= 30)
+        return T_CRITICAL_95[df] || T_CRITICAL_95[30];
+    return 1.96;
+}
+/**
+ * Compute 95% confidence interval for the population mean.
+ *
+ * Method: t-interval (appropriate for small samples from normal populations)
+ * Formula: mean ± t(α/2, n-1) × (s / √n)
+ */
+export function confidenceInterval(arr) {
+    const m = mean(arr);
+    const sd = stddev(arr);
+    if (!arr || arr.length < 2) {
+        return { mean: m, lower: m, upper: m, stddev: sd };
+    }
+    const n = arr.length;
+    const se = sd / Math.sqrt(n); // Standard error of the mean
+    const df = n - 1; // Degrees of freedom
+    const tc = tCritical(df); // Critical value
+    const margin = tc * se; // Margin of error
+    return {
+        mean: Number(m.toFixed(4)),
+        lower: Number((m - margin).toFixed(4)),
+        upper: Number((m + margin).toFixed(4)),
+        stddev: Number(sd.toFixed(4)),
+    };
+}
+/**
+ * Effect size for two independent samples.
+ *
+ * Cohen's d:   (mean_a - mean_b) / pooled_stddev
+ * Hedges' g:   J * d, where J = 1 - 3 / (4 * (n1 + n2) - 9)
+ *
+ * Hedges' g corrects Cohen's d's small-sample bias (Hedges 1981) and is
+ * preferred when n1 + n2 < 20. For n1 + n2 >= 20 the correction is < 5%
+ * and Cohen's d is the conventional choice.
+ *
+ * `primary` indicates which to emphasize in UI based on total sample size.
+ * `magnitude` uses standard thresholds (|effect| 0.2 / 0.5 / 0.8) applied
+ * to the primary metric.
+ *
+ * Returns effect 0 with primary 'none' for insufficient data or zero variance.
+ */
+export function effectSize(a, b) {
+    const n1 = a?.length ?? 0;
+    const n2 = b?.length ?? 0;
+    if (n1 < 2 || n2 < 2) {
+        return { cohensD: 0, hedgesG: 0, primary: 'none', magnitude: 'none', pooledStddev: 0, n1, n2 };
+    }
+    const mA = ss.mean(a);
+    const mB = ss.mean(b);
+    const varA = ss.sampleVariance(a);
+    const varB = ss.sampleVariance(b);
+    // Pooled standard deviation (assumes similar variances; acceptable for
+    // effect-size reporting even when Welch's t-test is used for significance)
+    const pooledVar = ((n1 - 1) * varA + (n2 - 1) * varB) / (n1 + n2 - 2);
+    const pooledStddev = Math.sqrt(pooledVar);
+    if (pooledStddev === 0) {
+        return { cohensD: 0, hedgesG: 0, primary: 'none', magnitude: 'none', pooledStddev: 0, n1, n2 };
+    }
+    const d = (mA - mB) / pooledStddev;
+    // Hedges' correction factor J. Denominator guard: 4*(n1+n2)-9 > 0 iff n1+n2 > 2.25,
+    // which is already satisfied by the n1>=2, n2>=2 check above.
+    const J = 1 - 3 / (4 * (n1 + n2) - 9);
+    const g = J * d;
+    const totalN = n1 + n2;
+    const primary = totalN < 20 ? 'g' : 'd';
+    const ref = primary === 'g' ? g : d;
+    const abs = Math.abs(ref);
+    const magnitude = abs < 0.2 ? 'negligible' :
+        abs < 0.5 ? 'small' :
+            abs < 0.8 ? 'medium' : 'large';
+    return {
+        cohensD: Number(d.toFixed(4)),
+        hedgesG: Number(g.toFixed(4)),
+        primary,
+        magnitude,
+        pooledStddev: Number(pooledStddev.toFixed(4)),
+        n1,
+        n2,
+    };
+}
+export function tTest(a, b) {
+    // Minimum sample size check
+    if (!a || !b || a.length < 2 || b.length < 2) {
+        return { tStatistic: 0, df: 0, significant: false };
+    }
+    const mA = ss.mean(a);
+    const mB = ss.mean(b);
+    const varA = ss.sampleVariance(a);
+    const varB = ss.sampleVariance(b);
+    // Estimated variances of sample means
+    const seA = varA / a.length;
+    const seB = varB / b.length;
+    const seDiff = Math.sqrt(seA + seB);
+    // Both groups have zero variance — means are exact, can't compute t
+    if (seDiff === 0) {
+        return {
+            tStatistic: 0,
+            df: a.length + b.length - 2,
+            significant: mA !== mB, // Different constants are trivially significant
+        };
+    }
+    // t-statistic
+    const tStat = (mA - mB) / seDiff;
+    // Welch-Satterthwaite degrees of freedom
+    const dfNum = (seA + seB) ** 2;
+    const dfDen = (seA ** 2 / (a.length - 1)) + (seB ** 2 / (b.length - 1));
+    const df = Math.max(1, Math.floor(dfNum / dfDen));
+    // Significance at α = 0.05 (two-tailed)
+    const tc = tCritical(df);
+    const significant = Math.abs(tStat) > tc;
+    return {
+        tStatistic: Number(tStat.toFixed(4)),
+        df,
+        significant,
+    };
+}
+//# sourceMappingURL=statistics.js.map

package/dist/src/eval-core/statistics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"statistics.js","sourceRoot":"","sources":["../../../src/eval-core/statistics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAExC;;GAEG;AACH,MAAM,UAAU,IAAI,CAAC,GAAa;IAChC,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACvC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACtB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,MAAM,CAAC,GAAa;IAClC,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACrC,OAAO,EAAE,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC;AACzC,CAAC;AAED,gFAAgF;AAChF,+EAA+E;AAC/E,6DAA6D;AAC7D,MAAM,aAAa,GAA2B;IAC5C,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK;IACjD,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK;IACjD,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK;IACrD,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK;IACrD,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK;IACrD,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK;CACtD,CAAC;AAEF,MAAM,aAAa,GAA2B;IAC5C,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK;IACjD,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK;IACjD,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK;IACrD,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK;IACrD,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK;IACrD,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK;CACtD,CAAC;AAEF,MAAM,cAAc,GAA2B;IAC7C,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK;IACpD,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK;IACjD,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK;IACrD,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK;IACrD,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK;IACrD,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK;CACtD,CAAC;AAIF;;;;;;;;;;GAUG;AACH,MAAM,UAAU,cAAc,CAAC,UAAkB,EAAE,EAAU;IAC3D,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC;QAAE,OAAO,OAAO,CAAC;IAC5D,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAClC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACxD,MAAM,IAAI,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC;IAChE,MAAM,GAAG,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC;IAC9D,MAAM,GAAG,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC;IAC5D,IAAI,IAAI,IAAI,IAAI;QAAE,OAAO,QAAQ,CAAC;IAClC,IAAI,IAAI,IAAI,GAAG;QAAE,OAAO,OAAO,CAAC;IAChC,IAAI,IAAI,IAAI,GAAG;QAAE,OAAO,OAAO,CAAC;IAChC,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,SAAS,SAAS,CAAC,EAAU;IAC3B,IAAI,EAAE,IAAI,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC7B,IAAI,EAAE,IAAI,EAAE;QAAE,OAAO,aAAa,CAAC,EAAE,CAAC,IAAI,aAAa,CAAC,EAAE,CAAC,CAAC;IAC5D,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAa;IAC9C,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;IACpB,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IAEvB,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IACrD,CAAC;IAED,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC;IACrB,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAQ,6BAA6B;IAClE,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAqB,qBAAqB;IAC3D,MAAM,EAAE,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC,CAAa,iBAAiB;IACvD,MAAM,MAAM,GAAG,EAAE,GAAG,EAAE,CAAC,CAAe,kBAAkB;IAExD,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC1B,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACtC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;KAC9B,CAAC;AACJ,CAAC;AAqBD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,UAAU,CAAC,CAAW,EAAE,CAAW;IACjD,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;IAC1B,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;IAE1B,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC;IACjG,CAAC;IAED,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtB,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtB,MAAM,IAAI,GAAG,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAElC,uEAAuE;IACvE,2EAA2E;IAC3E,MAAM,SAAS,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IACtE,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAE1C,IAAI,YAAY,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC;IACjG,CAAC;IAED,MAAM,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,YAAY,CAAC;IAEnC,oFAAoF;IACpF,8DAA8D;IAC9D,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACtC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAEhB,MAAM,MAAM,GAAG,EAAE,GAAG,EAAE,CAAC;IACvB,MAAM,OAAO,GAAc,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IACnD,MAAM,GAAG,GAAG,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC1B,MAAM,SAAS,GACb,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QAC1B,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACrB,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC;IAEjC,OAAO;QACL,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,OAAO;QACP,SAAS;QACT,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7C,EAAE;QACF,EAAE;KACH,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,CAAW,EAAE,CAAW;IAC5C,4BAA4B;IAC5B,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7C,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;IACtD,CAAC;IAED,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtB,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtB,MAAM,IAAI,GAAG,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAElC,sCAAsC;IACtC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5B,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;IAEpC,oEAAoE;IACpE,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,OAAO;YACL,UAAU,EAAE,CAAC;YACb,EAAE,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC;YAC3B,WAAW,EAAE,EAAE,KAAK,EAAE,EAAE,gDAAgD;SACzE,CAAC;IACJ,CAAC;IAED,cAAc;IACd,MAAM,KAAK,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,MAAM,CAAC;IAEjC,yCAAyC;IACzC,MAAM,KAAK,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IACxE,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC;IAElD,wCAAwC;IACxC,MAAM,EAAE,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;IACzB,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;IAEzC,OAAO;QACL,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACpC,EAAE;QACF,WAAW;KACZ,CAAC;AACJ,CAAC"}

package/dist/src/eval-core/task-planner.d.ts

@@ -0,0 +1,4 @@
+import type { Artifact, Sample, Task } from '../types.js';
+export declare function buildTasks(samples: Sample[], variants: string[], skills: Record<string, string | null>): Task[];
+export declare function buildTasksFromArtifacts(samples: Sample[], artifacts: Artifact[]): Task[];
+//# sourceMappingURL=task-planner.d.ts.map

package/dist/src/eval-core/task-planner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"task-planner.d.ts","sourceRoot":"","sources":["../../../src/eval-core/task-planner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAE1D,wBAAgB,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,GAAG,IAAI,EAAE,CAQ/G;AAED,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,CAyBxF"}

package/dist/src/eval-core/task-planner.js

@@ -0,0 +1,33 @@
+export function buildTasks(samples, variants, skills) {
+    const artifacts = variants.map((variant) => ({
+        name: variant,
+        kind: variant === 'baseline' ? 'baseline' : 'skill',
+        source: variant === 'baseline' ? 'baseline' : 'custom',
+        content: skills[variant] || null,
+    }));
+    return buildTasksFromArtifacts(samples, artifacts);
+}
+export function buildTasksFromArtifacts(samples, artifacts) {
+    const tasks = [];
+    for (const sample of samples) {
+        for (const artifact of artifacts) {
+            const userPrompt = sample.context
+                ? `${sample.prompt}\n\n\`\`\`\n${sample.context}\n\`\`\``
+                : sample.prompt;
+            tasks.push({
+                sample_id: sample.sample_id,
+                variant: artifact.name,
+                artifact,
+                prompt: userPrompt,
+                rubric: sample.rubric || null,
+                assertions: sample.assertions || null,
+                dimensions: sample.dimensions || null,
+                artifactContent: artifact.content,
+                cwd: artifact.cwd || sample.cwd || null,
+                _sample: sample,
+            });
+        }
+    }
+    return tasks;
+}
+//# sourceMappingURL=task-planner.js.map

package/dist/src/eval-core/task-planner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"task-planner.js","sourceRoot":"","sources":["../../../src/eval-core/task-planner.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,UAAU,CAAC,OAAiB,EAAE,QAAkB,EAAE,MAAqC;IACrG,MAAM,SAAS,GAAe,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACvD,IAAI,EAAE,OAAO;QACb,IAAI,EAAE,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO;QACnD,MAAM,EAAE,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ;QACtD,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI;KACjC,CAAC,CAAC,CAAC;IACJ,OAAO,uBAAuB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;AACrD,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,OAAiB,EAAE,SAAqB;IAC9E,MAAM,KAAK,GAAW,EAAE,CAAC;IAEzB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO;gBAC/B,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,eAAe,MAAM,CAAC,OAAO,UAAU;gBACzD,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;YAElB,KAAK,CAAC,IAAI,CAAC;gBACT,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,OAAO,EAAE,QAAQ,CAAC,IAAI;gBACtB,QAAQ;gBACR,MAAM,EAAE,UAAU;gBAClB,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,IAAI;gBAC7B,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,IAAI;gBACrC,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,IAAI;gBACrC,eAAe,EAAE,QAAQ,CAAC,OAAO;gBACjC,GAAG,EAAE,QAAQ,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,IAAI,IAAI;gBACvC,OAAO,EAAE,MAAM;aAChB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/src/eval-workflows/each-evaluation-workflow.d.ts

@@ -0,0 +1,133 @@
+import type { Artifact, JobStore, ProgressCallback, Report, VariantResult, VariantSummary } from '../types.js';
+interface RunSingleEvaluationOptions {
+    samplesPath: string;
+    skillDir: string;
+    artifacts: Artifact[];
+    model: string;
+    judgeModel: string;
+    outputDir: null;
+    noJudge: boolean;
+    concurrency: number;
+    timeoutMs?: number;
+    executorName: string;
+    judgeExecutorName?: string;
+    jobStore: null;
+    persistJob: false;
+    onProgress: ProgressCallback | null;
+    skipPreflight: boolean;
+    mcpConfig?: string;
+    verbose: boolean;
+}
+export interface EachSkillResult {
+    name: string;
+    artifactHash: string;
+    samplesPath: string;
+    sampleCount: number;
+    summary: Record<string, VariantSummary>;
+    results: Array<{
+        sample_id: string;
+        variants: {
+            baseline: VariantResult;
+            skill: VariantResult;
+        };
+    }>;
+}
+export declare function buildEachReport({ skillDir, skillEntries, skillResults, model, judgeModel, noJudge, executorName, judgeExecutorName, project, owner, tags, dryRun, concurrency, timeoutMs, totalCostUSD, }: {
+    skillDir: string;
+    skillEntries: Array<{
+        name: string;
+        skillPath: string;
+        samplesPath: string;
+    }>;
+    skillResults: EachSkillResult[];
+    model: string;
+    judgeModel: string;
+    noJudge: boolean;
+    executorName: string;
+    judgeExecutorName?: string;
+    project?: string;
+    owner?: string;
+    tags?: string[];
+    dryRun: boolean;
+    concurrency: number;
+    timeoutMs?: number;
+    totalCostUSD: number;
+}): {
+    report: {
+        id: string;
+        each: true;
+        meta: {
+            variants: string[];
+            model: string;
+            judgeModel: string | null;
+            executor: string;
+            sampleCount: number;
+            taskCount: number;
+            totalCostUSD: number;
+            timestamp: string;
+            cliVersion: string;
+            nodeVersion: string;
+            artifactHashes: {
+                [k: string]: string;
+            };
+            request: import("../types.js").EvaluationRequest;
+            run: import("../types.js").EvaluationRun;
+            job: import("../types.js").EvaluationJob;
+        };
+        summary: {};
+        results: never[];
+        overview: {
+            totalArtifacts: number;
+            totalSamples: number;
+            totalCostUSD: number;
+            artifacts: {
+                name: string;
+                baselineScore: number | null;
+                artifactScore: number | null;
+                improvement: string;
+            }[];
+        };
+        artifacts: EachSkillResult[];
+    };
+    job: import("../types.js").EvaluationJob;
+};
+export declare function executeEachEvaluationRuns({ skillDir, skillEntries, model, judgeModel, outputDir, project, owner, tags, noJudge, concurrency, timeoutMs, executorName, judgeExecutorName, jobStore, persistJob, onProgress, onSkillProgress, skipPreflight, mcpConfig, verbose, runSingleEvaluation, }: {
+    skillDir: string;
+    skillEntries: Array<{
+        name: string;
+        skillPath: string;
+        samplesPath: string;
+    }>;
+    model: string;
+    judgeModel: string;
+    outputDir?: string | null;
+    project?: string;
+    owner?: string;
+    tags?: string[];
+    noJudge?: boolean;
+    concurrency?: number;
+    timeoutMs?: number;
+    executorName: string;
+    judgeExecutorName?: string;
+    jobStore?: JobStore | null;
+    persistJob?: boolean;
+    onProgress?: ProgressCallback | null;
+    onSkillProgress?: ((info: {
+        phase: string;
+        skill: string;
+        current: number;
+        total: number;
+    }) => void) | null;
+    skipPreflight?: boolean;
+    mcpConfig?: string;
+    verbose?: boolean;
+    runSingleEvaluation: (options: RunSingleEvaluationOptions) => Promise<{
+        report: Report;
+        filePath: string | null;
+    }>;
+}): Promise<{
+    report: Report;
+    filePath: string | null;
+}>;
+export {};
+//# sourceMappingURL=each-evaluation-workflow.d.ts.map

package/dist/src/eval-workflows/each-evaluation-workflow.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"each-evaluation-workflow.d.ts","sourceRoot":"","sources":["../../../src/eval-workflows/each-evaluation-workflow.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE/G,UAAU,0BAA0B;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,IAAI,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,IAAI,CAAC;IACf,UAAU,EAAE,KAAK,CAAC;IAClB,UAAU,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACpC,aAAa,EAAE,OAAO,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IACxC,OAAO,EAAE,KAAK,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,EAAE;YACR,QAAQ,EAAE,aAAa,CAAC;YACxB,KAAK,EAAE,aAAa,CAAC;SACtB,CAAC;KACH,CAAC,CAAC;CACJ;AAsBD,wBAAgB,eAAe,CAAC,EAC9B,QAAQ,EACR,YAAY,EACZ,YAAY,EACZ,KAAK,EACL,UAAU,EACV,OAAO,EACP,YAAY,EACZ,iBAAiB,EACjB,OAAO,EACP,KAAK,EACL,IAAI,EACJ,MAAM,EACN,WAAW,EACX,SAAS,EACT,YAAY,GACb,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9E,YAAY,EAAE,eAAe,EAAE,CAAC;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAsEA;AAED,wBAAsB,yBAAyB,CAAC,EAC9C,QAAQ,EACR,YAAY,EACZ,KAAK,EACL,UAAU,EACV,SAA8B,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,OAAe,EACf,WAAe,EACf,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,QAAe,EACf,UAAiB,EACjB,UAAiB,EACjB,eAAsB,EACtB,aAAqB,EACrB,SAAS,EACT,OAAe,EACf,mBAAmB,GACpB,EAAE;IACD,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9E,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC;IAC3B,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACrC,eAAe,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IAC5G,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,mBAAmB,EAAE,CAAC,OAAO,EAAE,0BAA0B,KAAK,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC,CAAC;CACpH,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CA+EvD"}

package/dist/src/eval-workflows/each-evaluation-workflow.js

@@ -0,0 +1,169 @@
+import { resolve } from 'node:path';
+import { DEFAULT_OUTPUT_DIR, generateRunId, persistReport } from '../eval-core/evaluation-reporting.js';
+import { buildEvaluationRequest, createEvaluationRun, createSucceededJob, finalizeEvaluationRun } from '../eval-core/evaluation-job.js';
+import { createFileJobStore, DEFAULT_JOBS_DIR } from '../server/job-store.js';
+import { resolveArtifacts } from '../inputs/skill-loader.js';
+function buildEachOverview(skillResults, totalCostUSD) {
+    return {
+        totalArtifacts: skillResults.length,
+        totalSamples: skillResults.reduce((sum, skill) => sum + skill.sampleCount, 0),
+        totalCostUSD: Number(totalCostUSD.toFixed(6)),
+        artifacts: skillResults.map((skill) => {
+            const baselineSummary = skill.summary.baseline;
+            const artifactSummary = skill.summary.skill;
+            const baselineScore = baselineSummary?.avgCompositeScore ?? baselineSummary?.avgLlmScore ?? null;
+            const artifactScore = artifactSummary?.avgCompositeScore ?? artifactSummary?.avgLlmScore ?? null;
+            if (typeof baselineScore !== 'number' || typeof artifactScore !== 'number' || baselineScore <= 0) {
+                return { name: skill.name, baselineScore, artifactScore, improvement: '-' };
+            }
+            const delta = ((artifactScore - baselineScore) / baselineScore * 100).toFixed(0);
+            const improvement = artifactScore >= baselineScore ? `+${delta}%` : `${delta}%`;
+            return { name: skill.name, baselineScore, artifactScore, improvement };
+        }),
+    };
+}
+export function buildEachReport({ skillDir, skillEntries, skillResults, model, judgeModel, noJudge, executorName, judgeExecutorName, project, owner, tags, dryRun, concurrency, timeoutMs, totalCostUSD, }) {
+    const runId = generateRunId(['each']);
+    const request = buildEvaluationRequest({
+        samplesPath: '',
+        skillDir,
+        artifacts: skillEntries.map((entry) => ({
+            name: entry.name,
+            kind: 'skill',
+            source: 'file-path',
+            content: null,
+            locator: entry.skillPath,
+        })),
+        model,
+        judgeModel: noJudge ? null : judgeModel,
+        executor: executorName,
+        judgeExecutor: judgeExecutorName || executorName,
+        noJudge,
+        concurrency,
+        timeoutMs,
+        noCache: false,
+        dryRun,
+        blind: false,
+        project,
+        owner,
+        tags,
+    });
+    const createdAt = new Date().toISOString();
+    const { run: initialRun, startedAt } = createEvaluationRun(runId, createdAt);
+    const finishedAt = new Date().toISOString();
+    const run = finalizeEvaluationRun(initialRun, finishedAt);
+    const job = createSucceededJob({
+        jobId: `job-${runId}`,
+        runId,
+        reportId: runId,
+        request,
+        createdAt,
+        startedAt,
+        finishedAt,
+    });
+    const totalSampleCount = skillResults.reduce((sum, skill) => sum + skill.sampleCount, 0);
+    return {
+        report: {
+            id: runId,
+            each: true,
+            meta: {
+                variants: skillEntries.map((entry) => entry.name),
+                model,
+                judgeModel: noJudge ? null : judgeModel,
+                executor: executorName,
+                sampleCount: totalSampleCount,
+                taskCount: totalSampleCount * 2,
+                totalCostUSD: Number(totalCostUSD.toFixed(6)),
+                timestamp: new Date().toISOString(),
+                cliVersion: '',
+                nodeVersion: process.version,
+                artifactHashes: Object.fromEntries(skillResults.map((skill) => [skill.name, skill.artifactHash || 'no-skill'])),
+                request,
+                run,
+                job,
+            },
+            summary: {},
+            results: [],
+            overview: buildEachOverview(skillResults, totalCostUSD),
+            artifacts: skillResults,
+        },
+        job,
+    };
+}
+export async function executeEachEvaluationRuns({ skillDir, skillEntries, model, judgeModel, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, concurrency = 1, timeoutMs, executorName, judgeExecutorName, jobStore = null, persistJob = true, onProgress = null, onSkillProgress = null, skipPreflight = false, mcpConfig, verbose = false, runSingleEvaluation, }) {
+    const skillResults = [];
+    let totalCostUSD = 0;
+    for (let i = 0; i < skillEntries.length; i++) {
+        const entry = skillEntries[i];
+        onSkillProgress?.({ phase: 'start', skill: entry.name, current: i + 1, total: skillEntries.length });
+        // each mode 的实验结构固定为"baseline (control) vs skill (treatment)"。
+        // 显式在 artifact 上填 experimentRole，下游 buildVariantConfig 直接读取。
+        const skillArtifacts = resolveArtifacts(resolve(skillDir), ['baseline', entry.skillPath]).map((artifact) => {
+            if (artifact.name === entry.skillPath) {
+                return { ...artifact, name: 'skill', experimentRole: 'treatment' };
+            }
+            return { ...artifact, experimentRole: 'control' };
+        });
+        const { report } = await runSingleEvaluation({
+            samplesPath: entry.samplesPath,
+            skillDir,
+            artifacts: skillArtifacts,
+            model,
+            judgeModel,
+            outputDir: null,
+            noJudge,
+            concurrency,
+            timeoutMs,
+            executorName,
+            judgeExecutorName,
+            jobStore: null,
+            persistJob: false,
+            onProgress,
+            skipPreflight: skipPreflight || i > 0,
+            mcpConfig,
+            verbose,
+        });
+        skillResults.push({
+            name: entry.name,
+            artifactHash: report.meta.artifactHashes?.skill || '',
+            samplesPath: entry.samplesPath,
+            sampleCount: report.meta.sampleCount,
+            summary: {
+                baseline: report.summary.baseline || {},
+                skill: (report.summary.skill || {}),
+            },
+            results: report.results.map((result) => ({
+                sample_id: result.sample_id,
+                variants: {
+                    baseline: result.variants.baseline || result.variants['baseline'],
+                    skill: result.variants.skill,
+                },
+            })),
+        });
+        totalCostUSD += report.meta.totalCostUSD;
+        onSkillProgress?.({ phase: 'done', skill: entry.name, current: i + 1, total: skillEntries.length });
+    }
+    const { report: combinedReport, job } = buildEachReport({
+        skillDir,
+        skillEntries,
+        skillResults,
+        model,
+        judgeModel,
+        noJudge,
+        executorName,
+        judgeExecutorName,
+        project,
+        owner,
+        tags,
+        dryRun: false,
+        concurrency,
+        timeoutMs,
+        totalCostUSD,
+    });
+    const filePath = persistReport(combinedReport, outputDir);
+    const resolvedJobStore = persistJob ? (jobStore ?? createFileJobStore(DEFAULT_JOBS_DIR)) : null;
+    if (resolvedJobStore)
+        await resolvedJobStore.save(job.jobId, job);
+    return { report: combinedReport, filePath };
+}
+//# sourceMappingURL=each-evaluation-workflow.js.map

package/dist/src/eval-workflows/each-evaluation-workflow.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"each-evaluation-workflow.js","sourceRoot":"","sources":["../../../src/eval-workflows/each-evaluation-workflow.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AACxG,OAAO,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AACxI,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC9E,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAsC7D,SAAS,iBAAiB,CAAC,YAA+B,EAAE,YAAoB;IAC9E,OAAO;QACL,cAAc,EAAE,YAAY,CAAC,MAAM;QACnC,YAAY,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QAC7E,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7C,SAAS,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YACpC,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;YAC/C,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAC5C,MAAM,aAAa,GAAG,eAAe,EAAE,iBAAiB,IAAI,eAAe,EAAE,WAAW,IAAI,IAAI,CAAC;YACjG,MAAM,aAAa,GAAG,eAAe,EAAE,iBAAiB,IAAI,eAAe,EAAE,WAAW,IAAI,IAAI,CAAC;YACjG,IAAI,OAAO,aAAa,KAAK,QAAQ,IAAI,OAAO,aAAa,KAAK,QAAQ,IAAI,aAAa,IAAI,CAAC,EAAE,CAAC;gBACjG,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC;YAC9E,CAAC;YACD,MAAM,KAAK,GAAG,CAAC,CAAC,aAAa,GAAG,aAAa,CAAC,GAAG,aAAa,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACjF,MAAM,WAAW,GAAG,aAAa,IAAI,aAAa,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC;YAChF,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,CAAC;QACzE,CAAC,CAAC;KACH,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAC9B,QAAQ,EACR,YAAY,EACZ,YAAY,EACZ,KAAK,EACL,UAAU,EACV,OAAO,EACP,YAAY,EACZ,iBAAiB,EACjB,OAAO,EACP,KAAK,EACL,IAAI,EACJ,MAAM,EACN,WAAW,EACX,SAAS,EACT,YAAY,GAiBb;IACC,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,sBAAsB,CAAC;QACrC,WAAW,EAAE,EAAE;QACf,QAAQ;QACR,SAAS,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACtC,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,IAAI,EAAE,OAAO;YACb,MAAM,EAAE,WAAW;YACnB,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,KAAK,CAAC,SAAS;SACzB,CAAC,CAAC;QACH,KAAK;QACL,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU;QACvC,QAAQ,EAAE,YAAY;QACtB,aAAa,EAAE,iBAAiB,IAAI,YAAY;QAChD,OAAO;QACP,WAAW;QACX,SAAS;QACT,OAAO,EAAE,KAAK;QACd,MAAM;QACN,KAAK,EAAE,KAAK;QACZ,OAAO;QACP,KAAK;QACL,IAAI;KACL,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,mBAAmB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC7E,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC5C,MAAM,GAAG,GAAG,qBAAqB,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IAC1D,MAAM,GAAG,GAAG,kBAAkB,CAAC;QAC7B,KAAK,EAAE,OAAO,KAAK,EAAE;QACrB,KAAK;QACL,QAAQ,EAAE,KAAK;QACf,OAAO;QACP,SAAS;QACT,SAAS;QACT,UAAU;KACX,CAAC,CAAC;IACH,MAAM,gBAAgB,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAEzF,OAAO;QACL,MAAM,EAAE;YACN,EAAE,EAAE,KAAK;YACT,IAAI,EAAE,IAAI;YACV,IAAI,EAAE;gBACJ,QAAQ,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;gBACjD,KAAK;gBACL,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU;gBACvC,QAAQ,EAAE,YAAY;gBACtB,WAAW,EAAE,gBAAgB;gBAC7B,SAAS,EAAE,gBAAgB,GAAG,CAAC;gBAC/B,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAC7C,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,UAAU,EAAE,EAAE;gBACd,WAAW,EAAE,OAAO,CAAC,OAAO;gBAC5B,cAAc,EAAE,MAAM,CAAC,WAAW,CAChC,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,IAAI,UAAU,CAAC,CAAC,CAC5E;gBACD,OAAO;gBACP,GAAG;gBACH,GAAG;aACJ;YACD,OAAO,EAAE,EAAE;YACX,OAAO,EAAE,EAAE;YACX,QAAQ,EAAE,iBAAiB,CAAC,YAAY,EAAE,YAAY,CAAC;YACvD,SAAS,EAAE,YAAY;SACP;QAClB,GAAG;KACJ,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,EAC9C,QAAQ,EACR,YAAY,EACZ,KAAK,EACL,UAAU,EACV,SAAS,GAAG,kBAAkB,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,OAAO,GAAG,KAAK,EACf,WAAW,GAAG,CAAC,EACf,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,QAAQ,GAAG,IAAI,EACf,UAAU,GAAG,IAAI,EACjB,UAAU,GAAG,IAAI,EACjB,eAAe,GAAG,IAAI,EACtB,aAAa,GAAG,KAAK,EACrB,SAAS,EACT,OAAO,GAAG,KAAK,EACf,mBAAmB,GAuBpB;IACC,MAAM,YAAY,GAAsB,EAAE,CAAC;IAC3C,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;QAC9B,eAAe,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC;QAErG,+DAA+D;QAC/D,6DAA6D;QAC7D,MAAM,cAAc,GAAG,gBAAgB,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;YACzG,IAAI,QAAQ,CAAC,IAAI,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC;gBACtC,OAAO,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,cAAc,EAAE,WAAoB,EAAE,CAAC;YAC9E,CAAC;YACD,OAAO,EAAE,GAAG,QAAQ,EAAE,cAAc,EAAE,SAAkB,EAAE,CAAC;QAC7D,CAAC,CAAC,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,mBAAmB,CAAC;YAC3C,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,QAAQ;YACR,SAAS,EAAE,cAAc;YACzB,KAAK;YACL,UAAU;YACV,SAAS,EAAE,IAAI;YACf,OAAO;YACP,WAAW;YACX,SAAS;YACT,YAAY;YACZ,iBAAiB;YACjB,QAAQ,EAAE,IAAI;YACd,UAAU,EAAE,KAAK;YACjB,UAAU;YACV,aAAa,EAAE,aAAa,IAAI,CAAC,GAAG,CAAC;YACrC,SAAS;YACT,OAAO;SACR,CAAC,CAAC;QAEH,YAAY,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,cAAc,EAAE,KAAK,IAAI,EAAE;YACrD,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW;YACpC,OAAO,EAAE;gBACP,QAAQ,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAK,EAAqB;gBAC3D,KAAK,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,CAAmB;aACtD;YACD,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACvC,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,QAAQ,EAAE;oBACR,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC;oBACjE,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,KAAK;iBAC7B;aACF,CAAC,CAAC;SACJ,CAAC,CAAC;QAEH,YAAY,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC;QACzC,eAAe,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC;IACtG,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,GAAG,EAAE,GAAG,eAAe,CAAC;QACtD,QAAQ;QACR,YAAY;QACZ,YAAY;QACZ,KAAK;QACL,UAAU;QACV,OAAO;QACP,YAAY;QACZ,iBAAiB;QACjB,OAAO;QACP,KAAK;QACL,IAAI;QACJ,MAAM,EAAE,KAAK;QACb,WAAW;QACX,SAAS;QACT,YAAY;KACb,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,aAAa,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;IAC1D,MAAM,gBAAgB,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,kBAAkB,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAChG,IAAI,gBAAgB;QAAE,MAAM,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAClE,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC;AAC9C,CAAC"}

package/dist/src/eval-workflows/evaluation-pipeline.d.ts

@@ -0,0 +1,38 @@
+import type { DependencyRequirements } from '../eval-core/dependency-checker.js';
+import type { Artifact, ExecutorFn, JobStore, ProgressCallback, Report, Sample, Task, VariantResult } from '../types.js';
+export interface EvaluationPipelineOptions {
+    samplesPath: string;
+    skillDir: string;
+    samples: Sample[];
+    tasks: Task[];
+    artifacts: Artifact[];
+    model: string;
+    judgeModel: string;
+    noJudge: boolean;
+    executorName: string;
+    judgeExecutorName: string;
+    executor: ExecutorFn;
+    judgeExecutor: ExecutorFn;
+    outputDir?: string | null;
+    project?: string;
+    owner?: string;
+    tags?: string[];
+    blind?: boolean;
+    concurrency?: number;
+    timeoutMs?: number;
+    noCache?: boolean;
+    jobStore?: JobStore | null;
+    persistJob?: boolean;
+    onProgress?: ProgressCallback | null;
+    skipPreflight?: boolean;
+    verbose?: boolean;
+    retry?: number;
+    existingResults?: Record<string, Record<string, VariantResult>>;
+    requires?: DependencyRequirements;
+    layeredStats?: boolean;
+}
+export declare function executeEvaluationPipeline({ samplesPath, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir, project, owner, tags, blind, concurrency, timeoutMs, noCache, jobStore, persistJob, onProgress, skipPreflight, verbose, retry, existingResults, requires, layeredStats, }: EvaluationPipelineOptions): Promise<{
+    report: Report;
+    filePath: string | null;
+}>;
+//# sourceMappingURL=evaluation-pipeline.d.ts.map

package/dist/src/eval-workflows/evaluation-pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluation-pipeline.d.ts","sourceRoot":"","sources":["../../../src/eval-workflows/evaluation-pipeline.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,oCAAoC,CAAC;AAgBjF,OAAO,KAAK,EACV,QAAQ,EAIR,UAAU,EACV,QAAQ,EACR,gBAAgB,EAChB,MAAM,EACN,MAAM,EACN,IAAI,EACJ,aAAa,EACd,MAAM,aAAa,CAAC;AAuLrB,MAAM,WAAW,yBAAyB;IACxC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,QAAQ,EAAE,UAAU,CAAC;IACrB,aAAa,EAAE,UAAU,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC;IAC3B,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACrC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;IAChE,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED,wBAAsB,yBAAyB,CAAC,EAC9C,WAAW,EACX,QAAQ,EACR,OAAO,EACP,KAAK,EACL,SAAS,EACT,KAAK,EACL,UAAU,EACV,OAAO,EACP,YAAY,EACZ,iBAAiB,EACjB,QAAQ,EACR,aAAa,EACb,SAA8B,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,KAAa,EACb,WAAe,EACf,SAAS,EACT,OAAe,EACf,QAAe,EACf,UAAiB,EACjB,UAAiB,EACjB,aAAqB,EACrB,OAAe,EACf,KAAS,EACT,eAAe,EACf,QAAQ,EACR,YAAoB,GACrB,EAAE,yBAAyB,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CA4FlF"}