forgedock 1.0.22 → 1.1.2

package/scripts/eval-gate-scorecard.mjs

@@ -0,0 +1,242 @@
+#!/usr/bin/env node
+/**
+ * scripts/eval-gate-scorecard.mjs — CI regression gate comparator (#1286)
+ *
+ * Deterministic, zero-network comparator invoked by `.github/workflows/eval-gate.yml`.
+ * Compares a fresh scorecard (produced by scripts/eval-scorecard-aggregator.mjs, #1285)
+ * against the committed baseline at scripts/eval-gate-baseline.json and fails CI when
+ * the one-shot success rate regresses beyond a defined threshold.
+ *
+ * It does NOT call any model, network, or live SDK — it only diffs two scorecard
+ * JSON objects. See docs/design/eval-gate-runbook.md for the full behavior spec.
+ *
+ * Usage:
+ *   node scripts/eval-gate-scorecard.mjs <scorecard.json> <baseline.json>  # compare mode
+ *   node scripts/eval-gate-scorecard.mjs <scorecard.json> --seed            # seed mode
+ *
+ * Compare mode: exits 0 (pass) unless the fresh success_rate_pct drops more than
+ * RATE_THRESHOLD_PP percentage points below the baseline. The rate assertion is
+ * skipped (still exits 0) when the fresh run has fewer than MIN_ISSUES_FOR_RATE_ASSERTION
+ * issues — a subset run is too small for a meaningful rate comparison.
+ *
+ * Seed mode: overwrites scripts/eval-gate-baseline.json with the fresh scorecard.
+ * Used after a deliberate, confirmed improvement (see runbook "Updating the baseline").
+ *
+ * Exit codes:
+ *   0 = gate passed (or rate assertion skipped) / baseline seeded successfully
+ *   1 = gate failed (regression) or invalid input (bad JSON, missing required field)
+ */
+
+import { readFileSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+/** Fields the fresh scorecard must have (matches eval-scorecard-aggregator.mjs's output). */
+export const REQUIRED_FIELDS = [
+  "schema_version",
+  "issues_run",
+  "successful_runs",
+  "failed_runs",
+  "success_rate_pct",
+  "run_mode",
+];
+
+/** Regression threshold in percentage points (see docs/design/eval-gate-runbook.md). */
+export const RATE_THRESHOLD_PP = 5;
+
+/** Below this many issues, the rate assertion is skipped as statistically meaningless. */
+export const MIN_ISSUES_FOR_RATE_ASSERTION = 8;
+
+/** Round to 2 decimal places (matches eval-scorecard-aggregator.mjs's success_rate_pct rounding). */
+function round2(x) {
+  return Math.round(x * 100) / 100;
+}
+
+/**
+ * Validate a scorecard-shaped object. Throws a descriptive Error on any violation.
+ *
+ * @param {unknown} sc
+ * @param {object} [opts]
+ * @param {string[]} [opts.requiredFields] - Fields that must be present.
+ * @param {string} [opts.label] - Label used in error messages (e.g. "fresh scorecard").
+ * @returns {void}
+ */
+export function validateScorecard(sc, { requiredFields = REQUIRED_FIELDS, label = "scorecard" } = {}) {
+  if (!sc || typeof sc !== "object" || Array.isArray(sc)) {
+    throw new Error(`${label} must be a JSON object`);
+  }
+  for (const field of requiredFields) {
+    if (!Object.prototype.hasOwnProperty.call(sc, field)) {
+      throw new Error(`${label} missing required field: "${field}"`);
+    }
+  }
+  if (typeof sc.success_rate_pct !== "number" || !Number.isFinite(sc.success_rate_pct)) {
+    throw new Error(
+      `${label}.success_rate_pct must be a finite number (got ${JSON.stringify(sc.success_rate_pct)})`,
+    );
+  }
+}
+
+/**
+ * Compare a fresh scorecard against the baseline and decide pass/fail.
+ * Does NOT re-validate beyond what's needed to compute the diff — call
+ * validateScorecard() first for full field validation (this function calls
+ * it internally, so it is safe to call directly).
+ *
+ * @param {object} fresh - Scorecard from the current CI run.
+ * @param {object} baseline - Committed baseline scorecard.
+ * @param {object} [opts]
+ * @param {number} [opts.thresholdPp] - Regression threshold in percentage points.
+ * @param {number} [opts.minIssuesForAssertion] - Min issues before the rate assertion applies.
+ * @returns {object} Scorecard-diff result — see docs/design/eval-gate-runbook.md
+ *   "Reading the scorecard diff" for the field contract.
+ */
+export function evaluateGate(
+  fresh,
+  baseline,
+  { thresholdPp = RATE_THRESHOLD_PP, minIssuesForAssertion = MIN_ISSUES_FOR_RATE_ASSERTION } = {},
+) {
+  validateScorecard(fresh, { requiredFields: REQUIRED_FIELDS, label: "fresh scorecard" });
+  // Baseline may predate newer fields the aggregator emits — only success_rate_pct
+  // is strictly required for the comparison itself.
+  validateScorecard(baseline, { requiredFields: ["success_rate_pct"], label: "baseline" });
+
+  const freshIssuesRun = typeof fresh.issues_run === "number" ? fresh.issues_run : null;
+  const rateAssertionSkipped = freshIssuesRun !== null && freshIssuesRun < minIssuesForAssertion;
+
+  const dropPp = round2(baseline.success_rate_pct - fresh.success_rate_pct);
+
+  let pass;
+  let reason;
+  if (rateAssertionSkipped) {
+    pass = true;
+    reason =
+      `rate assertion skipped: fresh run has ${freshIssuesRun} issue(s), below the minimum of ` +
+      `${minIssuesForAssertion} required for a meaningful rate comparison`;
+  } else if (dropPp > thresholdPp) {
+    pass = false;
+    reason =
+      `REGRESSION: success rate dropped ${dropPp} pp (${fresh.success_rate_pct}% fresh vs ` +
+      `${baseline.success_rate_pct}% baseline). Threshold is ${thresholdPp} pp.`;
+  } else {
+    pass = true;
+    reason =
+      `success rate delta ${dropPp} pp is within the ${thresholdPp} pp threshold ` +
+      `(${fresh.success_rate_pct}% fresh vs ${baseline.success_rate_pct}% baseline).`;
+  }
+
+  return {
+    pass,
+    fresh_success_rate_pct: fresh.success_rate_pct,
+    baseline_success_rate_pct: baseline.success_rate_pct,
+    drop_pp: dropPp,
+    threshold_pp: thresholdPp,
+    rate_assertion_skipped: rateAssertionSkipped,
+    fresh_issues_run: freshIssuesRun,
+    fresh_run_mode: fresh.run_mode ?? null,
+    fresh_run_id: fresh.run_id ?? null,
+    baseline_run_id: baseline.run_id ?? null,
+    reason,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// CLI entry point
+// ---------------------------------------------------------------------------
+
+function readJsonFile(path) {
+  return JSON.parse(readFileSync(path, "utf8"));
+}
+
+/** Actionable next-steps text appended to a gate-failure message (see runbook). */
+function actionableStepsText() {
+  return (
+    "\n" +
+    "1. Review the diff in commands/ to identify what changed.\n" +
+    "2. Re-run the full corpus locally:\n" +
+    "   node scripts/eval-harness-runner.mjs --mode full\n" +
+    "3. If the regression is real: fix the spec, re-run, update the baseline\n" +
+    "   (see docs/design/eval-gate-runbook.md).\n" +
+    "4. If the regression is a false positive: see the Override section in\n" +
+    "   docs/design/eval-gate-runbook.md."
+  );
+}
+
+/** Resolve scripts/eval-gate-baseline.json relative to this script's own directory,
+ *  so seed mode is robust regardless of the invoking CWD. */
+function defaultBaselinePath() {
+  return join(dirname(fileURLToPath(import.meta.url)), "eval-gate-baseline.json");
+}
+
+function usageError() {
+  process.stderr.write(
+    "ERROR: usage: eval-gate-scorecard.mjs <scorecard.json> <baseline.json>\n" +
+      "   or: eval-gate-scorecard.mjs <scorecard.json> --seed\n",
+  );
+  process.exit(1);
+}
+
+function main() {
+  const [scorecardArg, secondArg] = process.argv.slice(2);
+
+  if (!scorecardArg) usageError();
+
+  let fresh;
+  try {
+    fresh = readJsonFile(scorecardArg);
+  } catch (e) {
+    process.stderr.write(`ERROR: cannot read scorecard "${scorecardArg}": ${e.message}\n`);
+    process.exit(1);
+  }
+
+  if (secondArg === "--seed") {
+    try {
+      validateScorecard(fresh, { requiredFields: REQUIRED_FIELDS, label: "scorecard" });
+    } catch (e) {
+      process.stderr.write(`ERROR: ${e.message}\n`);
+      process.exit(1);
+    }
+    const baselinePath = defaultBaselinePath();
+    try {
+      writeFileSync(baselinePath, JSON.stringify(fresh, null, 2) + "\n");
+    } catch (e) {
+      process.stderr.write(`ERROR: cannot write baseline "${baselinePath}": ${e.message}\n`);
+      process.exit(1);
+    }
+    process.stdout.write(`Baseline updated: ${baselinePath}\n`);
+    process.stdout.write(`New baseline success_rate_pct: ${fresh.success_rate_pct}\n`);
+    return;
+  }
+
+  if (!secondArg) usageError();
+
+  let baseline;
+  try {
+    baseline = readJsonFile(secondArg);
+  } catch (e) {
+    process.stderr.write(`ERROR: cannot read baseline "${secondArg}": ${e.message}\n`);
+    process.exit(1);
+  }
+
+  let result;
+  try {
+    result = evaluateGate(fresh, baseline);
+  } catch (e) {
+    process.stderr.write(`ERROR: ${e.message}\n`);
+    process.exit(1);
+  }
+
+  process.stdout.write(JSON.stringify(result, null, 2) + "\n");
+
+  if (!result.pass) {
+    process.stderr.write(`\neval-gate FAILED: ${result.reason}${actionableStepsText()}\n`);
+    process.exit(1);
+  }
+
+  process.stdout.write(`\neval-gate PASSED: ${result.reason}\n`);
+}
+
+// Run as CLI only when invoked directly, not when imported by tests.
+if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
+  main();
+}

package/scripts/eval-scorecard-aggregator.mjs

@@ -0,0 +1,252 @@
+#!/usr/bin/env node
+/**
+ * scripts/eval-scorecard-aggregator.mjs — Per-run results → scorecard aggregator (#1285)
+ *
+ * Deterministic, zero-network aggregator for the ForgeDock headless eval harness.
+ * Reads an array of per-run results (one per corpus issue) produced by
+ * bin/batch-runner.mjs and emits a scorecard JSON in the format consumed by
+ * scripts/eval-gate-scorecard.mjs (the CI regression gate, issue #1286).
+ *
+ * It does NOT call any model, network, or live SDK — it only aggregates the
+ * structured output that bin/batch-runner.mjs produces.
+ * See docs/spec/eval-run-result.md for the full JSON schema.
+ *
+ * Usage:
+ *   node scripts/eval-scorecard-aggregator.mjs <runs.json>   # read file, print scorecard JSON
+ *   node scripts/eval-scorecard-aggregator.mjs -             # read runs JSON from stdin
+ *   cat runs.json | node scripts/eval-scorecard-aggregator.mjs
+ *
+ * Input format: a JSON object with a "runs" array of per-run result objects,
+ * plus optional metadata fields (corpus_version, run_id, run_mode, spec_sha).
+ *
+ * Output format: scorecard JSON compatible with scripts/eval-gate-scorecard.mjs.
+ *
+ * Methodology rule (enforced): fewer than MIN_RUNS valid, scoreable runs is a
+ * HARD ERROR — the script exits non-zero rather than emit a misleading rate.
+ * Mirrors bench-scorecard.mjs's MIN_RUNS methodology rule.
+ *
+ * Exit codes:
+ *   0 = scorecard emitted successfully
+ *   1 = invalid input (bad JSON, n < MIN_RUNS, missing required fields)
+ */
+
+/** Minimum number of scoreable (non-error, non-incomplete) runs required. */
+export const MIN_RUNS = 5;
+
+/** Valid status values for a per-run result object. */
+export const VALID_STATUSES = new Set(["success", "failure", "incomplete", "error"]);
+
+/**
+ * Validate an array of raw run-result objects.
+ * Throws a descriptive Error on any violation.
+ *
+ * @param {unknown[]} runs
+ * @returns {void}
+ */
+export function validateRuns(runs) {
+  if (!Array.isArray(runs)) {
+    throw new Error('"runs" must be a JSON array');
+  }
+  if (runs.length === 0) {
+    throw new Error('"runs" array is empty — nothing to score');
+  }
+  for (let i = 0; i < runs.length; i++) {
+    const r = runs[i];
+    if (!r || typeof r !== "object") {
+      throw new Error(`run[${i}]: must be an object`);
+    }
+    if (typeof r.issue !== "number" || !Number.isInteger(r.issue)) {
+      throw new Error(`run[${i}]: "issue" must be an integer (got ${JSON.stringify(r.issue)})`);
+    }
+    if (!VALID_STATUSES.has(r.status)) {
+      throw new Error(
+        `run[${i}] (issue ${r.issue}): "status" must be one of ${[...VALID_STATUSES].join(", ")} (got ${JSON.stringify(r.status)})`,
+      );
+    }
+    if (typeof r.wallClockMs !== "number" || !Number.isFinite(r.wallClockMs) || r.wallClockMs < 0) {
+      throw new Error(
+        `run[${i}] (issue ${r.issue}): "wallClockMs" must be a non-negative finite number (got ${JSON.stringify(r.wallClockMs)})`,
+      );
+    }
+    if (typeof r.interventionCount !== "number" || !Number.isInteger(r.interventionCount) || r.interventionCount < 0) {
+      throw new Error(
+        `run[${i}] (issue ${r.issue}): "interventionCount" must be a non-negative integer (got ${JSON.stringify(r.interventionCount)})`,
+      );
+    }
+    // cost is optional/nullable — validate only when present and non-null
+    if (r.cost !== undefined && r.cost !== null) {
+      if (typeof r.cost !== "number" || !Number.isFinite(r.cost) || r.cost < 0) {
+        throw new Error(
+          `run[${i}] (issue ${r.issue}): "cost" must be a non-negative finite number or null (got ${JSON.stringify(r.cost)})`,
+        );
+      }
+    }
+  }
+
+  // Enforce minimum sample size on scoreable runs (excludes "error" / "incomplete").
+  const scoreable = runs.filter((r) => r.status === "success" || r.status === "failure");
+  if (scoreable.length < MIN_RUNS) {
+    throw new Error(
+      `only ${scoreable.length} scoreable run(s) (status "success" or "failure") found; ` +
+        `minimum is ${MIN_RUNS}. Add more corpus issues or fix erroring/incomplete runs.`,
+    );
+  }
+}
+
+/**
+ * Compute the median of a sorted numeric array.
+ * Returns null for an empty array.
+ *
+ * @param {number[]} sorted - Must be pre-sorted ascending.
+ * @returns {number|null}
+ */
+export function median(sorted) {
+  const n = sorted.length;
+  if (n === 0) return null;
+  const mid = Math.floor(n / 2);
+  return n % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
+}
+
+/**
+ * Arithmetic mean of a numeric array.
+ * Returns null for an empty array.
+ *
+ * @param {number[]} values
+ * @returns {number|null}
+ */
+export function mean(values) {
+  if (values.length === 0) return null;
+  return values.reduce((a, b) => a + b, 0) / values.length;
+}
+
+/** Round to 2 decimal places (matches eval-gate-scorecard.mjs success_rate_pct rounding). */
+function round2(x) {
+  return x === null ? null : Math.round(x * 100) / 100;
+}
+
+/** Round to 3 decimal places for wall-clock stats. */
+function round3(x) {
+  return x === null ? null : Math.round(x * 1000) / 1000;
+}
+
+/**
+ * Aggregate validated run-result objects into a scorecard compatible with
+ * scripts/eval-gate-scorecard.mjs (#1286).
+ *
+ * Does NOT re-validate — call validateRuns() first.
+ *
+ * @param {object[]} runs
+ * @param {object} [meta]
+ * @param {string|null} [meta.run_id]          - Unique run identifier.
+ * @param {string|null} [meta.corpus_version]  - Corpus version label.
+ * @param {"full"|"subset"} [meta.run_mode]    - "full" or "subset".
+ * @param {string|null} [meta.spec_sha]        - git SHA of commands/ at run time.
+ * @param {number|null} [meta.corpus_size]     - Size of the source corpus (defaults to issues_run).
+ * @returns {object} Scorecard compatible with eval-gate-scorecard.mjs.
+ */
+export function aggregate(runs, meta = {}) {
+  const issuesRun = runs.length;
+  const successfulRuns = runs.filter((r) => r.status === "success").length;
+  const failedRuns = runs.filter((r) => r.status === "failure").length;
+
+  // success_rate_pct: computed from ALL runs (including error/incomplete),
+  // matching the harness intent: incomplete runs count against the rate.
+  const successRatePct = issuesRun > 0 ? round2((successfulRuns / issuesRun) * 100) : 0;
+
+  // Wall-clock stats — include ALL runs.
+  const wallClockValues = runs.map((r) => r.wallClockMs).sort((a, b) => a - b);
+  const meanWallClockMs = round3(mean(wallClockValues));
+  const medianWallClockMs = round3(median(wallClockValues));
+
+  // Intervention stats.
+  const totalInterventionCount = runs.reduce((sum, r) => sum + r.interventionCount, 0);
+
+  // Cost: null if any run has cost === null or missing; sum otherwise.
+  const costValues = runs.map((r) => (r.cost !== undefined ? r.cost : null));
+  const totalCost = costValues.every((c) => c !== null)
+    ? round3(costValues.reduce((a, b) => a + b, 0))
+    : null;
+
+  const runMode = meta.run_mode ?? "full";
+  if (runMode !== "full" && runMode !== "subset") {
+    throw new Error(`run_mode must be "full" or "subset" (got ${JSON.stringify(runMode)})`);
+  }
+
+  return {
+    schema_version: "1",
+    run_id: meta.run_id ?? null,
+    generated_at: new Date().toISOString(),
+    corpus_size: meta.corpus_size ?? issuesRun,
+    issues_run: issuesRun,
+    successful_runs: successfulRuns,
+    failed_runs: failedRuns,
+    success_rate_pct: successRatePct,
+    mean_wall_clock_ms: meanWallClockMs,
+    median_wall_clock_ms: medianWallClockMs,
+    total_intervention_count: totalInterventionCount,
+    cost: totalCost,
+    run_mode: runMode,
+    spec_sha: meta.spec_sha ?? null,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// CLI entry point
+// ---------------------------------------------------------------------------
+
+async function readInput(arg) {
+  const { readFileSync } = await import("node:fs");
+  if (!arg || arg === "-") {
+    return readFileSync(0, "utf8"); // fd 0 = stdin
+  }
+  return readFileSync(arg, "utf8");
+}
+
+async function main() {
+  const arg = process.argv[2];
+  let raw;
+  try {
+    raw = await readInput(arg);
+  } catch (e) {
+    process.stderr.write(`ERROR: cannot read input: ${e.message}\n`);
+    process.exit(1);
+  }
+
+  let data;
+  try {
+    data = JSON.parse(raw);
+  } catch (e) {
+    process.stderr.write(`ERROR: invalid JSON: ${e.message}\n`);
+    process.exit(1);
+  }
+
+  if (!data || typeof data !== "object") {
+    process.stderr.write(`ERROR: input must be a JSON object with a "runs" array\n`);
+    process.exit(1);
+  }
+
+  const { runs, run_id, corpus_version, run_mode, spec_sha, corpus_size } = data;
+
+  try {
+    validateRuns(runs);
+  } catch (e) {
+    process.stderr.write(`ERROR: ${e.message}\n`);
+    process.exit(1);
+  }
+
+  let scorecard;
+  try {
+    scorecard = aggregate(runs, { run_id, corpus_version, run_mode, spec_sha, corpus_size });
+  } catch (e) {
+    process.stderr.write(`ERROR: ${e.message}\n`);
+    process.exit(1);
+  }
+
+  process.stdout.write(JSON.stringify(scorecard, null, 2) + "\n");
+}
+
+// Run as CLI only when invoked directly, not when imported by tests.
+import { fileURLToPath } from "node:url";
+if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
+  main();
+}