@aldus-runtime/regression 0.1.0

package/src/promotion.ts

@@ -0,0 +1,336 @@
+/**
+ * Evaluator promotion (architecture contract §12.1, §25 item 9; ADR-0010).
+ *
+ * §12.1: "An evaluator MAY become blocking only after it is calibrated against human-labeled
+ * examples." This module turns that permission into an evidence check.
+ *
+ * Two decisions shape everything here, both recorded in ADR-0010:
+ *
+ * 1. **Promotion is always scoped.** There is no such thing as promoting an evaluator outright.
+ *    A verdict names the slices where the bar is met, and §12.1's own list of scope dimensions
+ *    is why: calibration on one host, voice, or script form is not evidence about another.
+ * 2. **The whole-corpus figure never decides anything.** It is computed and reported because a
+ *    reader wants the corpus's shape, but no threshold is applied to it. An evaluator that looks
+ *    excellent in aggregate while failing one slice must not read as promotable, and the surest
+ *    way to guarantee that is to give the aggregate no vote.
+ */
+
+import type { BlindSpot, BlindSpotRegistry } from "./blindspots.js";
+import type { ComparisonReport, SliceMetrics } from "./metrics.js";
+import type { PolicyOrigin, PromotionPolicy } from "./policy.js";
+import type { ScopeSelector } from "./scope.js";
+
+/** Why a slice did not clear the bar. */
+export interface PromotionShortfall {
+  /** Machine-readable reason. */
+  code: PromotionShortfallCode;
+  /** What the slice needed. */
+  required: number | string;
+  /** What it had. `undefined` when the metric was unmeasurable. */
+  observed: number | string | undefined;
+  /** Operator-facing explanation. */
+  message: string;
+}
+
+/** Reasons a slice can fail (contract §12.1). */
+export type PromotionShortfallCode =
+  | "insufficient_cases"
+  | "insufficient_defective_cases"
+  | "insufficient_clean_cases"
+  | "insufficient_labellers"
+  | "recall_below_threshold"
+  | "severity_weighted_recall_below_threshold"
+  | "false_positive_rate_above_threshold"
+  | "unnecessary_correction_harm_above_threshold"
+  | "open_blind_spot"
+  | "unmeasurable";
+
+/** Whether one slice clears the bar, and why not when it does not. */
+export interface SliceVerdict {
+  /** Which slice. */
+  selector: ScopeSelector;
+  /** Stable key for the slice. */
+  key: string;
+  /** Metrics the verdict was computed from. */
+  metrics: SliceMetrics;
+  /**
+   * Whether this slice clears the §12.1 bar.
+   *
+   * Named for what it is: evidence sufficient to *permit* promotion, not proof the evaluator is
+   * right. §12 forbids presenting a machine pass as semantic correctness.
+   */
+  meetsPromotionBar: boolean;
+  /** Every reason it does not, empty when it does. */
+  shortfalls: readonly PromotionShortfall[];
+  /** Open blind spots applying to this slice (contract §12.1, §9.3). */
+  openBlindSpots: readonly BlindSpot[];
+}
+
+/** The full promotion verdict. */
+export interface PromotionVerdict {
+  /** Evaluator assessed. */
+  evaluatorId: string;
+  /** Version assessed. */
+  evaluatorVersion: string;
+  /** Corpus assessed against. */
+  corpusId: string;
+  /**
+   * Where the policy's numbers came from.
+   *
+   * `default-uncalibrated` means the bar itself has never been validated against a real corpus
+   * (ADR-0010). A verdict is only as trustworthy as the thresholds it was measured against, and
+   * hiding that would be its own dishonesty.
+   */
+  policyOrigin: PolicyOrigin;
+  /** Per-slice verdicts (contract §12.1). */
+  slices: readonly SliceVerdict[];
+  /** Slices that clear the bar. May be empty. */
+  promotableScopes: readonly string[];
+  /** Slices that do not. */
+  blockedScopes: readonly string[];
+  /**
+   * Whole-corpus metrics, for context only.
+   *
+   * Deliberately not accompanied by a whole-corpus verdict: no threshold is applied to it
+   * (ADR-0010 decision 3).
+   */
+  wholeCorpus: SliceMetrics;
+  /**
+   * True when the aggregate would flatter the evaluator relative to its worst slice.
+   *
+   * Surfaced so a report can say so out loud rather than leaving a reader to compare the numbers
+   * themselves — this is the specific failure §12.1's scope requirement exists to prevent.
+   */
+  aggregateFlattersWorstScope: boolean;
+  /** Cases the run never reported on. */
+  unevaluatedCaseIds: readonly string[];
+}
+
+function shortfall(
+  code: PromotionShortfallCode,
+  required: number | string,
+  observed: number | string | undefined,
+  message: string,
+): PromotionShortfall {
+  return { code, required, observed, message };
+}
+
+/** Assess one slice against the policy. */
+function assessSlice(
+  metrics: SliceMetrics,
+  policy: PromotionPolicy,
+  openBlindSpots: readonly BlindSpot[],
+): SliceVerdict {
+  const { thresholds } = policy;
+  const shortfalls: PromotionShortfall[] = [];
+
+  if (metrics.cases < thresholds.minCases) {
+    shortfalls.push(
+      shortfall(
+        "insufficient_cases",
+        thresholds.minCases,
+        metrics.cases,
+        `Only ${metrics.cases} labelled cases; the bar is ${thresholds.minCases}.`,
+      ),
+    );
+  }
+  if (metrics.defectiveCases < thresholds.minDefectiveCases) {
+    shortfalls.push(
+      shortfall(
+        "insufficient_defective_cases",
+        thresholds.minDefectiveCases,
+        metrics.defectiveCases,
+        `Only ${metrics.defectiveCases} cases a human labelled defective; recall over so few is ` +
+          "not a measurement.",
+      ),
+    );
+  }
+  if (metrics.cleanCases < thresholds.minCleanCases) {
+    shortfalls.push(
+      shortfall(
+        "insufficient_clean_cases",
+        thresholds.minCleanCases,
+        metrics.cleanCases,
+        `Only ${metrics.cleanCases} cases a human labelled clean, so the false-positive rate is ` +
+          "barely constrained.",
+      ),
+    );
+  }
+  if (metrics.labellers < thresholds.minLabellers) {
+    shortfalls.push(
+      shortfall(
+        "insufficient_labellers",
+        thresholds.minLabellers,
+        metrics.labellers,
+        `Labels come from ${metrics.labellers} person(s); the bar is ${thresholds.minLabellers}.`,
+      ),
+    );
+  }
+
+  // An unmeasurable metric is a shortfall, never a pass. A slice with no defective cases has an
+  // undefined recall, and treating undefined as satisfied would promote an evaluator that was
+  // never tested on a single defect.
+  if (metrics.recall === undefined) {
+    shortfalls.push(
+      shortfall(
+        "unmeasurable",
+        thresholds.minRecall,
+        undefined,
+        "Recall is unmeasurable here: no case in this slice was labelled defective.",
+      ),
+    );
+  } else if (metrics.recall < thresholds.minRecall) {
+    shortfalls.push(
+      shortfall(
+        "recall_below_threshold",
+        thresholds.minRecall,
+        metrics.recall,
+        `Recall ${metrics.recall.toFixed(3)} is below the required ${thresholds.minRecall}; the ` +
+          `evaluator missed ${metrics.falseNegatives} defect(s) a human found.`,
+      ),
+    );
+  }
+
+  if (metrics.severityWeightedRecall === undefined) {
+    if (metrics.recall !== undefined) {
+      shortfalls.push(
+        shortfall(
+          "unmeasurable",
+          thresholds.minSeverityWeightedRecall,
+          undefined,
+          "Severity-weighted recall is unmeasurable: the defective cases carry no severity weight.",
+        ),
+      );
+    }
+  } else if (metrics.severityWeightedRecall < thresholds.minSeverityWeightedRecall) {
+    shortfalls.push(
+      shortfall(
+        "severity_weighted_recall_below_threshold",
+        thresholds.minSeverityWeightedRecall,
+        metrics.severityWeightedRecall,
+        `Severity-weighted recall ${metrics.severityWeightedRecall.toFixed(3)} is below the ` +
+          `required ${thresholds.minSeverityWeightedRecall}. Weighted misses total ` +
+          `${metrics.severityWeightedFalseNegatives}, so what it missed was disproportionately ` +
+          "severe (architecture contract §12.1).",
+      ),
+    );
+  }
+
+  if (metrics.falsePositiveRate === undefined) {
+    shortfalls.push(
+      shortfall(
+        "unmeasurable",
+        thresholds.maxFalsePositiveRate,
+        undefined,
+        "False-positive rate is unmeasurable here: no case in this slice was labelled clean.",
+      ),
+    );
+  } else if (metrics.falsePositiveRate > thresholds.maxFalsePositiveRate) {
+    shortfalls.push(
+      shortfall(
+        "false_positive_rate_above_threshold",
+        thresholds.maxFalsePositiveRate,
+        metrics.falsePositiveRate,
+        `False-positive rate ${metrics.falsePositiveRate.toFixed(3)} exceeds the permitted ` +
+          `${thresholds.maxFalsePositiveRate}.`,
+      ),
+    );
+  }
+
+  if (
+    metrics.meanUnnecessaryCorrectionHarm !== undefined &&
+    metrics.meanUnnecessaryCorrectionHarm > thresholds.maxUnnecessaryCorrectionHarm
+  ) {
+    shortfalls.push(
+      shortfall(
+        "unnecessary_correction_harm_above_threshold",
+        thresholds.maxUnnecessaryCorrectionHarm,
+        metrics.meanUnnecessaryCorrectionHarm,
+        `Mean unnecessary-correction harm ${metrics.meanUnnecessaryCorrectionHarm.toFixed(3)} ` +
+          `exceeds the permitted ${thresholds.maxUnnecessaryCorrectionHarm}. Its false positives ` +
+          "trigger expensive repairs, which §12.1 weighs separately from how often they occur.",
+      ),
+    );
+  }
+
+  if (policy.openBlindSpotDisqualifies && openBlindSpots.length > 0) {
+    shortfalls.push(
+      shortfall(
+        "open_blind_spot",
+        0,
+        openBlindSpots.length,
+        `${openBlindSpots.length} open blind spot(s) apply here: ` +
+          `${openBlindSpots.map((record) => record.blindSpotId).join(", ")}. Corpus metrics are ` +
+          "not evidence against a blind spot — a blind spot is what the corpus did not sample " +
+          "(architecture contract §12.1, §9.3).",
+      ),
+    );
+  }
+
+  return {
+    selector: metrics.selector,
+    key: metrics.key,
+    metrics,
+    meetsPromotionBar: shortfalls.length === 0,
+    shortfalls,
+    openBlindSpots,
+  };
+}
+
+/**
+ * Decide whether an evaluator may be promoted to blocking, per scope (contract §12.1).
+ *
+ * Returns a verdict rather than throwing. A blocked promotion is information an operator needs
+ * rendered — several slices may fail for different reasons at once, and an exception would carry
+ * one and discard the rest. This follows ADR-0006 decision 5's reasoning for pack resolution.
+ */
+export function assessPromotion(
+  comparison: ComparisonReport,
+  policy: PromotionPolicy,
+  blindSpots?: BlindSpotRegistry,
+): PromotionVerdict {
+  const slices = comparison.slices.map((metrics) =>
+    assessSlice(
+      metrics,
+      policy,
+      blindSpots?.openFor(comparison.evaluatorId, metrics.selector) ?? [],
+    ),
+  );
+
+  const promotable = slices.filter((slice) => slice.meetsPromotionBar);
+  const blocked = slices.filter((slice) => !slice.meetsPromotionBar);
+
+  // Does the aggregate look better than the worst slice? Compared on agreement because that is
+  // the figure a reader is most likely to skim and mistake for a verdict.
+  const worstAgreement = comparison.slices.reduce<number | undefined>((worst, slice) => {
+    const value = slice.agreementWithHumanLabels;
+    if (value === undefined) return worst;
+    return worst === undefined || value < worst ? value : worst;
+  }, undefined);
+  const aggregate = comparison.wholeCorpus.agreementWithHumanLabels;
+  const aggregateFlattersWorstScope =
+    aggregate !== undefined && worstAgreement !== undefined && aggregate > worstAgreement;
+
+  return {
+    evaluatorId: comparison.evaluatorId,
+    evaluatorVersion: comparison.evaluatorVersion,
+    corpusId: comparison.corpusId,
+    policyOrigin: policy.origin,
+    slices,
+    promotableScopes: promotable.map((slice) => slice.key),
+    blockedScopes: blocked.map((slice) => slice.key),
+    wholeCorpus: comparison.wholeCorpus,
+    aggregateFlattersWorstScope,
+    unevaluatedCaseIds: comparison.unevaluatedCaseIds,
+  };
+}
+
+/**
+ * True when every measured slice clears the bar **and** at least one slice was measured.
+ *
+ * The second half is not pedantry: an empty slice list would otherwise satisfy `every` and read
+ * as universal approval of an evaluator nothing was measured about.
+ */
+export function isPromotableEverywhereMeasured(verdict: PromotionVerdict): boolean {
+  return verdict.slices.length > 0 && verdict.blockedScopes.length === 0;
+}

package/src/report.ts

@@ -0,0 +1,154 @@
+/**
+ * The promotion report (architecture contract §12, §12.1).
+ *
+ * §12 states: "Machine pass MUST NOT be presented as semantic correctness." That is a constraint
+ * on *this file* more than any other, because this is where numbers become prose a human acts
+ * on. Three rules follow, and each is pinned by a test:
+ *
+ * 1. **The aggregate is never rendered alone.** Every rendering that shows the whole-corpus
+ *    figure also shows the per-scope breakdown, and labels the aggregate as descriptive. §12.1
+ *    requires scope be considered; an aggregate-only report is how that requirement gets
+ *    quietly dropped.
+ * 2. **No word implying correctness.** The report says "agreed with human labels", never
+ *    "accurate", "correct", or "passed". A reader skimming for a verdict must not find one that
+ *    was never established.
+ * 3. **Blocked means explained.** A slice that fails lists every shortfall with the observed and
+ *    required values. "0.94" tells nobody whether to promote.
+ */
+
+import type { PromotionVerdict, SliceVerdict } from "./promotion.js";
+import type { SliceMetrics } from "./metrics.js";
+import { scopeLabel } from "./scope.js";
+
+/** Words this report must never use about an evaluator's output (contract §12). */
+const FORBIDDEN_CLAIM_WORDS: readonly string[] = [
+  "accurate",
+  "accuracy",
+  "correctness",
+  "correct",
+  "proven",
+  "verified",
+  "guarantees",
+];
+
+/**
+ * The standing caveat, printed on every report.
+ *
+ * Not decoration. §12 forbids presenting a machine pass as semantic correctness, and §12.1's
+ * scope requirement exists because corpora do not generalise. A reader who takes only the
+ * headline away should still have been told both.
+ */
+export const REPORT_CAVEAT =
+  "This report measures agreement with human labels on one corpus. It is not evidence that the " +
+  "evaluator is semantically right (architecture contract §12), and it says nothing about " +
+  "scopes the corpus does not contain — those were not evaluated and must not be assumed covered.";
+
+function formatRate(value: number | undefined): string {
+  return value === undefined ? "unmeasurable" : value.toFixed(3);
+}
+
+function renderMetrics(metrics: SliceMetrics, indent: string): string[] {
+  return [
+    `${indent}cases ${metrics.cases} (${metrics.defectiveCases} labelled defective, ${metrics.cleanCases} labelled clean, ${metrics.labellers} labeller(s))`,
+    `${indent}recall ${formatRate(metrics.recall)}   severity-weighted recall ${formatRate(metrics.severityWeightedRecall)}`,
+    `${indent}false-positive rate ${formatRate(metrics.falsePositiveRate)}   mean unnecessary-correction harm ${formatRate(metrics.meanUnnecessaryCorrectionHarm)}`,
+    `${indent}missed ${metrics.falseNegatives} (weighted ${metrics.severityWeightedFalseNegatives}), spurious flags ${metrics.falsePositives}, wrong-category detections ${metrics.categoryMismatches}`,
+    `${indent}agreement with human labels ${formatRate(metrics.agreementWithHumanLabels)}`,
+  ];
+}
+
+function renderSlice(slice: SliceVerdict): string[] {
+  const lines: string[] = [];
+  const status = slice.meetsPromotionBar ? "MEETS PROMOTION BAR" : "BLOCKED";
+  lines.push(`  [${status}] ${scopeLabel(slice.selector)}`);
+  lines.push(...renderMetrics(slice.metrics, "    "));
+  if (slice.shortfalls.length > 0) {
+    lines.push("    why it is blocked:");
+    for (const item of slice.shortfalls) lines.push(`      - ${item.message}`);
+  }
+  if (slice.openBlindSpots.length > 0) {
+    lines.push("    open blind spots:");
+    for (const record of slice.openBlindSpots) {
+      lines.push(`      - ${record.blindSpotId}: ${record.description}`);
+    }
+  }
+  return lines;
+}
+
+/**
+ * Render a promotion verdict as plain text (contract §12.1).
+ *
+ * The whole-corpus figure appears **after** the per-scope breakdown and is explicitly labelled
+ * descriptive, so a reader reaches the scope detail first. Ordering is part of the guarantee: a
+ * summary at the top is the one people quote.
+ */
+export function renderPromotionReport(verdict: PromotionVerdict): string {
+  const lines: string[] = [];
+
+  lines.push(
+    `Evaluator promotion evidence — ${verdict.evaluatorId} @ ${verdict.evaluatorVersion}`,
+    `Corpus: ${verdict.corpusId}`,
+    "",
+    REPORT_CAVEAT,
+    "",
+  );
+
+  if (verdict.policyOrigin === "default-uncalibrated") {
+    lines.push(
+      "NOTE: assessed against this package's default thresholds, which have not themselves been " +
+        "calibrated against any real corpus (ADR-0010). Treat the bar as provisional.",
+      "",
+    );
+  }
+
+  lines.push(`Scopes assessed: ${verdict.slices.length}`);
+  if (verdict.slices.length === 0) {
+    lines.push(
+      "  none — the corpus declares no scope dimensions, so nothing could be assessed per scope.",
+      "  Promotion is scoped (ADR-0010); with no scopes there is no promotion decision to make.",
+    );
+  }
+  lines.push("");
+
+  for (const slice of verdict.slices) lines.push(...renderSlice(slice), "");
+
+  lines.push("Whole corpus (descriptive only — no threshold is applied to it, ADR-0010):");
+  lines.push(...renderMetrics(verdict.wholeCorpus, "  "));
+
+  if (verdict.aggregateFlattersWorstScope) {
+    lines.push(
+      "",
+      "WARNING: the whole-corpus figure is better than the worst scope above. Reading the " +
+        "aggregate alone would overstate this evaluator (architecture contract §12.1).",
+    );
+  }
+
+  if (verdict.unevaluatedCaseIds.length > 0) {
+    lines.push(
+      "",
+      `${verdict.unevaluatedCaseIds.length} corpus case(s) the run never reported on; they count ` +
+        "as unflagged.",
+    );
+  }
+
+  lines.push(
+    "",
+    verdict.promotableScopes.length === 0
+      ? "Outcome: no scope meets the promotion bar. The evaluator stays advisory (§12 level 2)."
+      : `Outcome: ${verdict.promotableScopes.length} of ${verdict.slices.length} scope(s) meet ` +
+          `the bar — ${verdict.promotableScopes.join(", ")}. Promotion applies only to those ` +
+          "scopes; every other scope stays advisory.",
+  );
+
+  return lines.join("\n");
+}
+
+/**
+ * Words the report must not use, exported so a test can enforce rule 2 above.
+ *
+ * Exported rather than kept private because the guarantee is only real if something checks it,
+ * and a test importing the same list cannot drift from the implementation.
+ */
+export function forbiddenClaimWords(): readonly string[] {
+  return FORBIDDEN_CLAIM_WORDS;
+}

package/src/scope.ts

@@ -0,0 +1,100 @@
+/**
+ * Scope slicing (architecture contract §12.1).
+ *
+ * §12.1 lists "show, host, voice, model, and script-form scope" among what evaluator promotion
+ * must consider. The reason is that calibration does not generalise: an evaluator tuned on one
+ * host's cadence says nothing about another's, and one calibrated on a single voice says nothing
+ * about a second voice's artefacts.
+ *
+ * So a corpus is sliced, and every metric is computed per slice. Dimensions are caller-supplied
+ * `Record<string, string>` throughout, consistent with Knowledge Pack scope (§9.2, ADR-0006) —
+ * §12.1's list is illustrative and §4.2 forbids Core from naming a provider.
+ */
+
+import type { ScopeDimensions } from "./corpus.js";
+
+/**
+ * A slice of the corpus: which dimensions were held fixed, and at what values.
+ *
+ * `dimensions` is empty for the whole-corpus slice, which is why that slice is named rather than
+ * left implicit — see {@link WHOLE_CORPUS_SLICE}.
+ */
+export interface ScopeSelector {
+  /** Dimension names held fixed by this slice, sorted. */
+  dimensions: readonly string[];
+  /** Value of each held dimension. */
+  values: Readonly<Record<string, string>>;
+}
+
+/** The selector matching every case: no dimension held fixed. */
+export const WHOLE_CORPUS_SLICE: ScopeSelector = { dimensions: [], values: {} };
+
+/**
+ * Stable string key for a selector, for grouping and for report ordering.
+ *
+ * Dimensions are sorted before joining so `{host,voice}` and `{voice,host}` produce one key.
+ * Values are separated by a character that cannot appear in a dimension name.
+ */
+export function scopeKey(selector: ScopeSelector): string {
+  if (selector.dimensions.length === 0) return "*";
+  return [...selector.dimensions]
+    .sort()
+    .map((dimension) => `${dimension}=${selector.values[dimension] ?? ""}`)
+    .join(" & ");
+}
+
+/** Human-readable label for a selector. */
+export function scopeLabel(selector: ScopeSelector): string {
+  return selector.dimensions.length === 0 ? "whole corpus" : scopeKey(selector);
+}
+
+/** True if a case's scope satisfies a selector. */
+export function scopeMatches(scope: ScopeDimensions, selector: ScopeSelector): boolean {
+  return selector.dimensions.every((dimension) => scope[dimension] === selector.values[dimension]);
+}
+
+/** Every dimension name that appears anywhere in a set of scopes, sorted. */
+export function observedDimensions(scopes: readonly ScopeDimensions[]): string[] {
+  const names = new Set<string>();
+  for (const scope of scopes) for (const key of Object.keys(scope)) names.add(key);
+  return [...names].sort();
+}
+
+/**
+ * Derive the selectors a corpus should be reported against.
+ *
+ * The default is **each observed dimension, sliced individually** — one slice per distinct
+ * `host`, one per distinct `voice`, and so on. It deliberately does not default to the full
+ * cross-product: with five dimensions a corpus would shatter into slices of one or two cases
+ * each, and a metric over two cases is noise that reads like evidence.
+ *
+ * A caller that genuinely needs a joint slice — "this evaluator on this host *and* this voice" —
+ * passes the grouping explicitly through `groupings`. That makes combinatorial slicing a
+ * deliberate request rather than something that happens by accident.
+ *
+ * @param scopes every case's scope.
+ * @param groupings dimension groupings to slice by. Defaults to each observed dimension alone.
+ */
+export function deriveScopeSelectors(
+  scopes: readonly ScopeDimensions[],
+  groupings?: readonly (readonly string[])[],
+): ScopeSelector[] {
+  const effective = groupings ?? observedDimensions(scopes).map((dimension) => [dimension]);
+  const selectors = new Map<string, ScopeSelector>();
+
+  for (const grouping of effective) {
+    if (grouping.length === 0) continue;
+    const dimensions = [...grouping].sort();
+    for (const scope of scopes) {
+      // A case that does not declare every dimension in the grouping is not in any slice of it.
+      // Substituting a placeholder would invent a scope the labeller never asserted.
+      if (dimensions.some((dimension) => scope[dimension] === undefined)) continue;
+      const values: Record<string, string> = {};
+      for (const dimension of dimensions) values[dimension] = scope[dimension] as string;
+      const selector: ScopeSelector = { dimensions, values };
+      selectors.set(scopeKey(selector), selector);
+    }
+  }
+
+  return [...selectors.values()].sort((a, b) => scopeKey(a).localeCompare(scopeKey(b)));
+}