@deftai/directive-core 0.69.0 → 0.71.0

package/dist/eval/readback.js

@@ -0,0 +1,209 @@
+import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { MAX_LINE_CHARS } from "../triage/welcome/constants.js";
+import { evaluateHealth, healthHistoryPath } from "./health.js";
+/** Repeat-suppression window for the budgeted eval session nudge (#1703 / #1279 parity). */
+export const EVAL_READBACK_SUPPRESSION_HOURS = 4;
+export const EVAL_READBACK_HISTORY_REL = join(".deft-cache", "eval-readback-history.jsonl");
+export const EVAL_READBACK_HISTORY_SCHEMA = "deft.eval.readback.v1";
+function truncate(text, maxChars) {
+    if (text.length <= maxChars) {
+        return text;
+    }
+    if (maxChars <= 3) {
+        return text.slice(0, maxChars);
+    }
+    return `${text.slice(0, maxChars - 3)}...`;
+}
+function parseJsonObjectOrNull(text) {
+    try {
+        const parsed = JSON.parse(text);
+        if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) {
+            return parsed;
+        }
+    }
+    catch {
+        // malformed history line
+    }
+    return null;
+}
+function readHealthHistoryRows(projectRoot) {
+    const path = healthHistoryPath(projectRoot);
+    if (!existsSync(path)) {
+        return [];
+    }
+    let text;
+    try {
+        text = readFileSync(path, "utf8");
+    }
+    catch {
+        return [];
+    }
+    const rows = [];
+    for (const line of text.split("\n")) {
+        const stripped = line.trim();
+        if (stripped.length === 0) {
+            continue;
+        }
+        try {
+            rows.push(JSON.parse(stripped));
+        }
+        catch {
+            // skip malformed ledger rows
+        }
+    }
+    return rows;
+}
+function readbackHistoryPath(projectRoot) {
+    return resolve(projectRoot, EVAL_READBACK_HISTORY_REL);
+}
+function readLastReadbackRecord(path) {
+    if (!existsSync(path)) {
+        return null;
+    }
+    let text;
+    try {
+        text = readFileSync(path, "utf8");
+    }
+    catch {
+        return null;
+    }
+    const lines = text.split("\n").filter((line) => line.trim().length > 0);
+    if (lines.length === 0) {
+        return null;
+    }
+    const last = lines[lines.length - 1];
+    if (last === undefined) {
+        return null;
+    }
+    return parseJsonObjectOrNull(last);
+}
+function parseHistoryEmittedAt(record) {
+    const raw = record.emitted_at;
+    if (typeof raw !== "string" || raw.trim().length === 0) {
+        return null;
+    }
+    let candidate = raw.trim();
+    if (candidate.endsWith("Z")) {
+        candidate = `${candidate.slice(0, -1)}+00:00`;
+    }
+    const parsed = new Date(candidate);
+    return Number.isNaN(parsed.getTime()) ? null : parsed;
+}
+/** Stable nudge key for debounce — score + contradiction ids. */
+export function evalReadbackNudgeKey(report) {
+    const contradictionIds = report.contradictions
+        .map((c) => c.id)
+        .sort()
+        .join(",");
+    return `eval-health:${report.score}:${contradictionIds}`;
+}
+/** True when the same eval nudge was read back within the suppression window. */
+export function shouldSuppressEvalReadback(nudgeKey, historyFile, options = {}) {
+    const prior = readLastReadbackRecord(historyFile);
+    if (prior === null) {
+        return false;
+    }
+    const emittedAt = parseHistoryEmittedAt(prior);
+    if (emittedAt === null) {
+        return false;
+    }
+    const now = options.now ?? new Date();
+    const ageMs = now.getTime() - emittedAt.getTime();
+    if (ageMs < 0 || ageMs >= EVAL_READBACK_SUPPRESSION_HOURS * 3_600_000) {
+        return false;
+    }
+    return prior.nudge_key === nudgeKey;
+}
+function appendEvalReadbackHistory(projectRoot, nudgeKey, line, options = {}) {
+    const path = readbackHistoryPath(projectRoot);
+    const record = {
+        schema: EVAL_READBACK_HISTORY_SCHEMA,
+        emitted_at: (options.now ?? new Date()).toISOString().replace(/\.\d{3}Z$/, "Z"),
+        nudge_key: nudgeKey,
+        line,
+    };
+    try {
+        mkdirSync(join(path, ".."), { recursive: true });
+        appendFileSync(path, `${JSON.stringify(record)}\n`, "utf8");
+    }
+    catch {
+        // observability only
+    }
+}
+function failedGateCount(report) {
+    return report.gates.filter((g) => !g.skipped && !g.pass).length;
+}
+/** Whether the current health report warrants a session-start advisory nudge (#2336). */
+export function shouldNudgeEvalHealth(report, previous) {
+    if (report.contradictions.length > 0) {
+        return true;
+    }
+    if (previous !== null && report.score < previous.score) {
+        return true;
+    }
+    if (previous === null && report.score < 100 && failedGateCount(report) > 0) {
+        return true;
+    }
+    return false;
+}
+export function formatEvalHealthSessionLine(report, previous) {
+    if (report.contradictions.length > 0) {
+        const first = report.contradictions[0];
+        const summary = first?.summary ?? "contradictory gate detected";
+        return (`[eval] Framework health ${report.score}/100 — contradictory gate ${first?.id ?? "unknown"}: ` +
+            `${summary.replace(/\r?\n/g, " ")} — run \`task eval:health\` for detail.`);
+    }
+    if (previous !== null && report.score < previous.score) {
+        return (`[eval] Framework health dropped ${previous.score}->${report.score}/100 — ` +
+            "run `task eval:health` for gate detail.");
+    }
+    const failed = failedGateCount(report);
+    return (`[eval] Framework health ${report.score}/100 (${failed} gate${failed === 1 ? "" : "s"} failing) — ` +
+        "run `task eval:health` for detail.");
+}
+/** Budgeted session eval one-liner — silent when healthy or debounced (#1703 / #2336). */
+export function renderSessionEvalReadback(projectRoot, options = {}) {
+    try {
+        const root = resolve(projectRoot);
+        const priorRows = readHealthHistoryRows(root);
+        const previous = priorRows.length > 0 ? (priorRows[priorRows.length - 1] ?? null) : null;
+        const evaluateFn = options.evaluate ??
+            ((cwd) => {
+                const result = evaluateHealth({ projectRoot: cwd, persist: true });
+                return { report: result.report };
+            });
+        const { report } = evaluateFn(root);
+        if (report === null) {
+            return { line: null, suppressed: false };
+        }
+        if (!shouldNudgeEvalHealth(report, previous)) {
+            return { line: null, suppressed: false };
+        }
+        const nudgeKey = evalReadbackNudgeKey(report);
+        const hist = readbackHistoryPath(root);
+        if (shouldSuppressEvalReadback(nudgeKey, hist, { now: options.now })) {
+            return { line: null, suppressed: true };
+        }
+        const maxChars = options.maxChars ?? MAX_LINE_CHARS;
+        const line = truncate(formatEvalHealthSessionLine(report, previous), maxChars);
+        if (options.writeHistory !== false) {
+            appendEvalReadbackHistory(root, nudgeKey, line, { now: options.now });
+        }
+        return { line, suppressed: false };
+    }
+    catch {
+        return { line: null, suppressed: false };
+    }
+}
+/** Emit the eval session readback line when present. Returns the line or null. */
+export function emitSessionEvalReadback(projectRoot, options = {}) {
+    const result = renderSessionEvalReadback(projectRoot, options);
+    if (result.line === null) {
+        return null;
+    }
+    const out = options.output ?? ((line) => process.stdout.write(`${line}\n`));
+    out(result.line);
+    return result.line;
+}
+//# sourceMappingURL=readback.js.map

package/dist/eval/report.d.ts

@@ -0,0 +1,53 @@
+import { type GoldenRunRecord } from "./run.js";
+export declare const REPORT_SCHEMA_VERSION: 1;
+/** Metric delta with a simple two-proportion significance test (#896). */
+export interface MetricDelta {
+    readonly metric: string;
+    readonly champion: number;
+    readonly challenger: number;
+    readonly delta: number;
+    readonly zScore: number | null;
+    readonly pValue: number | null;
+    readonly significantAt05: boolean;
+}
+/** Holdout tripwire when primary gains do not generalize (#1703 Goodhart guard). */
+export interface HoldoutTripwire {
+    readonly triggered: boolean;
+    readonly summary: string;
+    readonly primaryDelta: number;
+    readonly holdoutDelta: number;
+}
+/** Version-diff report between champion and challenger golden runs. */
+export interface GoldenEvalReport {
+    readonly schemaVersion: typeof REPORT_SCHEMA_VERSION;
+    readonly championVersion: string;
+    readonly challengerVersion: string;
+    readonly model: string;
+    readonly championRunId: string;
+    readonly challengerRunId: string;
+    readonly deltas: readonly MetricDelta[];
+    readonly holdoutTripwire: HoldoutTripwire;
+}
+export interface ReportGoldenEvalOptions {
+    readonly projectRoot?: string;
+    readonly championVersion: string;
+    readonly challengerVersion: string;
+    readonly model: string;
+}
+export interface ReportGoldenEvalResult {
+    readonly code: 0 | 1 | 2;
+    readonly report: GoldenEvalReport | null;
+    readonly message: string;
+}
+/** Pick the latest run for a directive version and model. */
+export declare function findLatestGoldenRun(records: readonly GoldenRunRecord[], directiveVersion: string, model: string): GoldenRunRecord | null;
+/** Two-proportion z-test (normal approximation). */
+export declare function twoProportionZTest(passedA: number, totalA: number, passedB: number, totalB: number): {
+    zScore: number | null;
+    pValue: number | null;
+};
+/** Detect holdout tripwire: primary improves while holdout regresses (#1703). */
+export declare function evaluateHoldoutTripwire(champion: GoldenRunRecord, challenger: GoldenRunRecord): HoldoutTripwire;
+/** Diff two directive versions with metric deltas and significance (#1703 Tier 2). */
+export declare function reportGoldenEval(options: ReportGoldenEvalOptions): ReportGoldenEvalResult;
+//# sourceMappingURL=report.d.ts.map

package/dist/eval/report.js

@@ -0,0 +1,161 @@
+import { existsSync, readFileSync } from "node:fs";
+import { resolveEvalPath } from "../layout/resolve.js";
+import { GOLDEN_RUNS_HISTORY_REL } from "./run.js";
+export const REPORT_SCHEMA_VERSION = 1;
+function readGoldenRuns(projectRoot) {
+    const path = resolveEvalPath(projectRoot, GOLDEN_RUNS_HISTORY_REL);
+    if (!existsSync(path)) {
+        return [];
+    }
+    const lines = readFileSync(path, "utf8")
+        .split("\n")
+        .filter((line) => line.trim().length > 0);
+    const records = [];
+    for (const line of lines) {
+        try {
+            records.push(JSON.parse(line));
+        }
+        catch {
+            // Skip malformed ledger rows.
+        }
+    }
+    return records;
+}
+/** Pick the latest run for a directive version and model. */
+export function findLatestGoldenRun(records, directiveVersion, model) {
+    const matches = records.filter((r) => r.directiveVersion === directiveVersion && r.model === model);
+    if (matches.length === 0) {
+        return null;
+    }
+    return matches.reduce((latest, current) => current.recordedAt >= latest.recordedAt ? current : latest);
+}
+function countPasses(results) {
+    return {
+        passed: results.filter((r) => r.passed).length,
+        total: results.length,
+    };
+}
+/** Two-proportion z-test (normal approximation). */
+export function twoProportionZTest(passedA, totalA, passedB, totalB) {
+    if (totalA === 0 || totalB === 0) {
+        return { zScore: null, pValue: null };
+    }
+    const p1 = passedA / totalA;
+    const p2 = passedB / totalB;
+    const pooled = (passedA + passedB) / (totalA + totalB);
+    const se = Math.sqrt(pooled * (1 - pooled) * (1 / totalA + 1 / totalB));
+    if (se === 0) {
+        return { zScore: null, pValue: null };
+    }
+    const z = (p2 - p1) / se;
+    const pValue = 2 * (1 - normalCdf(Math.abs(z)));
+    return { zScore: z, pValue };
+}
+function normalCdf(x) {
+    return 0.5 * (1 + erf(x / Math.SQRT2));
+}
+function erf(x) {
+    const sign = x < 0 ? -1 : 1;
+    const ax = Math.abs(x);
+    const t = 1 / (1 + 0.3275911 * ax);
+    const y = 1 -
+        ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) *
+            t *
+            Math.exp(-ax * ax);
+    return sign * y;
+}
+function metricDelta(metric, championRate, challengerRate, championPassed, championTotal, challengerPassed, challengerTotal) {
+    const { zScore, pValue } = twoProportionZTest(championPassed, championTotal, challengerPassed, challengerTotal);
+    return {
+        metric,
+        champion: championRate,
+        challenger: challengerRate,
+        delta: challengerRate - championRate,
+        zScore,
+        pValue,
+        significantAt05: pValue !== null && pValue < 0.05,
+    };
+}
+/** Detect holdout tripwire: primary improves while holdout regresses (#1703). */
+export function evaluateHoldoutTripwire(champion, challenger) {
+    const primaryDelta = challenger.summary.primaryPassRate - champion.summary.primaryPassRate;
+    const holdoutDelta = challenger.summary.holdoutPassRate - champion.summary.holdoutPassRate;
+    const triggered = primaryDelta > 0.05 && holdoutDelta < -0.05;
+    const summary = triggered
+        ? "Holdout tripwire: challenger improved primary metrics but holdout regressed -- possible tuning to gated corpus."
+        : "Holdout tripwire clear.";
+    return { triggered, summary, primaryDelta, holdoutDelta };
+}
+function formatPercent(value) {
+    return `${(value * 100).toFixed(1)}%`;
+}
+/** Diff two directive versions with metric deltas and significance (#1703 Tier 2). */
+export function reportGoldenEval(options) {
+    if (!options.championVersion.trim() || !options.challengerVersion.trim()) {
+        return {
+            code: 2,
+            report: null,
+            message: "eval:report: --champion and --challenger are required",
+        };
+    }
+    if (!options.model.trim()) {
+        return { code: 2, report: null, message: "eval:report: --model is required" };
+    }
+    const projectRoot = options.projectRoot ?? process.cwd();
+    const records = readGoldenRuns(projectRoot);
+    const champion = findLatestGoldenRun(records, options.championVersion, options.model);
+    const challenger = findLatestGoldenRun(records, options.challengerVersion, options.model);
+    if (champion === null) {
+        return {
+            code: 1,
+            report: null,
+            message: `eval:report: no golden run found for champion v${options.championVersion} model=${options.model}`,
+        };
+    }
+    if (challenger === null) {
+        return {
+            code: 1,
+            report: null,
+            message: `eval:report: no golden run found for challenger v${options.challengerVersion} model=${options.model}`,
+        };
+    }
+    const championPrimary = champion.results.filter((r) => !r.holdout);
+    const challengerPrimary = challenger.results.filter((r) => !r.holdout);
+    const championHoldout = champion.results.filter((r) => r.holdout);
+    const challengerHoldout = challenger.results.filter((r) => r.holdout);
+    const championPrimaryCount = countPasses(championPrimary);
+    const challengerPrimaryCount = countPasses(challengerPrimary);
+    const championHoldoutCount = countPasses(championHoldout);
+    const challengerHoldoutCount = countPasses(challengerHoldout);
+    const deltas = [
+        metricDelta("primaryPassRate", champion.summary.primaryPassRate, challenger.summary.primaryPassRate, championPrimaryCount.passed, championPrimaryCount.total, challengerPrimaryCount.passed, challengerPrimaryCount.total),
+        metricDelta("holdoutPassRate", champion.summary.holdoutPassRate, challenger.summary.holdoutPassRate, championHoldoutCount.passed, championHoldoutCount.total, challengerHoldoutCount.passed, challengerHoldoutCount.total),
+        metricDelta("overallPassRate", champion.summary.passRate, challenger.summary.passRate, countPasses(champion.results).passed, countPasses(champion.results).total, countPasses(challenger.results).passed, countPasses(challenger.results).total),
+    ];
+    const holdoutTripwire = evaluateHoldoutTripwire(champion, challenger);
+    const report = {
+        schemaVersion: REPORT_SCHEMA_VERSION,
+        championVersion: options.championVersion,
+        challengerVersion: options.challengerVersion,
+        model: options.model,
+        championRunId: champion.runId,
+        challengerRunId: challenger.runId,
+        deltas,
+        holdoutTripwire,
+    };
+    const lines = [
+        `eval:report champion=v${options.championVersion} challenger=v${options.challengerVersion} model=${options.model}`,
+        ...deltas.map((d) => {
+            const sig = d.pValue === null
+                ? "n/a"
+                : d.significantAt05
+                    ? `p=${d.pValue.toFixed(4)} *`
+                    : `p=${d.pValue.toFixed(4)}`;
+            return `  ${d.metric}: ${formatPercent(d.champion)} -> ${formatPercent(d.challenger)} (delta ${(d.delta * 100).toFixed(1)}pp, ${sig})`;
+        }),
+        `  ${holdoutTripwire.summary}`,
+    ];
+    const code = holdoutTripwire.triggered ? 1 : 0;
+    return { code, report, message: lines.join("\n") };
+}
+//# sourceMappingURL=report.js.map

package/dist/eval/run.d.ts

@@ -0,0 +1,79 @@
+export declare const GOLDEN_RUNS_HISTORY_REL = "results/golden-runs.jsonl";
+export declare const GOLDEN_RUN_SCHEMA_VERSION: 1;
+export declare const GOLDEN_CORPUS_VERSION = "2026-07-05-tier2-fixture-v1";
+/** One synthetic golden-task definition with an objective grader. */
+export interface GoldenTaskDefinition {
+    readonly id: string;
+    readonly title: string;
+    readonly holdout: boolean;
+    readonly grade: (context: GoldenTaskContext) => GoldenTaskGrade;
+}
+/** Runtime context passed to golden-task graders. */
+export interface GoldenTaskContext {
+    readonly tempDir: string;
+    readonly seed: number;
+    readonly directiveVersion: string;
+    readonly model: string;
+}
+/** Objective grader output for a single task × seed. */
+export interface GoldenTaskGrade {
+    readonly passed: boolean;
+    readonly metrics: Readonly<Record<string, number | boolean>>;
+}
+/** One graded task outcome within a golden run. */
+export interface GoldenTaskResult {
+    readonly taskId: string;
+    readonly seed: number;
+    readonly passed: boolean;
+    readonly holdout: boolean;
+    readonly metrics: Readonly<Record<string, number | boolean>>;
+}
+/** Versioned golden-run record persisted to the `.eval` ledger (#1703 Tier 2). */
+export interface GoldenRunRecord {
+    readonly schemaVersion: typeof GOLDEN_RUN_SCHEMA_VERSION;
+    readonly runId: string;
+    readonly directiveVersion: string;
+    readonly model: string;
+    readonly harness: string;
+    readonly seeds: readonly number[];
+    readonly corpusVersion: string;
+    readonly recordedAt: string;
+    readonly results: readonly GoldenTaskResult[];
+    readonly summary: GoldenRunSummary;
+}
+/** Aggregate metrics for one golden run. */
+export interface GoldenRunSummary {
+    readonly primaryPassRate: number;
+    readonly holdoutPassRate: number;
+    readonly passRate: number;
+    readonly primaryTotal: number;
+    readonly holdoutTotal: number;
+}
+export interface RunGoldenEvalOptions {
+    readonly projectRoot?: string;
+    readonly model: string;
+    readonly seeds?: readonly number[];
+    readonly directiveVersion?: string;
+    readonly harness?: string;
+    readonly persist?: boolean;
+    readonly now?: () => Date;
+    readonly mkTempDir?: () => string;
+}
+export interface RunGoldenEvalResult {
+    readonly code: 0 | 1 | 2;
+    readonly record: GoldenRunRecord | null;
+    readonly message: string;
+}
+/** Fixed golden corpus with objective graders (#1703 Tier 2). */
+export declare const GOLDEN_CORPUS: readonly GoldenTaskDefinition[];
+/** Absolute path to the golden-run results ledger. */
+export declare function goldenRunsHistoryPath(projectRoot: string): string;
+/** Append one golden run to the versioned ledger (#1703 Tier 2). */
+export declare function persistGoldenRun(projectRoot: string, record: GoldenRunRecord): void;
+/** Stable hash for rotating holdout selection (#1703 Goodhart mitigation). */
+export declare function holdoutRotationIndex(directiveVersion: string, model: string, holdoutCount: number): number;
+/** Select the active holdout task for this version × model tuple. */
+export declare function selectRotatingHoldoutTask(directiveVersion: string, model: string): GoldenTaskDefinition | null;
+/** Execute the golden corpus for one model × seed set and optionally persist (#1703 Tier 2). */
+export declare function runGoldenEval(options: RunGoldenEvalOptions): RunGoldenEvalResult;
+//# sourceMappingURL=run.d.ts.map