@deftai/directive-core 0.70.0 → 0.71.1

package/dist/eval/health.js

@@ -0,0 +1,240 @@
+import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { agentsRefreshPlan } from "../doctor/agents-md.js";
+import { evaluate as evaluateEncoding } from "../encoding/evaluate.js";
+import { readCorePackageVersion } from "../engine-version.js";
+import { resolveEvalPath, resolveProjectDefinitionPath } from "../layout/resolve.js";
+import { readPlanPolicy } from "../policy/plan-extensions.js";
+import { classifyOnboarding, detectPriorState } from "../triage/welcome/prior-state.js";
+import { validateLinks } from "../validate-content/index.js";
+import { evaluateConformance } from "../vbrief-validate/conformance.js";
+import { validateWipCapOnPlan } from "../vbrief-validate/plan-hooks.js";
+import { evaluateContentManifest } from "../verify-source/content-manifest.js";
+export const HEALTH_SCHEMA_VERSION = 1;
+export const HEALTH_HISTORY_REL = "results/health-history.jsonl";
+function sanitizeOneLine(value) {
+    return value.replace(/\r?\n/g, " ");
+}
+function isFrameworkSourceCheckout(projectRoot) {
+    const manifest = resolve(projectRoot, "conventions/content-manifest.json");
+    const cliPkg = resolve(projectRoot, "packages/cli/package.json");
+    return existsSync(manifest) && existsSync(cliPkg);
+}
+function loadPlan(projectRoot) {
+    const filepath = resolveProjectDefinitionPath(projectRoot);
+    if (!existsSync(filepath)) {
+        return null;
+    }
+    try {
+        const data = JSON.parse(readFileSync(filepath, "utf8"));
+        if (data === null || typeof data !== "object" || Array.isArray(data)) {
+            return null;
+        }
+        return { plan: data.plan, filepath };
+    }
+    catch {
+        return null;
+    }
+}
+function probeEncoding(projectRoot) {
+    const result = evaluateEncoding(projectRoot, { mode: "all" });
+    return {
+        id: "encoding",
+        title: "verify:encoding",
+        pass: result.exitCode === 0,
+        exitCode: result.exitCode,
+        detail: result.exitCode === 0 ? undefined : result.message.split("\n")[0],
+    };
+}
+function probeLinks(projectRoot) {
+    const result = validateLinks.evaluate({
+        cwd: projectRoot,
+        strict: false,
+        argv: [],
+        linkCheckStrict: process.env.LINK_CHECK_STRICT === "1",
+    });
+    return {
+        id: "links",
+        title: "verify:links",
+        pass: result.code === 0,
+        exitCode: result.code,
+        detail: result.code === 0 ? undefined : result.message.split("\n")[0],
+    };
+}
+function probeVbriefConformance(projectRoot) {
+    const result = evaluateConformance(projectRoot, { mode: "all" });
+    return {
+        id: "vbrief-conformance",
+        title: "verify:vbrief-conformance",
+        pass: result.exitCode === 0,
+        exitCode: result.exitCode,
+        detail: result.exitCode === 0 ? undefined : result.message.split("\n")[0],
+    };
+}
+function probeAgentsMdFreshness(projectRoot) {
+    const plan = agentsRefreshPlan(projectRoot);
+    const state = String(plan.state ?? "unknown");
+    const pass = state === "current";
+    return {
+        id: "agents-md-freshness",
+        title: "AGENTS.md managed-section freshness",
+        pass,
+        exitCode: pass ? 0 : 1,
+        detail: pass ? undefined : `state=${state}`,
+    };
+}
+function probeContentManifest(projectRoot) {
+    const result = evaluateContentManifest(projectRoot, { root: projectRoot });
+    return {
+        id: "content-manifest",
+        title: "verify:content-manifest",
+        pass: result.code === 0,
+        exitCode: result.code,
+        detail: result.code === 0 ? undefined : result.message.split("\n")[0],
+    };
+}
+/** Absolute path to the versioned health history ledger. */
+export function healthHistoryPath(projectRoot) {
+    return resolveEvalPath(projectRoot, HEALTH_HISTORY_REL);
+}
+/** Detect the canonical wipCap unsatisfiable-nudge contradiction (#1694). */
+export function detectWipCapUnsatisfiableNudge(projectRoot) {
+    const loaded = loadPlan(projectRoot);
+    if (loaded === null) {
+        return null;
+    }
+    const { plan, filepath } = loaded;
+    const policy = readPlanPolicy(plan);
+    const wipCapPresent = typeof policy === "object" &&
+        policy !== null &&
+        !Array.isArray(policy) &&
+        "wipCap" in policy;
+    if (wipCapPresent) {
+        return null;
+    }
+    const state = detectPriorState(projectRoot);
+    const [, missing] = classifyOnboarding(state);
+    if (!missing.includes("wipCap")) {
+        return null;
+    }
+    const validatorErrors = validateWipCapOnPlan(plan, filepath);
+    if (validatorErrors.length > 0) {
+        return null;
+    }
+    return {
+        id: "wipCap-unsatisfiable-nudge",
+        kind: "unsatisfiable-nudge",
+        summary: "Onboarding completeness treats absent plan.policy.wipCap as incomplete, but omit-by-design accepts absence as valid (#1694 / #1186 D1).",
+        signals: [
+            "classifyOnboarding: wipCap listed in missing onboarding signals",
+            "validateWipCapOnPlan: omitted wipCap is valid",
+            "triage:welcome --onboard nudge cannot clear without violating omit-by-design contract",
+        ],
+    };
+}
+/** Run all registered contradictory-gate detectors. */
+export function detectContradictoryGates(projectRoot) {
+    const wipCap = detectWipCapUnsatisfiableNudge(projectRoot);
+    return wipCap === null ? [] : [wipCap];
+}
+/** Compute a 0-100 score from gate pass rate minus contradiction penalty. */
+export function computeHealthScore(gates, contradictions) {
+    const active = gates.filter((g) => !g.skipped);
+    if (active.length === 0) {
+        return contradictions.length > 0 ? 0 : 100;
+    }
+    const passed = active.filter((g) => g.pass).length;
+    const base = Math.round((passed / active.length) * 100);
+    const penalty = contradictions.length * 15;
+    return Math.max(0, base - penalty);
+}
+/** Append one health run to the versioned ledger (#1703 Tier 0). */
+export function persistHealthRun(projectRoot, report) {
+    const path = healthHistoryPath(projectRoot);
+    mkdirSync(dirname(path), { recursive: true });
+    appendFileSync(path, `${JSON.stringify(report)}\n`, "utf8");
+}
+function collectStaticGates(projectRoot, frameworkSource) {
+    const gates = [
+        probeEncoding(projectRoot),
+        probeLinks(projectRoot),
+        probeVbriefConformance(projectRoot),
+        probeAgentsMdFreshness(projectRoot),
+    ];
+    if (frameworkSource) {
+        gates.push(probeContentManifest(projectRoot));
+    }
+    else {
+        gates.push({
+            id: "content-manifest",
+            title: "verify:content-manifest",
+            pass: true,
+            exitCode: 0,
+            skipped: true,
+            skipReason: "framework-source-only gate",
+        });
+    }
+    return gates;
+}
+function formatHumanReport(report) {
+    const lines = [
+        `eval:health v${report.version} score=${report.score}/100 (${report.recordedAt})`,
+        ...report.gates.map((g) => {
+            const status = g.skipped ? "SKIP" : g.pass ? "PASS" : "FAIL";
+            const suffix = g.skipped
+                ? ` (${sanitizeOneLine(g.skipReason ?? "")})`
+                : g.detail
+                    ? ` -- ${sanitizeOneLine(g.detail)}`
+                    : "";
+            return `  [${status}] ${g.title}${suffix}`;
+        }),
+    ];
+    if (report.contradictions.length > 0) {
+        lines.push("  Contradictory gates:");
+        for (const c of report.contradictions) {
+            lines.push(`    - ${c.id}: ${sanitizeOneLine(c.summary)}`);
+        }
+    }
+    return lines.join("\n");
+}
+/** Aggregate Tier-0 static gates into a versioned framework health score (#1703). */
+export function evaluateHealth(options = {}) {
+    const projectRoot = resolve(options.projectRoot ?? process.cwd());
+    const frameworkSource = options.frameworkSource ?? isFrameworkSourceCheckout(projectRoot);
+    const now = options.now ?? (() => new Date());
+    const persist = options.persist ?? true;
+    const gates = collectStaticGates(projectRoot, frameworkSource);
+    const contradictions = detectContradictoryGates(projectRoot);
+    const score = computeHealthScore(gates, contradictions);
+    const report = {
+        schemaVersion: HEALTH_SCHEMA_VERSION,
+        version: readCorePackageVersion(),
+        recordedAt: now()
+            .toISOString()
+            .replace(/\.\d{3}Z$/, "Z"),
+        score,
+        gates,
+        contradictions,
+    };
+    if (persist) {
+        try {
+            persistHealthRun(projectRoot, report);
+        }
+        catch (err) {
+            const persistError = `eval:health: failed to persist health history: ${String(err)}`;
+            const healthy = score === 100 && contradictions.length === 0;
+            return {
+                code: healthy ? 0 : 1,
+                report,
+                message: `${formatHumanReport(report)}\n${persistError}`,
+            };
+        }
+    }
+    const healthy = score === 100 && contradictions.length === 0;
+    return {
+        code: healthy ? 0 : 1,
+        report,
+        message: formatHumanReport(report),
+    };
+}
+//# sourceMappingURL=health.js.map

package/dist/eval/readback.d.ts

@@ -0,0 +1,38 @@
+import { type HealthReport } from "./health.js";
+/** Repeat-suppression window for the budgeted eval session nudge (#1703 / #1279 parity). */
+export declare const EVAL_READBACK_SUPPRESSION_HOURS = 4;
+export declare const EVAL_READBACK_HISTORY_REL: string;
+export declare const EVAL_READBACK_HISTORY_SCHEMA = "deft.eval.readback.v1";
+export interface SessionEvalReadbackResult {
+    readonly line: string | null;
+    readonly suppressed: boolean;
+}
+/** Stable nudge key for debounce — score + contradiction ids. */
+export declare function evalReadbackNudgeKey(report: HealthReport): string;
+/** True when the same eval nudge was read back within the suppression window. */
+export declare function shouldSuppressEvalReadback(nudgeKey: string, historyFile: string, options?: {
+    now?: Date;
+}): boolean;
+/** Whether the current health report warrants a session-start advisory nudge (#2336). */
+export declare function shouldNudgeEvalHealth(report: HealthReport, previous: HealthReport | null): boolean;
+export declare function formatEvalHealthSessionLine(report: HealthReport, previous: HealthReport | null): string;
+export interface RenderSessionEvalReadbackOptions {
+    readonly now?: Date;
+    readonly maxChars?: number;
+    readonly writeHistory?: boolean;
+    readonly evaluate?: (projectRoot: string) => {
+        report: HealthReport | null;
+    };
+}
+/** Budgeted session eval one-liner — silent when healthy or debounced (#1703 / #2336). */
+export declare function renderSessionEvalReadback(projectRoot: string, options?: RenderSessionEvalReadbackOptions): SessionEvalReadbackResult;
+/** Emit the eval session readback line when present. Returns the line or null. */
+export declare function emitSessionEvalReadback(projectRoot: string, options?: {
+    output?: (line: string) => void;
+    now?: Date;
+    writeHistory?: boolean;
+    evaluate?: (projectRoot: string) => {
+        report: HealthReport | null;
+    };
+}): string | null;
+//# sourceMappingURL=readback.d.ts.map

package/dist/eval/readback.js

@@ -0,0 +1,209 @@
+import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { MAX_LINE_CHARS } from "../triage/welcome/constants.js";
+import { evaluateHealth, healthHistoryPath } from "./health.js";
+/** Repeat-suppression window for the budgeted eval session nudge (#1703 / #1279 parity). */
+export const EVAL_READBACK_SUPPRESSION_HOURS = 4;
+export const EVAL_READBACK_HISTORY_REL = join(".deft-cache", "eval-readback-history.jsonl");
+export const EVAL_READBACK_HISTORY_SCHEMA = "deft.eval.readback.v1";
+function truncate(text, maxChars) {
+    if (text.length <= maxChars) {
+        return text;
+    }
+    if (maxChars <= 3) {
+        return text.slice(0, maxChars);
+    }
+    return `${text.slice(0, maxChars - 3)}...`;
+}
+function parseJsonObjectOrNull(text) {
+    try {
+        const parsed = JSON.parse(text);
+        if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) {
+            return parsed;
+        }
+    }
+    catch {
+        // malformed history line
+    }
+    return null;
+}
+function readHealthHistoryRows(projectRoot) {
+    const path = healthHistoryPath(projectRoot);
+    if (!existsSync(path)) {
+        return [];
+    }
+    let text;
+    try {
+        text = readFileSync(path, "utf8");
+    }
+    catch {
+        return [];
+    }
+    const rows = [];
+    for (const line of text.split("\n")) {
+        const stripped = line.trim();
+        if (stripped.length === 0) {
+            continue;
+        }
+        try {
+            rows.push(JSON.parse(stripped));
+        }
+        catch {
+            // skip malformed ledger rows
+        }
+    }
+    return rows;
+}
+function readbackHistoryPath(projectRoot) {
+    return resolve(projectRoot, EVAL_READBACK_HISTORY_REL);
+}
+function readLastReadbackRecord(path) {
+    if (!existsSync(path)) {
+        return null;
+    }
+    let text;
+    try {
+        text = readFileSync(path, "utf8");
+    }
+    catch {
+        return null;
+    }
+    const lines = text.split("\n").filter((line) => line.trim().length > 0);
+    if (lines.length === 0) {
+        return null;
+    }
+    const last = lines[lines.length - 1];
+    if (last === undefined) {
+        return null;
+    }
+    return parseJsonObjectOrNull(last);
+}
+function parseHistoryEmittedAt(record) {
+    const raw = record.emitted_at;
+    if (typeof raw !== "string" || raw.trim().length === 0) {
+        return null;
+    }
+    let candidate = raw.trim();
+    if (candidate.endsWith("Z")) {
+        candidate = `${candidate.slice(0, -1)}+00:00`;
+    }
+    const parsed = new Date(candidate);
+    return Number.isNaN(parsed.getTime()) ? null : parsed;
+}
+/** Stable nudge key for debounce — score + contradiction ids. */
+export function evalReadbackNudgeKey(report) {
+    const contradictionIds = report.contradictions
+        .map((c) => c.id)
+        .sort()
+        .join(",");
+    return `eval-health:${report.score}:${contradictionIds}`;
+}
+/** True when the same eval nudge was read back within the suppression window. */
+export function shouldSuppressEvalReadback(nudgeKey, historyFile, options = {}) {
+    const prior = readLastReadbackRecord(historyFile);
+    if (prior === null) {
+        return false;
+    }
+    const emittedAt = parseHistoryEmittedAt(prior);
+    if (emittedAt === null) {
+        return false;
+    }
+    const now = options.now ?? new Date();
+    const ageMs = now.getTime() - emittedAt.getTime();
+    if (ageMs < 0 || ageMs >= EVAL_READBACK_SUPPRESSION_HOURS * 3_600_000) {
+        return false;
+    }
+    return prior.nudge_key === nudgeKey;
+}
+function appendEvalReadbackHistory(projectRoot, nudgeKey, line, options = {}) {
+    const path = readbackHistoryPath(projectRoot);
+    const record = {
+        schema: EVAL_READBACK_HISTORY_SCHEMA,
+        emitted_at: (options.now ?? new Date()).toISOString().replace(/\.\d{3}Z$/, "Z"),
+        nudge_key: nudgeKey,
+        line,
+    };
+    try {
+        mkdirSync(join(path, ".."), { recursive: true });
+        appendFileSync(path, `${JSON.stringify(record)}\n`, "utf8");
+    }
+    catch {
+        // observability only
+    }
+}
+function failedGateCount(report) {
+    return report.gates.filter((g) => !g.skipped && !g.pass).length;
+}
+/** Whether the current health report warrants a session-start advisory nudge (#2336). */
+export function shouldNudgeEvalHealth(report, previous) {
+    if (report.contradictions.length > 0) {
+        return true;
+    }
+    if (previous !== null && report.score < previous.score) {
+        return true;
+    }
+    if (previous === null && report.score < 100 && failedGateCount(report) > 0) {
+        return true;
+    }
+    return false;
+}
+export function formatEvalHealthSessionLine(report, previous) {
+    if (report.contradictions.length > 0) {
+        const first = report.contradictions[0];
+        const summary = first?.summary ?? "contradictory gate detected";
+        return (`[eval] Framework health ${report.score}/100 — contradictory gate ${first?.id ?? "unknown"}: ` +
+            `${summary.replace(/\r?\n/g, " ")} — run \`task eval:health\` for detail.`);
+    }
+    if (previous !== null && report.score < previous.score) {
+        return (`[eval] Framework health dropped ${previous.score}->${report.score}/100 — ` +
+            "run `task eval:health` for gate detail.");
+    }
+    const failed = failedGateCount(report);
+    return (`[eval] Framework health ${report.score}/100 (${failed} gate${failed === 1 ? "" : "s"} failing) — ` +
+        "run `task eval:health` for detail.");
+}
+/** Budgeted session eval one-liner — silent when healthy or debounced (#1703 / #2336). */
+export function renderSessionEvalReadback(projectRoot, options = {}) {
+    try {
+        const root = resolve(projectRoot);
+        const priorRows = readHealthHistoryRows(root);
+        const previous = priorRows.length > 0 ? (priorRows[priorRows.length - 1] ?? null) : null;
+        const evaluateFn = options.evaluate ??
+            ((cwd) => {
+                const result = evaluateHealth({ projectRoot: cwd, persist: true });
+                return { report: result.report };
+            });
+        const { report } = evaluateFn(root);
+        if (report === null) {
+            return { line: null, suppressed: false };
+        }
+        if (!shouldNudgeEvalHealth(report, previous)) {
+            return { line: null, suppressed: false };
+        }
+        const nudgeKey = evalReadbackNudgeKey(report);
+        const hist = readbackHistoryPath(root);
+        if (shouldSuppressEvalReadback(nudgeKey, hist, { now: options.now })) {
+            return { line: null, suppressed: true };
+        }
+        const maxChars = options.maxChars ?? MAX_LINE_CHARS;
+        const line = truncate(formatEvalHealthSessionLine(report, previous), maxChars);
+        if (options.writeHistory !== false) {
+            appendEvalReadbackHistory(root, nudgeKey, line, { now: options.now });
+        }
+        return { line, suppressed: false };
+    }
+    catch {
+        return { line: null, suppressed: false };
+    }
+}
+/** Emit the eval session readback line when present. Returns the line or null. */
+export function emitSessionEvalReadback(projectRoot, options = {}) {
+    const result = renderSessionEvalReadback(projectRoot, options);
+    if (result.line === null) {
+        return null;
+    }
+    const out = options.output ?? ((line) => process.stdout.write(`${line}\n`));
+    out(result.line);
+    return result.line;
+}
+//# sourceMappingURL=readback.js.map

package/dist/eval/report.d.ts

@@ -0,0 +1,53 @@
+import { type GoldenRunRecord } from "./run.js";
+export declare const REPORT_SCHEMA_VERSION: 1;
+/** Metric delta with a simple two-proportion significance test (#896). */
+export interface MetricDelta {
+    readonly metric: string;
+    readonly champion: number;
+    readonly challenger: number;
+    readonly delta: number;
+    readonly zScore: number | null;
+    readonly pValue: number | null;
+    readonly significantAt05: boolean;
+}
+/** Holdout tripwire when primary gains do not generalize (#1703 Goodhart guard). */
+export interface HoldoutTripwire {
+    readonly triggered: boolean;
+    readonly summary: string;
+    readonly primaryDelta: number;
+    readonly holdoutDelta: number;
+}
+/** Version-diff report between champion and challenger golden runs. */
+export interface GoldenEvalReport {
+    readonly schemaVersion: typeof REPORT_SCHEMA_VERSION;
+    readonly championVersion: string;
+    readonly challengerVersion: string;
+    readonly model: string;
+    readonly championRunId: string;
+    readonly challengerRunId: string;
+    readonly deltas: readonly MetricDelta[];
+    readonly holdoutTripwire: HoldoutTripwire;
+}
+export interface ReportGoldenEvalOptions {
+    readonly projectRoot?: string;
+    readonly championVersion: string;
+    readonly challengerVersion: string;
+    readonly model: string;
+}
+export interface ReportGoldenEvalResult {
+    readonly code: 0 | 1 | 2;
+    readonly report: GoldenEvalReport | null;
+    readonly message: string;
+}
+/** Pick the latest run for a directive version and model. */
+export declare function findLatestGoldenRun(records: readonly GoldenRunRecord[], directiveVersion: string, model: string): GoldenRunRecord | null;
+/** Two-proportion z-test (normal approximation). */
+export declare function twoProportionZTest(passedA: number, totalA: number, passedB: number, totalB: number): {
+    zScore: number | null;
+    pValue: number | null;
+};
+/** Detect holdout tripwire: primary improves while holdout regresses (#1703). */
+export declare function evaluateHoldoutTripwire(champion: GoldenRunRecord, challenger: GoldenRunRecord): HoldoutTripwire;
+/** Diff two directive versions with metric deltas and significance (#1703 Tier 2). */
+export declare function reportGoldenEval(options: ReportGoldenEvalOptions): ReportGoldenEvalResult;
+//# sourceMappingURL=report.d.ts.map