@deftai/directive-core 0.98.0 → 0.99.0

package/dist/eval/run.d.ts

@@ -1,3 +1,4 @@
+import { type FrameworkVersionPin } from "./version-pin.js";
 export declare const GOLDEN_RUNS_HISTORY_REL = "results/golden-runs.jsonl";
 export declare const GOLDEN_RUN_SCHEMA_VERSION: 1;
 export declare const GOLDEN_CORPUS_VERSION = "2026-07-05-tier2-fixture-v1";
@@ -32,7 +33,13 @@ export interface GoldenTaskResult {
 export interface GoldenRunRecord {
     readonly schemaVersion: typeof GOLDEN_RUN_SCHEMA_VERSION;
     readonly runId: string;
+    /** Framework version pin for this run (alias of frameworkVersionPin.frameworkVersion). */
     readonly directiveVersion: string;
+    /**
+     * Resolved framework version pin captured at run start (#3215).
+     * Optional for ledger rows written before the purity gate shipped.
+     */
+    readonly frameworkVersionPin?: FrameworkVersionPin;
     readonly model: string;
     readonly harness: string;
     readonly seeds: readonly number[];
@@ -63,6 +70,8 @@ export interface RunGoldenEvalResult {
     readonly code: 0 | 1 | 2;
     readonly record: GoldenRunRecord | null;
     readonly message: string;
+    /** #1584 shared-benchmark manifest after version pin wire, when present on disk. */
+    readonly sharedBenchmarkManifest?: Record<string, unknown> | null;
 }
 /** Fixed golden corpus with objective graders (#1703 Tier 2). */
 export declare const GOLDEN_CORPUS: readonly GoldenTaskDefinition[];

package/dist/eval/run.js

@@ -2,11 +2,11 @@ import { createHash } from "node:crypto";
 import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join, resolve } from "node:path";
-import { readCorePackageVersion } from "../engine-version.js";
 import { containedWrite } from "../fs/contained-write.js";
 import { resolveEvalPath } from "../layout/resolve.js";
 import { BYTE_DIFF_WHOLE_FILE_THRESHOLD, InstrumentedVbriefCrud } from "./crud-telemetry.js";
 import { evaluateHealth } from "./health.js";
+import { applyVersionPinToSharedBenchmark, resolveFrameworkVersionPin, } from "./version-pin.js";
 export const GOLDEN_RUNS_HISTORY_REL = "results/golden-runs.jsonl";
 export const GOLDEN_RUN_SCHEMA_VERSION = 1;
 export const GOLDEN_CORPUS_VERSION = "2026-07-05-tier2-fixture-v1";
@@ -250,10 +250,15 @@ export function runGoldenEval(options) {
         return { code: 2, record: null, message: "eval:run: --model is required" };
     }
     const projectRoot = options.projectRoot ?? process.cwd();
-    const directiveVersion = options.directiveVersion ?? readCorePackageVersion();
+    const now = options.now ?? (() => new Date());
+    // #3215: resolve framework version once at run start (package.json pin or override).
+    const frameworkVersionPin = resolveFrameworkVersionPin({
+        override: options.directiveVersion,
+        now,
+    });
+    const directiveVersion = frameworkVersionPin.frameworkVersion;
     const harness = options.harness ?? "deterministic-fixture";
     const seeds = options.seeds ?? [1, 2, 3];
-    const now = options.now ?? (() => new Date());
     const persist = options.persist ?? true;
     if (seeds.length === 0) {
         return { code: 2, record: null, message: "eval:run: at least one seed is required" };
@@ -293,6 +298,7 @@ export function runGoldenEval(options) {
         schemaVersion: GOLDEN_RUN_SCHEMA_VERSION,
         runId: runIdFor(directiveVersion, options.model, harness, seeds),
         directiveVersion,
+        frameworkVersionPin,
         model: options.model,
         harness,
         seeds,
@@ -303,6 +309,8 @@ export function runGoldenEval(options) {
         results,
         summary,
     };
+    // Persist ledger first so a failed golden-run write never leaves a dangling
+    // shared-benchmark pin without a matching run row (#3215 Greptile).
     if (persist) {
         try {
             persistGoldenRun(projectRoot, record);
@@ -312,16 +320,44 @@ export function runGoldenEval(options) {
                 code: 2,
                 record,
                 message: `eval:run: failed to persist golden run: ${String(err)}`,
+                sharedBenchmarkManifest: null,
             };
         }
     }
+    // #1584 / #3215: wire (+ persist when ledger was written) shared-benchmark pin.
+    let sharedBenchmark;
+    try {
+        sharedBenchmark = applyVersionPinToSharedBenchmark(projectRoot, frameworkVersionPin, {
+            persist,
+        });
+    }
+    catch (err) {
+        return {
+            code: 2,
+            record,
+            message: `eval:run: failed to wire shared-benchmark pin: ${String(err)}`,
+            sharedBenchmarkManifest: null,
+        };
+    }
     const lines = [
         `eval:run v${record.directiveVersion} model=${record.model} harness=${record.harness} seeds=[${record.seeds.join(",")}]`,
+        `  framework version pin: ${frameworkVersionPin.frameworkVersion} (source=${frameworkVersionPin.source})`,
         `  primary pass rate: ${(summary.primaryPassRate * 100).toFixed(1)}% (${summary.primaryTotal} trials)`,
         `  holdout pass rate: ${(summary.holdoutPassRate * 100).toFixed(1)}% (${summary.holdoutTotal} trials)`,
         `  rotating holdout task: ${rotatingHoldout?.id ?? "none"}`,
         `  runId=${record.runId}`,
     ];
-    return { code: 0, record, message: lines.join("\n") };
+    if (sharedBenchmark.persisted) {
+        lines.push("  shared-benchmark manifest: frameworkVersion pin persisted (#1584 / #3215)");
+    }
+    else if (sharedBenchmark.applied) {
+        lines.push("  shared-benchmark manifest: frameworkVersion wired in-memory (#1584 / #3215)");
+    }
+    return {
+        code: 0,
+        record,
+        message: lines.join("\n"),
+        sharedBenchmarkManifest: sharedBenchmark.manifest,
+    };
 }
 //# sourceMappingURL=run.js.map

package/dist/eval/version-pin.d.ts

@@ -0,0 +1,99 @@
+/**
+ * #3215 — pin and record framework version per eval cell; refuse mixed-version aggregation.
+ * Extends #3081 empiricism; wires into the #1584 shared-benchmark manifest shape when present.
+ */
+/** Repo-relative path for the #1584 shared-benchmark manifest. */
+export declare const SHARED_BENCHMARK_MANIFEST_REL = "evals/shared-benchmark.json";
+/** How the framework version was resolved at eval run start. */
+export type FrameworkVersionSource = "package.json" | "override";
+/** Policy when a treatment cell contains disagreeing framework versions. */
+export type MixedVersionPolicy = "refuse" | "flag";
+/** Resolved framework version pin captured once at eval run start. */
+export interface FrameworkVersionPin {
+    readonly frameworkVersion: string;
+    readonly source: FrameworkVersionSource;
+    readonly resolvedAt: string;
+}
+/** Minimal run identity for cell purity checks (version + treatment grouping). */
+export interface VersionedEvalRun {
+    readonly frameworkVersion: string;
+    readonly treatment?: string;
+    readonly model?: string;
+    readonly harness?: string;
+    readonly runId?: string;
+}
+/** Result of checking whether one treatment cell is version-pure. */
+export interface CellVersionPurity {
+    readonly pure: boolean;
+    readonly frameworkVersion: string | null;
+    readonly versions: readonly string[];
+    readonly runCount: number;
+    readonly treatment: string;
+    readonly message: string;
+}
+export interface AggregateCellOptions {
+    readonly runs: readonly VersionedEvalRun[];
+    /** Treatment key when runs omit `treatment` (e.g. model or with_skill). */
+    readonly treatment?: string;
+    /** Default `refuse`: mixed cells block aggregation. `flag` allows with purity evidence. */
+    readonly policy?: MixedVersionPolicy;
+}
+export interface AggregateCellResult {
+    readonly purity: CellVersionPurity;
+    readonly allowed: boolean;
+    readonly policy: MixedVersionPolicy;
+    readonly frameworkVersion: string | null;
+}
+/** Version block stamped onto a #1584-shaped shared-benchmark manifest. */
+export interface SharedBenchmarkVersionBlock {
+    readonly frameworkVersion: string;
+    readonly frameworkVersionSource: FrameworkVersionSource;
+    readonly frameworkVersionResolvedAt: string;
+    readonly versionPurityGate: "#3215";
+}
+/**
+ * Resolve the directive framework version once at eval run start.
+ * Prefers an explicit override (tests / pinned doctor output); else package.json pin.
+ */
+export declare function resolveFrameworkVersionPin(options?: {
+    readonly override?: string;
+    readonly now?: () => Date;
+}): FrameworkVersionPin;
+/**
+ * Cell-level purity: all runs in one treatment must share a single framework version.
+ * Empty run sets are vacuously pure.
+ */
+export declare function evaluateCellVersionPurity(runs: readonly VersionedEvalRun[], treatment?: string): CellVersionPurity;
+/**
+ * Aggregate (or refuse) runs for one treatment under the mixed-version policy.
+ * `refuse` (default) sets `allowed=false` when versions disagree; `flag` allows with evidence.
+ */
+export declare function aggregateCellWithVersionPurity(options: AggregateCellOptions): AggregateCellResult;
+/**
+ * Group runs by treatment (or model×harness fallback) and evaluate purity per cell.
+ */
+export declare function evaluateLedgerVersionPurity(runs: readonly VersionedEvalRun[]): {
+    readonly pure: boolean;
+    readonly cells: readonly CellVersionPurity[];
+    readonly summary: string;
+};
+/**
+ * Merge a framework version pin into a #1584-shaped shared-benchmark manifest object.
+ * Stamps top-level `frameworkVersion` plus metadata for external consumers.
+ */
+export declare function wireFrameworkVersionIntoManifest(manifest: Record<string, unknown>, pin: FrameworkVersionPin): Record<string, unknown>;
+/** Load `evals/shared-benchmark.json` when present; otherwise null. */
+export declare function loadSharedBenchmarkManifest(projectRoot: string): Record<string, unknown> | null;
+/**
+ * When a #1584 shared-benchmark manifest exists under the project root, return it
+ * with the framework version pin wired in. When `persist` is true, write the
+ * wired manifest back to disk so subsequent consumers see the pin (#3215 Greptile).
+ */
+export declare function applyVersionPinToSharedBenchmark(projectRoot: string, pin: FrameworkVersionPin, options?: {
+    readonly persist?: boolean;
+}): {
+    readonly applied: boolean;
+    readonly persisted: boolean;
+    readonly manifest: Record<string, unknown> | null;
+};
+//# sourceMappingURL=version-pin.d.ts.map

package/dist/eval/version-pin.js

@@ -0,0 +1,181 @@
+/**
+ * #3215 — pin and record framework version per eval cell; refuse mixed-version aggregation.
+ * Extends #3081 empiricism; wires into the #1584 shared-benchmark manifest shape when present.
+ */
+import { existsSync, readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { readCorePackageVersion } from "../engine-version.js";
+import { containedWrite } from "../fs/contained-write.js";
+/** Repo-relative path for the #1584 shared-benchmark manifest. */
+export const SHARED_BENCHMARK_MANIFEST_REL = "evals/shared-benchmark.json";
+function toIsoZ(date) {
+    return date.toISOString().replace(/\.\d{3}Z$/, "Z");
+}
+/**
+ * Resolve the directive framework version once at eval run start.
+ * Prefers an explicit override (tests / pinned doctor output); else package.json pin.
+ */
+export function resolveFrameworkVersionPin(options) {
+    const now = options?.now ?? (() => new Date());
+    const override = options?.override?.trim();
+    if (override !== undefined && override.length > 0) {
+        return {
+            frameworkVersion: override,
+            source: "override",
+            resolvedAt: toIsoZ(now()),
+        };
+    }
+    return {
+        frameworkVersion: readCorePackageVersion(),
+        source: "package.json",
+        resolvedAt: toIsoZ(now()),
+    };
+}
+/**
+ * Cell-level purity: all runs in one treatment must share a single framework version.
+ * Empty run sets are vacuously pure.
+ */
+export function evaluateCellVersionPurity(runs, treatment) {
+    const label = treatment?.trim() ||
+        runs.find((r) => typeof r.treatment === "string" && r.treatment.length > 0)?.treatment ||
+        "default";
+    const versions = [
+        ...new Set(runs.map((r) => r.frameworkVersion.trim()).filter((v) => v.length > 0)),
+    ].sort();
+    if (runs.length === 0) {
+        return {
+            pure: true,
+            frameworkVersion: null,
+            versions: [],
+            runCount: 0,
+            treatment: label,
+            message: `Cell "${label}": no runs (vacuously pure).`,
+        };
+    }
+    if (versions.length <= 1) {
+        const version = versions[0] ?? null;
+        return {
+            pure: true,
+            frameworkVersion: version,
+            versions,
+            runCount: runs.length,
+            treatment: label,
+            message: `Cell "${label}": version-pure (v${version ?? "unknown"}, ${runs.length} run(s)).`,
+        };
+    }
+    return {
+        pure: false,
+        frameworkVersion: null,
+        versions,
+        runCount: runs.length,
+        treatment: label,
+        message: `Cell "${label}": mixed framework versions [${versions.join(", ")}] across ${runs.length} run(s) — mixed-version aggregation blocked (#3215).`,
+    };
+}
+/**
+ * Aggregate (or refuse) runs for one treatment under the mixed-version policy.
+ * `refuse` (default) sets `allowed=false` when versions disagree; `flag` allows with evidence.
+ */
+export function aggregateCellWithVersionPurity(options) {
+    const policy = options.policy ?? "refuse";
+    const purity = evaluateCellVersionPurity(options.runs, options.treatment);
+    const allowed = purity.pure || policy === "flag";
+    return {
+        purity,
+        allowed,
+        policy,
+        frameworkVersion: purity.frameworkVersion,
+    };
+}
+/**
+ * Group runs by treatment (or model×harness fallback) and evaluate purity per cell.
+ */
+export function evaluateLedgerVersionPurity(runs) {
+    const groups = new Map();
+    for (const run of runs) {
+        const key = run.treatment?.trim() ||
+            [run.model ?? "", run.harness ?? ""].filter((p) => p.length > 0).join("@") ||
+            "default";
+        const list = groups.get(key) ?? [];
+        list.push(run);
+        groups.set(key, list);
+    }
+    const cells = [...groups.entries()]
+        .sort(([a], [b]) => a.localeCompare(b))
+        .map(([key, group]) => evaluateCellVersionPurity(group, key));
+    const pure = cells.every((c) => c.pure);
+    const mixed = cells.filter((c) => !c.pure);
+    const summary = cells.length === 0
+        ? "Version purity: no cells."
+        : pure
+            ? `Version purity: all ${cells.length} cell(s) pure.`
+            : `Version purity: ${mixed.length}/${cells.length} cell(s) mixed — ${mixed.map((c) => c.treatment).join(", ")}.`;
+    return { pure, cells, summary };
+}
+/**
+ * Merge a framework version pin into a #1584-shaped shared-benchmark manifest object.
+ * Stamps top-level `frameworkVersion` plus metadata for external consumers.
+ */
+export function wireFrameworkVersionIntoManifest(manifest, pin) {
+    const priorMeta = typeof manifest.metadata === "object" &&
+        manifest.metadata !== null &&
+        !Array.isArray(manifest.metadata)
+        ? { ...manifest.metadata }
+        : {};
+    const versionBlock = {
+        frameworkVersion: pin.frameworkVersion,
+        frameworkVersionSource: pin.source,
+        frameworkVersionResolvedAt: pin.resolvedAt,
+        versionPurityGate: "#3215",
+    };
+    return {
+        ...manifest,
+        frameworkVersion: pin.frameworkVersion,
+        metadata: {
+            ...priorMeta,
+            ...versionBlock,
+        },
+    };
+}
+/** Load `evals/shared-benchmark.json` when present; otherwise null. */
+export function loadSharedBenchmarkManifest(projectRoot) {
+    const path = resolve(projectRoot, SHARED_BENCHMARK_MANIFEST_REL);
+    if (!existsSync(path)) {
+        return null;
+    }
+    try {
+        const parsed = JSON.parse(readFileSync(path, "utf8"));
+        if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) {
+            return parsed;
+        }
+        return null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * When a #1584 shared-benchmark manifest exists under the project root, return it
+ * with the framework version pin wired in. When `persist` is true, write the
+ * wired manifest back to disk so subsequent consumers see the pin (#3215 Greptile).
+ */
+export function applyVersionPinToSharedBenchmark(projectRoot, pin, options) {
+    const existing = loadSharedBenchmarkManifest(projectRoot);
+    if (existing === null) {
+        return { applied: false, persisted: false, manifest: null };
+    }
+    const manifest = wireFrameworkVersionIntoManifest(existing, pin);
+    const shouldPersist = options?.persist === true;
+    if (!shouldPersist) {
+        return { applied: true, persisted: false, manifest };
+    }
+    // #2980: product write sink routes through containedWrite.
+    containedWrite({
+        root: resolve(projectRoot),
+        target: SHARED_BENCHMARK_MANIFEST_REL,
+        data: `${JSON.stringify(manifest, null, 2)}\n`,
+        mode: "replace",
+    });
+    return { applied: true, persisted: true, manifest };
+}
+//# sourceMappingURL=version-pin.js.map

package/dist/index.d.ts

@@ -24,6 +24,7 @@ export * as evalCrud from "./eval/crud-telemetry.js";
 export * as evalHealth from "./eval/health.js";
 export * as evalReport from "./eval/report.js";
 export * as evalRun from "./eval/run.js";
+export * as evalVersionPin from "./eval/version-pin.js";
 export * as events from "./events/attribution-ledger.js";
 export * from "./forward-coverage/evaluate.js";
 export * as freshness from "./freshness/index.js";

package/dist/index.js

@@ -24,6 +24,7 @@ export * as evalCrud from "./eval/crud-telemetry.js";
 export * as evalHealth from "./eval/health.js";
 export * as evalReport from "./eval/report.js";
 export * as evalRun from "./eval/run.js";
+export * as evalVersionPin from "./eval/version-pin.js";
 export * as events from "./events/attribution-ledger.js";
 export * from "./forward-coverage/evaluate.js";
 export * as freshness from "./freshness/index.js";

package/dist/platform/host-content-surface.d.ts

@@ -0,0 +1,74 @@
+/**
+ * Host content-surface class + managed-section drift (#3162).
+ *
+ * Directive file gates and #830/#2508 pins assume filesystem-visible work product
+ * and a host that does not rewrite constitution mid-run. REPL-first and
+ * self-mutating hosts break those assumptions. This module classifies the host
+ * surface (capability pointer #1461/#1357) and reports managed AGENTS section
+ * drift without inventing a full pin product.
+ *
+ * Never blocks session-start — advisory only.
+ */
+import { type AgentsMdSeams } from "./agents-md.js";
+/** Host content-surface classes for gate / pin honesty (#3162). */
+export declare const HOST_CONTENT_SURFACE_CLASSES: readonly ["file-first", "repl-first", "self-mutating", "unknown"];
+export type HostContentSurfaceClass = (typeof HOST_CONTENT_SURFACE_CLASSES)[number];
+/** Env key for explicit class override (capability descriptor pointer). */
+export declare const ENV_HOST_CONTENT_SURFACE = "DEFT_HOST_CONTENT_SURFACE";
+/** Env opt-in: host holds executable work product before files exist. */
+export declare const ENV_HOST_REPL_FIRST = "DEFT_HOST_REPL_FIRST";
+/** Env opt-in: host refine/kernel may CRUD skills/prompts mid-run. */
+export declare const ENV_HOST_SELF_MUTATE = "DEFT_HOST_SELF_MUTATE";
+export interface HostContentSurfaceClassResult {
+    readonly contentClass: HostContentSurfaceClass;
+    readonly source: string;
+    /** Signals observed (env keys present). */
+    readonly signals: readonly string[];
+}
+export type ManagedSectionDriftState = "current" | "stale" | "missing" | "absent" | "unreadable" | "template-missing" | "template-malformed" | "unknown";
+export interface ManagedSectionDriftReport {
+    readonly state: ManagedSectionDriftState;
+    readonly embeddedSha: string | null;
+    /** SHA-256 (12 hex) of stripped managed-section body when readable. */
+    readonly bodyHash: string | null;
+    readonly path: string;
+}
+export interface HostContentSurfaceReport {
+    readonly contentClass: HostContentSurfaceClass;
+    readonly classSource: string;
+    readonly signals: readonly string[];
+    readonly managedSection: ManagedSectionDriftReport;
+    /** Runtime mode from platform capability descriptor when provided. */
+    readonly runtimeMode: string | null;
+}
+export interface HostContentSurfaceSeams {
+    readonly environ?: Readonly<Record<string, string | undefined>>;
+    readonly agentsMdSeams?: AgentsMdSeams;
+    readonly runtimeMode?: string | null;
+}
+/**
+ * Classify host content surface from explicit env / capability signals (#1461/#1357 pointer).
+ * Defaults to file-first (historical Directive assumption) when no signal is set.
+ */
+export declare function classifyHostContentSurface(environ?: Readonly<Record<string, string | undefined>>): HostContentSurfaceClassResult;
+/**
+ * Probe managed AGENTS.md section drift (tamper-evident boundary, not full #830 pins).
+ * Uses agents:refresh plan state + embedded marker sha + body hash.
+ */
+export declare function probeManagedSectionDrift(projectRoot: string, seams?: HostContentSurfaceSeams): ManagedSectionDriftReport;
+/** Full host-surface report for session-start payload + lines. */
+export declare function probeHostContentSurface(projectRoot: string, seams?: HostContentSurfaceSeams): HostContentSurfaceReport;
+export declare function hostContentSurfaceToDict(report: HostContentSurfaceReport): Record<string, unknown>;
+/**
+ * Format operator-facing host-surface lines (#3162).
+ * Always emits one summary line; adds honesty note for non-file-first or drift.
+ */
+export declare function formatHostContentSurfaceLines(report: HostContentSurfaceReport): string[];
+/**
+ * Probe + format; fail-open for session-start (never throws to callers that wrap).
+ */
+export declare function maybeFormatHostContentSurfaceLines(projectRoot: string, seams?: HostContentSurfaceSeams): {
+    report: HostContentSurfaceReport;
+    lines: string[];
+};
+//# sourceMappingURL=host-content-surface.d.ts.map