harnery 0.23.0 → 0.25.0

package/dist/core/harnesses/attest.js

@@ -0,0 +1,110 @@
+/**
+ * The opt-in live probe that produces an attestation (ADR 0038).
+ *
+ * One bounded turn per harness, through the same `spawn` the workflow engine
+ * uses, so what gets attested is the path production takes rather than a
+ * parallel test rig.
+ */
+import { profileDigest, sealAttestation, writeAttestation } from "./attestation.js";
+import { probeBinaryVersion } from "./bench.js";
+/** Fixed and content-free, so nothing user-supplied can reach the record and
+ * the turn stays as cheap as a turn can be. */
+export const ATTESTATION_PROMPT = "Reply with the single word: ok";
+export const DEFAULT_ATTESTATION_TIMEOUT_MS = 120_000;
+/** Vendor failures arrive as whole console transcripts, banner and prompt echo
+ * included. Reports are bounded evidence, so the reason is collapsed to one
+ * short line and the echoed prompt is removed. */
+const MAX_NOTE_REASON_CHARS = 200;
+function boundedReason(reason) {
+    if (!reason)
+        return "no error reported";
+    const collapsed = reason.split(ATTESTATION_PROMPT).join("<prompt>").replace(/\s+/g, " ").trim();
+    return collapsed.length > MAX_NOTE_REASON_CHARS
+        ? `${collapsed.slice(0, MAX_NOTE_REASON_CHARS)}…`
+        : collapsed;
+}
+export async function runHarnessAttestation(registry, opts = {}) {
+    const ids = opts.harnesses?.length ? [...opts.harnesses] : registry.ids();
+    const timeoutMs = opts.timeoutMs ?? DEFAULT_ATTESTATION_TIMEOUT_MS;
+    const cwd = opts.cwd ?? process.cwd();
+    const versionProbe = opts.versionProbe ?? probeBinaryVersion;
+    const now = opts.now ?? (() => new Date());
+    const results = [];
+    for (const id of ids) {
+        const adapter = registry.require(id);
+        const binaryVersion = versionProbe(adapter.profile.binary);
+        if (!binaryVersion) {
+            results.push({
+                harness: id,
+                outcome: "skipped",
+                note: `${adapter.profile.binary} not found on PATH`,
+            });
+            continue;
+        }
+        let result;
+        try {
+            result = opts.spawn
+                ? await opts.spawn(id, ATTESTATION_PROMPT, timeoutMs)
+                : await adapter.spawn({
+                    prompt: ATTESTATION_PROMPT,
+                    timeoutMs,
+                    maxTurns: 1,
+                    cwd,
+                });
+        }
+        catch (error) {
+            results.push({
+                harness: id,
+                outcome: "failed",
+                binaryVersion,
+                note: `probe threw: ${boundedReason(error.message)}`,
+            });
+            continue;
+        }
+        // Prerequisite rule: an unreachable subject cannot evidence anything, so a
+        // failed turn records nothing at all rather than a page of `unsupported`.
+        if (!result.ok) {
+            results.push({
+                harness: id,
+                outcome: "unreachable",
+                binaryVersion,
+                durationMs: result.durationMs,
+                note: `the probe turn did not complete (${boundedReason(result.error)}); nothing recorded`,
+            });
+            continue;
+        }
+        const observations = {
+            invocation: "supported",
+            finalResult: result.text.trim().length > 0 ? "supported" : "unsupported",
+            sessionId: result.sessionId ? "supported" : "unsupported",
+            cost: result.costUsd !== undefined ? "supported" : "unsupported",
+        };
+        const record = sealAttestation({
+            schema_version: 1,
+            harness: id,
+            binary_version: binaryVersion,
+            profile_digest: profileDigest(adapter.profile),
+            observed_at: now().toISOString(),
+            observations,
+        });
+        if (opts.persist)
+            opts.persist(record);
+        else
+            writeAttestation(record, { coordRoot: opts.coordRoot });
+        results.push({
+            harness: id,
+            outcome: "recorded",
+            binaryVersion,
+            observations,
+            durationMs: result.durationMs,
+            note: `observed on ${binaryVersion}`,
+        });
+    }
+    return {
+        generatedAt: now().toISOString(),
+        harnesses: ids,
+        results,
+        recorded: results.filter((row) => row.outcome === "recorded").length,
+        incomplete: results.some((row) => row.outcome !== "recorded"),
+    };
+}

package/dist/core/harnesses/attestation.d.ts

@@ -0,0 +1,75 @@
+/**
+ * Durable record of what an installed vendor CLI was actually observed doing
+ * (ADR 0038).
+ *
+ * The conformance bench proves Harnery's own planner and normalizer against a
+ * committed fixture. That is an adapter check, not a vendor check
+ * (ADR 0037). An attestation is the other half: one bounded live turn, its
+ * observations, and the vendor version they were observed on.
+ *
+ * A record stores structural facts only. No prompt text, no completion text,
+ * no host paths. It lives under the host's coordination directory and is never
+ * published.
+ */
+import type { CapabilitySupport, HarnessId, HarnessProfile } from "./types.js";
+export declare const ATTESTATION_SCHEMA_VERSION = 1;
+/** Dimensions one minimal live turn can honestly establish. Everything else
+ * needs a purpose-built scenario and stays outside the record rather than
+ * being guessed at. */
+export declare const ATTESTABLE_DIMENSIONS: readonly ["invocation", "finalResult", "sessionId", "cost"];
+export type AttestableDimension = (typeof ATTESTABLE_DIMENSIONS)[number];
+export interface HarnessAttestation {
+    schema_version: number;
+    harness: HarnessId;
+    /** What the vendor binary reported when this was recorded. Staleness is
+     * keyed on this, so a vendor upgrade invalidates the record automatically. */
+    binary_version: string;
+    /** Digest of the capability declaration at record time, so an edited
+     * declaration also invalidates the record. */
+    profile_digest: string;
+    observed_at: string;
+    /** Only what the probe actually saw. A dimension absent from this map was
+     * not observed, which is not the same as unsupported. */
+    observations: Partial<Record<AttestableDimension, CapabilitySupport>>;
+    /** Over every field above. A hand-edited record fails to load. */
+    record_digest: string;
+}
+export interface AttestationStoreOptions {
+    /** Test seam and alternate host. Defaults to the resolved coord root. */
+    coordRoot?: string;
+}
+export declare function attestationsDir(opts?: AttestationStoreOptions): string;
+/** Stable identity of a declaration, so editing a claim invalidates the
+ * attestation that was recorded against the old one. */
+export declare function profileDigest(profile: HarnessProfile): string;
+export declare function sealAttestation(record: Omit<HarnessAttestation, "record_digest">): HarnessAttestation;
+/** Replace-in-place write. Unlike a workflow record an attestation is meant to
+ * be re-recorded, so this is a mutable atomic swap rather than an immutable
+ * create. */
+export declare function writeAttestation(record: HarnessAttestation, opts?: AttestationStoreOptions): string;
+/** Null for absent, unreadable, malformed, wrong-schema, or tampered records.
+ * A record that fails its own digest is discarded rather than trusted, because
+ * the whole point of the file is that it was not hand-written. */
+export declare function readAttestation(harness: HarnessId, opts?: AttestationStoreOptions): HarnessAttestation | null;
+export declare function validateAttestation(value: unknown, harness?: HarnessId): HarnessAttestation | null;
+/** An attestation speaks only for the vendor version and declaration it was
+ * recorded against. */
+export declare function isAttestationCurrent(record: HarnessAttestation | null, binaryVersion: string | null, profile: HarnessProfile): record is HarnessAttestation;
+export declare function listAttestations(opts?: AttestationStoreOptions): HarnessAttestation[];
+/** Both harness-derived proof inputs, read once, for a workflow run
+ * (ADR 0038). Callers inject the result so the engine performs no capability
+ * lookups of its own. A harness with no current attestation simply has no
+ * citation; that absence is not a proof unknown. */
+export declare function harnessProofInputs(profiles: readonly HarnessProfile[], opts: AttestationStoreOptions & {
+    versionProbe: (binary: string) => string | null;
+}): {
+    harnessEvidence: Record<string, {
+        toolEvidence: HarnessProfile["capabilities"]["toolEvidence"];
+    }>;
+    harnessAttestations: Record<string, {
+        binary_version: string;
+        observed_at: string;
+        record_digest: string;
+    }>;
+};
+//# sourceMappingURL=attestation.d.ts.map

package/dist/core/harnesses/attestation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"attestation.d.ts","sourceRoot":"","sources":["../../../src/core/harnesses/attestation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAaH,OAAO,KAAK,EAAE,iBAAiB,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE/E,eAAO,MAAM,0BAA0B,IAAI,CAAC;AAE5C;;uBAEuB;AACvB,eAAO,MAAM,qBAAqB,6DAA8D,CAAC;AAEjG,MAAM,MAAM,mBAAmB,GAAG,CAAC,OAAO,qBAAqB,CAAC,CAAC,MAAM,CAAC,CAAC;AAEzE,MAAM,WAAW,kBAAkB;IACjC,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,SAAS,CAAC;IACnB;kFAC8E;IAC9E,cAAc,EAAE,MAAM,CAAC;IACvB;kDAC8C;IAC9C,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB;6DACyD;IACzD,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,mBAAmB,EAAE,iBAAiB,CAAC,CAAC,CAAC;IACtE,kEAAkE;IAClE,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,uBAAuB;IACtC,yEAAyE;IACzE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,wBAAgB,eAAe,CAAC,IAAI,GAAE,uBAA4B,GAAG,MAAM,CAI1E;AAED;wDACwD;AACxD,wBAAgB,aAAa,CAAC,OAAO,EAAE,cAAc,GAAG,MAAM,CAM7D;AAMD,wBAAgB,eAAe,CAC7B,MAAM,EAAE,IAAI,CAAC,kBAAkB,EAAE,eAAe,CAAC,GAChD,kBAAkB,CAEpB;AASD;;aAEa;AACb,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,kBAAkB,EAC1B,IAAI,GAAE,uBAA4B,GACjC,MAAM,CAmBR;AAED;;kEAEkE;AAClE,wBAAgB,eAAe,CAC7B,OAAO,EAAE,SAAS,EAClB,IAAI,GAAE,uBAA4B,GACjC,kBAAkB,GAAG,IAAI,CAc3B;AAED,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,OAAO,EACd,OAAO,CAAC,EAAE,SAAS,GAClB,kBAAkB,GAAG,IAAI,CAe3B;AAED;uBACuB;AACvB,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,kBAAkB,GAAG,IAAI,EACjC,aAAa,EAAE,MAAM,GAAG,IAAI,EAC5B,OAAO,EAAE,cAAc,GACtB,MAAM,IAAI,kBAAkB,CAI9B;AAED,wBAAgB,gBAAgB,CAAC,IAAI,GAAE,uBAA4B,GAAG,kBAAkB,EAAE,CAczF;AAED;;;oDAGoD;AACpD,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,SAAS,cAAc,EAAE,EACnC,IAAI,EAAE,uBAAuB,GAAG;IAC9B,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;CACjD,GACA;IACD,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,YAAY,EAAE,cAAc,CAAC,cAAc,CAAC,CAAC,cAAc,CAAC,CAAA;KAAE,CAAC,CAAC;IAClG,mBAAmB,EAAE,MAAM,CACzB,MAAM,EACN;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CACvE,CAAC;CACH,CA2BA"}

package/dist/core/harnesses/attestation.js

@@ -0,0 +1,168 @@
+/**
+ * Durable record of what an installed vendor CLI was actually observed doing
+ * (ADR 0038).
+ *
+ * The conformance bench proves Harnery's own planner and normalizer against a
+ * committed fixture. That is an adapter check, not a vendor check
+ * (ADR 0037). An attestation is the other half: one bounded live turn, its
+ * observations, and the vendor version they were observed on.
+ *
+ * A record stores structural facts only. No prompt text, no completion text,
+ * no host paths. It lives under the host's coordination directory and is never
+ * published.
+ */
+import { mkdirSync, readdirSync, readFileSync, renameSync, unlinkSync, writeFileSync, } from "node:fs";
+import { resolve } from "node:path";
+import { monorepoRoot } from "../agents/coord-client.js";
+import { stableDigest } from "../workflow/durable-record.js";
+export const ATTESTATION_SCHEMA_VERSION = 1;
+/** Dimensions one minimal live turn can honestly establish. Everything else
+ * needs a purpose-built scenario and stays outside the record rather than
+ * being guessed at. */
+export const ATTESTABLE_DIMENSIONS = ["invocation", "finalResult", "sessionId", "cost"];
+export function attestationsDir(opts = {}) {
+    const root = opts.coordRoot ?? monorepoRoot();
+    if (!root)
+        throw new Error("Not in a coord-aware repo (coord root resolved to null).");
+    return resolve(root, ".harnery", "harnesses", "attestations");
+}
+/** Stable identity of a declaration, so editing a claim invalidates the
+ * attestation that was recorded against the old one. */
+export function profileDigest(profile) {
+    return stableDigest({
+        id: profile.id,
+        binary: profile.binary,
+        capabilities: profile.capabilities,
+    });
+}
+function digestOf(record) {
+    return stableDigest(record);
+}
+export function sealAttestation(record) {
+    return { ...record, record_digest: digestOf(record) };
+}
+function attestationPath(harness, opts) {
+    if (!/^[a-z0-9][a-z0-9._-]*$/i.test(harness)) {
+        throw new Error(`unsafe harness id for an attestation path: ${harness}`);
+    }
+    return resolve(attestationsDir(opts), `${harness}.json`);
+}
+/** Replace-in-place write. Unlike a workflow record an attestation is meant to
+ * be re-recorded, so this is a mutable atomic swap rather than an immutable
+ * create. */
+export function writeAttestation(record, opts = {}) {
+    const path = attestationPath(record.harness, opts);
+    mkdirSync(attestationsDir(opts), { recursive: true, mode: 0o700 });
+    const temporary = `${path}.tmp-${process.pid}`;
+    try {
+        writeFileSync(temporary, `${JSON.stringify(record, null, 2)}\n`, {
+            encoding: "utf8",
+            mode: 0o600,
+        });
+        renameSync(temporary, path);
+    }
+    catch (error) {
+        try {
+            unlinkSync(temporary);
+        }
+        catch {
+            // The temp file may never have been created; the original error wins.
+        }
+        throw error;
+    }
+    return path;
+}
+/** Null for absent, unreadable, malformed, wrong-schema, or tampered records.
+ * A record that fails its own digest is discarded rather than trusted, because
+ * the whole point of the file is that it was not hand-written. */
+export function readAttestation(harness, opts = {}) {
+    let raw;
+    try {
+        raw = readFileSync(attestationPath(harness, opts), "utf8");
+    }
+    catch {
+        return null;
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch {
+        return null;
+    }
+    return validateAttestation(parsed, harness);
+}
+export function validateAttestation(value, harness) {
+    if (!value || typeof value !== "object")
+        return null;
+    const record = value;
+    if (record.schema_version !== ATTESTATION_SCHEMA_VERSION)
+        return null;
+    if (typeof record.harness !== "string" || (harness && record.harness !== harness))
+        return null;
+    if (typeof record.binary_version !== "string" || typeof record.profile_digest !== "string") {
+        return null;
+    }
+    if (typeof record.observed_at !== "string" || typeof record.record_digest !== "string")
+        return null;
+    if (!record.observations || typeof record.observations !== "object")
+        return null;
+    const { record_digest, ...body } = record;
+    if (digestOf(body) !== record_digest)
+        return null;
+    return record;
+}
+/** An attestation speaks only for the vendor version and declaration it was
+ * recorded against. */
+export function isAttestationCurrent(record, binaryVersion, profile) {
+    if (!record || !binaryVersion)
+        return false;
+    if (record.binary_version !== binaryVersion)
+        return false;
+    return record.profile_digest === profileDigest(profile);
+}
+export function listAttestations(opts = {}) {
+    let names;
+    try {
+        names = readdirSync(attestationsDir(opts));
+    }
+    catch {
+        return [];
+    }
+    const records = [];
+    for (const name of names) {
+        if (!name.endsWith(".json"))
+            continue;
+        const record = readAttestation(name.slice(0, -".json".length), opts);
+        if (record)
+            records.push(record);
+    }
+    return records.sort((a, b) => a.harness.localeCompare(b.harness));
+}
+/** Both harness-derived proof inputs, read once, for a workflow run
+ * (ADR 0038). Callers inject the result so the engine performs no capability
+ * lookups of its own. A harness with no current attestation simply has no
+ * citation; that absence is not a proof unknown. */
+export function harnessProofInputs(profiles, opts) {
+    const harnessEvidence = {};
+    const harnessAttestations = {};
+    for (const profile of profiles) {
+        harnessEvidence[profile.id] = { toolEvidence: profile.capabilities.toolEvidence };
+        let record = null;
+        try {
+            record = readAttestation(profile.id, opts);
+        }
+        catch {
+            // No coord root or unreadable store: run unattested rather than fail.
+            continue;
+        }
+        if (!isAttestationCurrent(record, opts.versionProbe(profile.binary), profile))
+            continue;
+        harnessAttestations[profile.id] = {
+            binary_version: record.binary_version,
+            observed_at: record.observed_at,
+            record_digest: record.record_digest,
+        };
+    }
+    return { harnessEvidence, harnessAttestations };
+}

package/dist/core/harnesses/bench.d.ts

@@ -1,13 +1,23 @@
+import type { HarnessAttestation } from "./attestation.js";
 import type { HarnessRegistry } from "./registry.js";
 import type { CapabilitySupport, HarnessCapabilityDimension, HarnessId } from "./types.js";
 export type BenchVerdict = "supported" | "partial" | "unsupported" | "not_applicable" | "unknown" | "skipped" | "drift";
-export type BenchDimension = "registration" | "binary" | HarnessCapabilityDimension;
+export type BenchDimension = "registration" | "binary" | "contract" | HarnessCapabilityDimension;
+/** How a result was established (ADR 0037). Orthogonal to the verdict: this
+ * answers "how do we know", not "what is true".
+ *
+ * - `adapter`: checked against Harnery's own planner, normalizer, or fixture.
+ *   Proves the adapter contract, says nothing about the installed vendor CLI.
+ * - `attested`: checked by observing the installed vendor CLI on this host.
+ * - `declared`: not checked. The value is the declaration, repeated. */
+export type BenchBasis = "adapter" | "attested" | "declared";
 export interface BenchResult {
     harness: HarnessId;
     dimension: BenchDimension;
     declared: CapabilitySupport | "not_applicable";
     observed: BenchVerdict;
     verdict: BenchVerdict;
+    basis: BenchBasis;
     note?: string;
 }
 export interface HarnessBenchReport {
@@ -16,6 +26,9 @@ export interface HarnessBenchReport {
     harnesses: HarnessId[];
     results: BenchResult[];
     summary: Record<BenchVerdict, number>;
+    /** Result counts per basis, so a caller can tell a clean report from an
+     * unmeasured one without walking every result. */
+    basisSummary: Record<BenchBasis, number>;
     drift: boolean;
     skipped: boolean;
 }
@@ -24,6 +37,15 @@ export interface HarnessBenchOptions {
     dimensions?: readonly HarnessCapabilityDimension[];
     /** Test seam and alternate host probe. A null version means unavailable. */
     versionProbe?: (binary: string) => string | null;
+    /** Where to look for live attestations (ADR 0038). Defaults to the coord
+     * root. Reading them never runs a model turn. */
+    coordRoot?: string;
+    /** Test seam. Defaults to reading the attestation store. */
+    attestationReader?: (harness: HarnessId) => HarnessAttestation | null;
 }
 export declare function runHarnessBench(registry: HarnessRegistry, opts?: HarnessBenchOptions): HarnessBenchReport;
+/** Ask an installed vendor CLI what version it is. Null when it is absent or
+ * refuses to answer. Shared with the live attestation probe so both halves of
+ * the capability story key on the same string. */
+export declare function probeBinaryVersion(binary: string): string | null;
 //# sourceMappingURL=bench.d.ts.map

package/dist/core/harnesses/bench.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"bench.d.ts","sourceRoot":"","sources":["../../../src/core/harnesses/bench.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,KAAK,EACV,iBAAiB,EAEjB,0BAA0B,EAC1B,SAAS,EACV,MAAM,YAAY,CAAC;AAGpB,MAAM,MAAM,YAAY,GACpB,WAAW,GACX,SAAS,GACT,aAAa,GACb,gBAAgB,GAChB,SAAS,GACT,SAAS,GACT,OAAO,CAAC;AAEZ,MAAM,MAAM,cAAc,GAAG,cAAc,GAAG,QAAQ,GAAG,0BAA0B,CAAC;AAEpF,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,SAAS,CAAC;IACnB,SAAS,EAAE,cAAc,CAAC;IAC1B,QAAQ,EAAE,iBAAiB,GAAG,gBAAgB,CAAC;IAC/C,QAAQ,EAAE,YAAY,CAAC;IACvB,OAAO,EAAE,YAAY,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,SAAS,CAAC;IAChB,SAAS,EAAE,SAAS,EAAE,CAAC;IACvB,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IACtC,KAAK,EAAE,OAAO,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,mBAAmB;IAClC,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9B,UAAU,CAAC,EAAE,SAAS,0BAA0B,EAAE,CAAC;IACnD,4EAA4E;IAC5E,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;CAClD;AAYD,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,eAAe,EACzB,IAAI,GAAE,mBAAwB,GAC7B,kBAAkB,CAuDpB"}
+{"version":3,"file":"bench.d.ts","sourceRoot":"","sources":["../../../src/core/harnesses/bench.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAuB,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEhF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,KAAK,EACV,iBAAiB,EAEjB,0BAA0B,EAC1B,SAAS,EACV,MAAM,YAAY,CAAC;AAGpB,MAAM,MAAM,YAAY,GACpB,WAAW,GACX,SAAS,GACT,aAAa,GACb,gBAAgB,GAChB,SAAS,GACT,SAAS,GACT,OAAO,CAAC;AAEZ,MAAM,MAAM,cAAc,GAAG,cAAc,GAAG,QAAQ,GAAG,UAAU,GAAG,0BAA0B,CAAC;AAEjG;;;;;;wEAMwE;AACxE,MAAM,MAAM,UAAU,GAAG,SAAS,GAAG,UAAU,GAAG,UAAU,CAAC;AAE7D,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,SAAS,CAAC;IACnB,SAAS,EAAE,cAAc,CAAC;IAC1B,QAAQ,EAAE,iBAAiB,GAAG,gBAAgB,CAAC;IAC/C,QAAQ,EAAE,YAAY,CAAC;IACvB,OAAO,EAAE,YAAY,CAAC;IACtB,KAAK,EAAE,UAAU,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,SAAS,CAAC;IAChB,SAAS,EAAE,SAAS,EAAE,CAAC;IACvB,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IACtC;sDACkD;IAClD,YAAY,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IACzC,KAAK,EAAE,OAAO,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,mBAAmB;IAClC,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9B,UAAU,CAAC,EAAE,SAAS,0BAA0B,EAAE,CAAC;IACnD,4EAA4E;IAC5E,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IACjD;qDACiD;IACjD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,4DAA4D;IAC5D,iBAAiB,CAAC,EAAE,CAAC,OAAO,EAAE,SAAS,KAAK,kBAAkB,GAAG,IAAI,CAAC;CACvE;AAgDD,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,eAAe,EACzB,IAAI,GAAE,mBAAwB,GAC7B,kBAAkB,CAwEpB;AA2KD;;kDAEkD;AAClD,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAIhE"}

package/dist/core/harnesses/bench.js

@@ -1,5 +1,6 @@
 import { spawnSync } from "node:child_process";
 import { HARNESS_SPECS } from "../hooks/harness/events.js";
+import { isAttestationCurrent, readAttestation } from "./attestation.js";
 import { HARNESS_CAPABILITY_DIMENSIONS } from "./types.js";
 const EMPTY_SUMMARY = {
     supported: 0,
@@ -10,12 +11,42 @@ const EMPTY_SUMMARY = {
     skipped: 0,
     drift: 0,
 };
+const EMPTY_BASIS_SUMMARY = {
+    adapter: 0,
+    attested: 0,
+    declared: 0,
+};
+/** A stored attestation only counts when it still matches the installed
+ * version and the current declaration. A stale one is ignored rather than
+ * trusted, so an upgraded vendor CLI silently drops back to adapter basis. */
+function loadAttestation(id, version, adapter, opts) {
+    let record = null;
+    try {
+        record = opts.attestationReader
+            ? opts.attestationReader(id)
+            : readAttestation(id, { coordRoot: opts.coordRoot });
+    }
+    catch {
+        // No coord root, unreadable store: the bench still runs, just unattested.
+        return null;
+    }
+    return isAttestationCurrent(record, version, adapter.profile) ? record : null;
+}
+/** First version-shaped token in a string, or null when there is none.
+ * Tolerates the vendor prefixes and suffixes real CLIs print, so
+ * `codex-cli 0.144.5` and `2.1.197 (Claude Code)` both reduce to a token. */
+function versionToken(value) {
+    if (!value)
+        return null;
+    const match = value.match(/\d+(?:\.\d+)+(?:[-.][0-9a-z]+)*/i);
+    return match ? match[0].toLowerCase() : null;
+}
 export function runHarnessBench(registry, opts = {}) {
     const ids = opts.harnesses?.length ? [...opts.harnesses] : registry.ids();
     const dimensions = opts.dimensions?.length
         ? [...new Set(opts.dimensions)]
         : [...HARNESS_CAPABILITY_DIMENSIONS];
-    const versionProbe = opts.versionProbe ?? probeVersion;
+    const versionProbe = opts.versionProbe ?? probeBinaryVersion;
     const results = [];
     for (const id of ids) {
         const adapter = registry.require(id);
@@ -25,6 +56,7 @@ export function runHarnessBench(registry, opts = {}) {
             declared: "supported",
             observed: "supported",
             verdict: "supported",
+            basis: "adapter",
             note: `${adapter.profile.integrationMode}; ${adapter.profile.authModel}`,
         });
         const version = versionProbe(adapter.profile.binary);
@@ -34,35 +66,105 @@ export function runHarnessBench(registry, opts = {}) {
             declared: "supported",
             observed: version ? "supported" : "skipped",
             verdict: version ? "supported" : "skipped",
+            // A skip establishes nothing about the vendor, so it is not an attestation.
+            basis: version ? "attested" : "declared",
             note: version ?? `${adapter.profile.binary} not found on PATH`,
         });
+        results.push(contractResult(id, adapter, version));
         const observations = observeAdapter(adapter);
+        const attestation = loadAttestation(id, version, adapter, opts);
         for (const dimension of dimensions) {
             const claim = adapter.profile.capabilities[dimension];
-            const observed = observations[dimension];
+            const live = attestation?.observations[dimension];
+            // A live observation of the installed CLI outranks a fixture check of
+            // Harnery's own normalizer. Everything else keeps its adapter basis.
+            const { observed, basis } = live
+                ? { observed: live, basis: "attested" }
+                : observations[dimension];
             results.push({
                 harness: id,
                 dimension,
                 declared: claim.support,
                 observed,
                 verdict: reconcile(claim.support, observed),
-                note: claim.note,
+                basis,
+                note: live ? `attested on ${attestation?.binary_version}` : claim.note,
             });
         }
     }
     const summary = { ...EMPTY_SUMMARY };
-    for (const result of results)
+    const basisSummary = { ...EMPTY_BASIS_SUMMARY };
+    for (const result of results) {
         summary[result.verdict]++;
+        basisSummary[result.basis]++;
+    }
     return {
         generatedAt: new Date().toISOString(),
         mode: "offline",
         harnesses: ids,
         results,
         summary,
+        basisSummary,
         drift: summary.drift > 0,
         skipped: summary.skipped > 0,
     };
 }
+/** Compare the vendor contract a declaration was validated against with the
+ * one actually installed (ADR 0037). This is the only dimension attested
+ * without a model turn, because a version string costs nothing to read.
+ *
+ * A mismatch is `drift`, not failure: a newer vendor CLI is not automatically
+ * broken, but the declaration is no longer backed by an observation. An
+ * unrecorded or unparseable declared version stays `unknown` rather than being
+ * inferred from the installed one. */
+function contractResult(id, adapter, version) {
+    const recorded = adapter.profile.verified;
+    const base = { harness: id, dimension: "contract", declared: "supported" };
+    if (!version) {
+        return {
+            ...base,
+            observed: "skipped",
+            verdict: "skipped",
+            basis: "declared",
+            note: `${adapter.profile.binary} not found on PATH; installed contract unobservable`,
+        };
+    }
+    const declaredToken = versionToken(recorded?.version);
+    if (!declaredToken) {
+        return {
+            ...base,
+            observed: "unknown",
+            verdict: "unknown",
+            basis: "declared",
+            note: recorded
+                ? `verified.version "${recorded.version}" is not a version; installed ${version}`
+                : `no verified vendor contract recorded; installed ${version}`,
+        };
+    }
+    const installedToken = versionToken(version) ?? version.trim().toLowerCase();
+    if (installedToken === declaredToken) {
+        return {
+            ...base,
+            observed: "supported",
+            verdict: "supported",
+            basis: "attested",
+            note: `declaration validated against ${recorded?.version} on ${recorded?.date}`,
+        };
+    }
+    return {
+        ...base,
+        observed: "drift",
+        verdict: "drift",
+        basis: "attested",
+        note: `declaration validated against ${recorded?.version} (${recorded?.date}); installed ${version}`,
+    };
+}
+/** Checked against Harnery's planner, normalizer, or fixture. */
+function fromAdapter(observed) {
+    return { observed, basis: "adapter" };
+}
+/** No check exists for this dimension yet. */
+const NOT_CHECKED = { observed: "unknown", basis: "declared" };
 function observeAdapter(adapter) {
     const profile = adapter.profile;
     const effort = profile.effortValues[0];
@@ -98,43 +200,41 @@ function observeAdapter(adapter) {
         : undefined;
     const hookSubcommands = new Set(hookSpec?.events.map((event) => event.subcommand) ?? []);
     return {
-        invocation: !planningFailed && argv[0] === profile.binary && argv.includes(request.prompt)
+        invocation: fromAdapter(!planningFailed && argv[0] === profile.binary && argv.includes(request.prompt)
             ? "supported"
-            : "unsupported",
-        modelSelection: !planningFailed && argv.includes(request.model ?? "") ? "supported" : "unsupported",
-        effortSelection: effort === undefined
+            : "unsupported"),
+        modelSelection: fromAdapter(!planningFailed && argv.includes(request.model ?? "") ? "supported" : "unsupported"),
+        effortSelection: fromAdapter(effort === undefined
             ? "unsupported"
             : !planningFailed && argv.some((arg) => arg.includes(effort))
                 ? "supported"
-                : "unsupported",
-        maxTurns: !planningFailed && argv.includes(String(request.maxTurns)) ? "supported" : "unsupported",
-        finalResult: finalMatches ? "supported" : "unsupported",
-        sessionId: sessionObserved === "supported" && normalized?.sessionId !== fixture.sessionId
+                : "unsupported"),
+        maxTurns: fromAdapter(!planningFailed && argv.includes(String(request.maxTurns)) ? "supported" : "unsupported"),
+        finalResult: fromAdapter(finalMatches ? "supported" : "unsupported"),
+        sessionId: fromAdapter(sessionObserved === "supported" && normalized?.sessionId !== fixture.sessionId
             ? "unsupported"
-            : sessionObserved,
-        cost: costObserved === "supported" && normalized?.costUsd !== fixture.costUsd
+            : sessionObserved),
+        cost: fromAdapter(costObserved === "supported" && normalized?.costUsd !== fixture.costUsd
             ? "unsupported"
-            : costObserved,
-        toolEvidence: normalized && "toolEvidence" in normalized ? "supported" : "unsupported",
-        policyMapping: "unknown",
-        interruption: "unknown",
-        streaming: "unknown",
-        steering: "unknown",
-        resume: "unknown",
-        images: "unknown",
-        contextTelemetry: "unknown",
+            : costObserved),
+        toolEvidence: fromAdapter(normalized && "toolEvidence" in normalized ? "supported" : "unsupported"),
+        policyMapping: NOT_CHECKED,
+        interruption: NOT_CHECKED,
+        streaming: NOT_CHECKED,
+        steering: NOT_CHECKED,
+        resume: NOT_CHECKED,
+        images: NOT_CHECKED,
+        contextTelemetry: NOT_CHECKED,
         preCompactionSignal: hookSpec
-            ? hookSubcommands.has("pre-compact")
-                ? "supported"
-                : "unsupported"
-            : "unknown",
+            ? fromAdapter(hookSubcommands.has("pre-compact") ? "supported" : "unsupported")
+            : NOT_CHECKED,
         postCompactionSignal: hookSpec
-            ? hookSubcommands.has("post-compact") ||
+            ? fromAdapter(hookSubcommands.has("post-compact") ||
                 (adapter.profile.id === "claude-code" && hookSubcommands.has("session-start"))
                 ? "supported"
-                : "unsupported"
-            : "unknown",
-        compaction: "unknown",
+                : "unsupported")
+            : NOT_CHECKED,
+        compaction: NOT_CHECKED,
     };
 }
 function reconcile(declared, observed) {
@@ -143,7 +243,10 @@ function reconcile(declared, observed) {
     }
     return declared === observed ? observed : "drift";
 }
-function probeVersion(binary) {
+/** Ask an installed vendor CLI what version it is. Null when it is absent or
+ * refuses to answer. Shared with the live attestation probe so both halves of
+ * the capability story key on the same string. */
+export function probeBinaryVersion(binary) {
     const result = spawnSync(binary, ["--version"], { encoding: "utf8", timeout: 5_000 });
     if (result.error || result.status !== 0)
         return null;