harnery 0.23.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/commands/harness.d.ts +2 -1
- package/dist/commands/harness.d.ts.map +1 -1
- package/dist/commands/harness.js +105 -3
- package/dist/commands/supervisor.d.ts.map +1 -1
- package/dist/commands/supervisor.js +3 -13
- package/dist/commands/work.d.ts.map +1 -1
- package/dist/commands/work.js +2 -7
- package/dist/commands/workflow.d.ts.map +1 -1
- package/dist/commands/workflow.js +215 -17
- package/dist/core/harnesses/attest.d.ts +48 -0
- package/dist/core/harnesses/attest.d.ts.map +1 -0
- package/dist/core/harnesses/attest.js +110 -0
- package/dist/core/harnesses/attestation.d.ts +75 -0
- package/dist/core/harnesses/attestation.d.ts.map +1 -0
- package/dist/core/harnesses/attestation.js +168 -0
- package/dist/core/harnesses/bench.d.ts +23 -1
- package/dist/core/harnesses/bench.d.ts.map +1 -1
- package/dist/core/harnesses/bench.js +135 -32
- package/dist/core/harnesses/index.d.ts +6 -2
- package/dist/core/harnesses/index.d.ts.map +1 -1
- package/dist/core/harnesses/index.js +3 -1
- package/dist/core/workflow/engine.js +3 -0
- package/dist/core/workflow/index.d.ts +1 -1
- package/dist/core/workflow/index.d.ts.map +1 -1
- package/dist/core/workflow/index.js +1 -1
- package/dist/core/workflow/proof.d.ts +4 -1
- package/dist/core/workflow/proof.d.ts.map +1 -1
- package/dist/core/workflow/proof.js +3 -2
- package/dist/core/workflow/spawn-cursor.d.ts +3 -0
- package/dist/core/workflow/spawn-cursor.d.ts.map +1 -1
- package/dist/core/workflow/spawn-cursor.js +8 -4
- package/dist/core/workflow/types.d.ts +14 -0
- package/dist/core/workflow/types.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/index.d.ts +1 -1
- package/dist/core/workflow/workspaces/index.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/index.js +1 -1
- package/dist/core/workflow/workspaces/integration.d.ts +8 -0
- package/dist/core/workflow/workspaces/integration.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/integration.js +15 -1
- package/dist/core/workflow/workspaces/leases.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/leases.js +19 -2
- package/package.json +1 -1
- package/src/commands/harness.ts +123 -2
- package/src/commands/supervisor.ts +11 -15
- package/src/commands/work.ts +8 -8
- package/src/commands/workflow.ts +268 -20
- package/src/core/harnesses/attest.ts +162 -0
- package/src/core/harnesses/attestation.ts +231 -0
- package/src/core/harnesses/bench.ts +187 -32
- package/src/core/harnesses/index.ts +26 -1
- package/src/core/workflow/engine.ts +3 -0
- package/src/core/workflow/index.ts +1 -0
- package/src/core/workflow/proof.ts +7 -1
- package/src/core/workflow/spawn-cursor.ts +8 -4
- package/src/core/workflow/types.ts +15 -0
- package/src/core/workflow/workspaces/index.ts +5 -1
- package/src/core/workflow/workspaces/integration.ts +17 -1
- package/src/core/workflow/workspaces/leases.ts +24 -2
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Durable record of what an installed vendor CLI was actually observed doing
|
|
3
|
+
* (ADR 0038).
|
|
4
|
+
*
|
|
5
|
+
* The conformance bench proves Harnery's own planner and normalizer against a
|
|
6
|
+
* committed fixture. That is an adapter check, not a vendor check
|
|
7
|
+
* (ADR 0037). An attestation is the other half: one bounded live turn, its
|
|
8
|
+
* observations, and the vendor version they were observed on.
|
|
9
|
+
*
|
|
10
|
+
* A record stores structural facts only. No prompt text, no completion text,
|
|
11
|
+
* no host paths. It lives under the host's coordination directory and is never
|
|
12
|
+
* published.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import {
|
|
16
|
+
mkdirSync,
|
|
17
|
+
readdirSync,
|
|
18
|
+
readFileSync,
|
|
19
|
+
renameSync,
|
|
20
|
+
unlinkSync,
|
|
21
|
+
writeFileSync,
|
|
22
|
+
} from "node:fs";
|
|
23
|
+
import { resolve } from "node:path";
|
|
24
|
+
import { monorepoRoot } from "../agents/coord-client.ts";
|
|
25
|
+
import { stableDigest } from "../workflow/durable-record.ts";
|
|
26
|
+
import type { CapabilitySupport, HarnessId, HarnessProfile } from "./types.ts";
|
|
27
|
+
|
|
28
|
+
export const ATTESTATION_SCHEMA_VERSION = 1;
|
|
29
|
+
|
|
30
|
+
/** Dimensions one minimal live turn can honestly establish. Everything else
|
|
31
|
+
* needs a purpose-built scenario and stays outside the record rather than
|
|
32
|
+
* being guessed at. */
|
|
33
|
+
export const ATTESTABLE_DIMENSIONS = ["invocation", "finalResult", "sessionId", "cost"] as const;
|
|
34
|
+
|
|
35
|
+
export type AttestableDimension = (typeof ATTESTABLE_DIMENSIONS)[number];
|
|
36
|
+
|
|
37
|
+
export interface HarnessAttestation {
|
|
38
|
+
schema_version: number;
|
|
39
|
+
harness: HarnessId;
|
|
40
|
+
/** What the vendor binary reported when this was recorded. Staleness is
|
|
41
|
+
* keyed on this, so a vendor upgrade invalidates the record automatically. */
|
|
42
|
+
binary_version: string;
|
|
43
|
+
/** Digest of the capability declaration at record time, so an edited
|
|
44
|
+
* declaration also invalidates the record. */
|
|
45
|
+
profile_digest: string;
|
|
46
|
+
observed_at: string;
|
|
47
|
+
/** Only what the probe actually saw. A dimension absent from this map was
|
|
48
|
+
* not observed, which is not the same as unsupported. */
|
|
49
|
+
observations: Partial<Record<AttestableDimension, CapabilitySupport>>;
|
|
50
|
+
/** Over every field above. A hand-edited record fails to load. */
|
|
51
|
+
record_digest: string;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export interface AttestationStoreOptions {
|
|
55
|
+
/** Test seam and alternate host. Defaults to the resolved coord root. */
|
|
56
|
+
coordRoot?: string;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function attestationsDir(opts: AttestationStoreOptions = {}): string {
|
|
60
|
+
const root = opts.coordRoot ?? monorepoRoot();
|
|
61
|
+
if (!root) throw new Error("Not in a coord-aware repo (coord root resolved to null).");
|
|
62
|
+
return resolve(root, ".harnery", "harnesses", "attestations");
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Stable identity of a declaration, so editing a claim invalidates the
|
|
66
|
+
* attestation that was recorded against the old one. */
|
|
67
|
+
export function profileDigest(profile: HarnessProfile): string {
|
|
68
|
+
return stableDigest({
|
|
69
|
+
id: profile.id,
|
|
70
|
+
binary: profile.binary,
|
|
71
|
+
capabilities: profile.capabilities,
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function digestOf(record: Omit<HarnessAttestation, "record_digest">): string {
|
|
76
|
+
return stableDigest(record);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export function sealAttestation(
|
|
80
|
+
record: Omit<HarnessAttestation, "record_digest">,
|
|
81
|
+
): HarnessAttestation {
|
|
82
|
+
return { ...record, record_digest: digestOf(record) };
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function attestationPath(harness: HarnessId, opts: AttestationStoreOptions): string {
|
|
86
|
+
if (!/^[a-z0-9][a-z0-9._-]*$/i.test(harness)) {
|
|
87
|
+
throw new Error(`unsafe harness id for an attestation path: ${harness}`);
|
|
88
|
+
}
|
|
89
|
+
return resolve(attestationsDir(opts), `${harness}.json`);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Replace-in-place write. Unlike a workflow record an attestation is meant to
|
|
93
|
+
* be re-recorded, so this is a mutable atomic swap rather than an immutable
|
|
94
|
+
* create. */
|
|
95
|
+
export function writeAttestation(
|
|
96
|
+
record: HarnessAttestation,
|
|
97
|
+
opts: AttestationStoreOptions = {},
|
|
98
|
+
): string {
|
|
99
|
+
const path = attestationPath(record.harness, opts);
|
|
100
|
+
mkdirSync(attestationsDir(opts), { recursive: true, mode: 0o700 });
|
|
101
|
+
const temporary = `${path}.tmp-${process.pid}`;
|
|
102
|
+
try {
|
|
103
|
+
writeFileSync(temporary, `${JSON.stringify(record, null, 2)}\n`, {
|
|
104
|
+
encoding: "utf8",
|
|
105
|
+
mode: 0o600,
|
|
106
|
+
});
|
|
107
|
+
renameSync(temporary, path);
|
|
108
|
+
} catch (error) {
|
|
109
|
+
try {
|
|
110
|
+
unlinkSync(temporary);
|
|
111
|
+
} catch {
|
|
112
|
+
// The temp file may never have been created; the original error wins.
|
|
113
|
+
}
|
|
114
|
+
throw error;
|
|
115
|
+
}
|
|
116
|
+
return path;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** Null for absent, unreadable, malformed, wrong-schema, or tampered records.
|
|
120
|
+
* A record that fails its own digest is discarded rather than trusted, because
|
|
121
|
+
* the whole point of the file is that it was not hand-written. */
|
|
122
|
+
export function readAttestation(
|
|
123
|
+
harness: HarnessId,
|
|
124
|
+
opts: AttestationStoreOptions = {},
|
|
125
|
+
): HarnessAttestation | null {
|
|
126
|
+
let raw: string;
|
|
127
|
+
try {
|
|
128
|
+
raw = readFileSync(attestationPath(harness, opts), "utf8");
|
|
129
|
+
} catch {
|
|
130
|
+
return null;
|
|
131
|
+
}
|
|
132
|
+
let parsed: unknown;
|
|
133
|
+
try {
|
|
134
|
+
parsed = JSON.parse(raw);
|
|
135
|
+
} catch {
|
|
136
|
+
return null;
|
|
137
|
+
}
|
|
138
|
+
return validateAttestation(parsed, harness);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
export function validateAttestation(
|
|
142
|
+
value: unknown,
|
|
143
|
+
harness?: HarnessId,
|
|
144
|
+
): HarnessAttestation | null {
|
|
145
|
+
if (!value || typeof value !== "object") return null;
|
|
146
|
+
const record = value as HarnessAttestation;
|
|
147
|
+
if (record.schema_version !== ATTESTATION_SCHEMA_VERSION) return null;
|
|
148
|
+
if (typeof record.harness !== "string" || (harness && record.harness !== harness)) return null;
|
|
149
|
+
if (typeof record.binary_version !== "string" || typeof record.profile_digest !== "string") {
|
|
150
|
+
return null;
|
|
151
|
+
}
|
|
152
|
+
if (typeof record.observed_at !== "string" || typeof record.record_digest !== "string")
|
|
153
|
+
return null;
|
|
154
|
+
if (!record.observations || typeof record.observations !== "object") return null;
|
|
155
|
+
|
|
156
|
+
const { record_digest, ...body } = record;
|
|
157
|
+
if (digestOf(body) !== record_digest) return null;
|
|
158
|
+
return record;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** An attestation speaks only for the vendor version and declaration it was
|
|
162
|
+
* recorded against. */
|
|
163
|
+
export function isAttestationCurrent(
|
|
164
|
+
record: HarnessAttestation | null,
|
|
165
|
+
binaryVersion: string | null,
|
|
166
|
+
profile: HarnessProfile,
|
|
167
|
+
): record is HarnessAttestation {
|
|
168
|
+
if (!record || !binaryVersion) return false;
|
|
169
|
+
if (record.binary_version !== binaryVersion) return false;
|
|
170
|
+
return record.profile_digest === profileDigest(profile);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export function listAttestations(opts: AttestationStoreOptions = {}): HarnessAttestation[] {
|
|
174
|
+
let names: string[];
|
|
175
|
+
try {
|
|
176
|
+
names = readdirSync(attestationsDir(opts));
|
|
177
|
+
} catch {
|
|
178
|
+
return [];
|
|
179
|
+
}
|
|
180
|
+
const records: HarnessAttestation[] = [];
|
|
181
|
+
for (const name of names) {
|
|
182
|
+
if (!name.endsWith(".json")) continue;
|
|
183
|
+
const record = readAttestation(name.slice(0, -".json".length), opts);
|
|
184
|
+
if (record) records.push(record);
|
|
185
|
+
}
|
|
186
|
+
return records.sort((a, b) => a.harness.localeCompare(b.harness));
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** Both harness-derived proof inputs, read once, for a workflow run
|
|
190
|
+
* (ADR 0038). Callers inject the result so the engine performs no capability
|
|
191
|
+
* lookups of its own. A harness with no current attestation simply has no
|
|
192
|
+
* citation; that absence is not a proof unknown. */
|
|
193
|
+
export function harnessProofInputs(
|
|
194
|
+
profiles: readonly HarnessProfile[],
|
|
195
|
+
opts: AttestationStoreOptions & {
|
|
196
|
+
versionProbe: (binary: string) => string | null;
|
|
197
|
+
},
|
|
198
|
+
): {
|
|
199
|
+
harnessEvidence: Record<string, { toolEvidence: HarnessProfile["capabilities"]["toolEvidence"] }>;
|
|
200
|
+
harnessAttestations: Record<
|
|
201
|
+
string,
|
|
202
|
+
{ binary_version: string; observed_at: string; record_digest: string }
|
|
203
|
+
>;
|
|
204
|
+
} {
|
|
205
|
+
const harnessEvidence: Record<
|
|
206
|
+
string,
|
|
207
|
+
{ toolEvidence: HarnessProfile["capabilities"]["toolEvidence"] }
|
|
208
|
+
> = {};
|
|
209
|
+
const harnessAttestations: Record<
|
|
210
|
+
string,
|
|
211
|
+
{ binary_version: string; observed_at: string; record_digest: string }
|
|
212
|
+
> = {};
|
|
213
|
+
|
|
214
|
+
for (const profile of profiles) {
|
|
215
|
+
harnessEvidence[profile.id] = { toolEvidence: profile.capabilities.toolEvidence };
|
|
216
|
+
let record: HarnessAttestation | null = null;
|
|
217
|
+
try {
|
|
218
|
+
record = readAttestation(profile.id, opts);
|
|
219
|
+
} catch {
|
|
220
|
+
// No coord root or unreadable store: run unattested rather than fail.
|
|
221
|
+
continue;
|
|
222
|
+
}
|
|
223
|
+
if (!isAttestationCurrent(record, opts.versionProbe(profile.binary), profile)) continue;
|
|
224
|
+
harnessAttestations[profile.id] = {
|
|
225
|
+
binary_version: record.binary_version,
|
|
226
|
+
observed_at: record.observed_at,
|
|
227
|
+
record_digest: record.record_digest,
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
return { harnessEvidence, harnessAttestations };
|
|
231
|
+
}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { spawnSync } from "node:child_process";
|
|
2
2
|
import { HARNESS_SPECS } from "../hooks/harness/events.ts";
|
|
3
3
|
import type { SpawnRequest, SpawnResult } from "../workflow/types.ts";
|
|
4
|
+
import type { AttestableDimension, HarnessAttestation } from "./attestation.ts";
|
|
5
|
+
import { isAttestationCurrent, readAttestation } from "./attestation.ts";
|
|
4
6
|
import type { HarnessRegistry } from "./registry.ts";
|
|
5
7
|
import type {
|
|
6
8
|
CapabilitySupport,
|
|
@@ -19,7 +21,16 @@ export type BenchVerdict =
|
|
|
19
21
|
| "skipped"
|
|
20
22
|
| "drift";
|
|
21
23
|
|
|
22
|
-
export type BenchDimension = "registration" | "binary" | HarnessCapabilityDimension;
|
|
24
|
+
export type BenchDimension = "registration" | "binary" | "contract" | HarnessCapabilityDimension;
|
|
25
|
+
|
|
26
|
+
/** How a result was established (ADR 0037). Orthogonal to the verdict: this
|
|
27
|
+
* answers "how do we know", not "what is true".
|
|
28
|
+
*
|
|
29
|
+
* - `adapter`: checked against Harnery's own planner, normalizer, or fixture.
|
|
30
|
+
* Proves the adapter contract, says nothing about the installed vendor CLI.
|
|
31
|
+
* - `attested`: checked by observing the installed vendor CLI on this host.
|
|
32
|
+
* - `declared`: not checked. The value is the declaration, repeated. */
|
|
33
|
+
export type BenchBasis = "adapter" | "attested" | "declared";
|
|
23
34
|
|
|
24
35
|
export interface BenchResult {
|
|
25
36
|
harness: HarnessId;
|
|
@@ -27,6 +38,7 @@ export interface BenchResult {
|
|
|
27
38
|
declared: CapabilitySupport | "not_applicable";
|
|
28
39
|
observed: BenchVerdict;
|
|
29
40
|
verdict: BenchVerdict;
|
|
41
|
+
basis: BenchBasis;
|
|
30
42
|
note?: string;
|
|
31
43
|
}
|
|
32
44
|
|
|
@@ -36,6 +48,9 @@ export interface HarnessBenchReport {
|
|
|
36
48
|
harnesses: HarnessId[];
|
|
37
49
|
results: BenchResult[];
|
|
38
50
|
summary: Record<BenchVerdict, number>;
|
|
51
|
+
/** Result counts per basis, so a caller can tell a clean report from an
|
|
52
|
+
* unmeasured one without walking every result. */
|
|
53
|
+
basisSummary: Record<BenchBasis, number>;
|
|
39
54
|
drift: boolean;
|
|
40
55
|
skipped: boolean;
|
|
41
56
|
}
|
|
@@ -45,6 +60,11 @@ export interface HarnessBenchOptions {
|
|
|
45
60
|
dimensions?: readonly HarnessCapabilityDimension[];
|
|
46
61
|
/** Test seam and alternate host probe. A null version means unavailable. */
|
|
47
62
|
versionProbe?: (binary: string) => string | null;
|
|
63
|
+
/** Where to look for live attestations (ADR 0038). Defaults to the coord
|
|
64
|
+
* root. Reading them never runs a model turn. */
|
|
65
|
+
coordRoot?: string;
|
|
66
|
+
/** Test seam. Defaults to reading the attestation store. */
|
|
67
|
+
attestationReader?: (harness: HarnessId) => HarnessAttestation | null;
|
|
48
68
|
}
|
|
49
69
|
|
|
50
70
|
const EMPTY_SUMMARY: Record<BenchVerdict, number> = {
|
|
@@ -57,6 +77,42 @@ const EMPTY_SUMMARY: Record<BenchVerdict, number> = {
|
|
|
57
77
|
drift: 0,
|
|
58
78
|
};
|
|
59
79
|
|
|
80
|
+
const EMPTY_BASIS_SUMMARY: Record<BenchBasis, number> = {
|
|
81
|
+
adapter: 0,
|
|
82
|
+
attested: 0,
|
|
83
|
+
declared: 0,
|
|
84
|
+
};
|
|
85
|
+
|
|
86
|
+
/** A stored attestation only counts when it still matches the installed
|
|
87
|
+
* version and the current declaration. A stale one is ignored rather than
|
|
88
|
+
* trusted, so an upgraded vendor CLI silently drops back to adapter basis. */
|
|
89
|
+
function loadAttestation(
|
|
90
|
+
id: HarnessId,
|
|
91
|
+
version: string | null,
|
|
92
|
+
adapter: HarnessAdapter,
|
|
93
|
+
opts: HarnessBenchOptions,
|
|
94
|
+
): HarnessAttestation | null {
|
|
95
|
+
let record: HarnessAttestation | null = null;
|
|
96
|
+
try {
|
|
97
|
+
record = opts.attestationReader
|
|
98
|
+
? opts.attestationReader(id)
|
|
99
|
+
: readAttestation(id, { coordRoot: opts.coordRoot });
|
|
100
|
+
} catch {
|
|
101
|
+
// No coord root, unreadable store: the bench still runs, just unattested.
|
|
102
|
+
return null;
|
|
103
|
+
}
|
|
104
|
+
return isAttestationCurrent(record, version, adapter.profile) ? record : null;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** First version-shaped token in a string, or null when there is none.
|
|
108
|
+
* Tolerates the vendor prefixes and suffixes real CLIs print, so
|
|
109
|
+
* `codex-cli 0.144.5` and `2.1.197 (Claude Code)` both reduce to a token. */
|
|
110
|
+
function versionToken(value: string | null | undefined): string | null {
|
|
111
|
+
if (!value) return null;
|
|
112
|
+
const match = value.match(/\d+(?:\.\d+)+(?:[-.][0-9a-z]+)*/i);
|
|
113
|
+
return match ? match[0].toLowerCase() : null;
|
|
114
|
+
}
|
|
115
|
+
|
|
60
116
|
export function runHarnessBench(
|
|
61
117
|
registry: HarnessRegistry,
|
|
62
118
|
opts: HarnessBenchOptions = {},
|
|
@@ -65,7 +121,7 @@ export function runHarnessBench(
|
|
|
65
121
|
const dimensions = opts.dimensions?.length
|
|
66
122
|
? [...new Set(opts.dimensions)]
|
|
67
123
|
: [...HARNESS_CAPABILITY_DIMENSIONS];
|
|
68
|
-
const versionProbe = opts.versionProbe ??
|
|
124
|
+
const versionProbe = opts.versionProbe ?? probeBinaryVersion;
|
|
69
125
|
const results: BenchResult[] = [];
|
|
70
126
|
|
|
71
127
|
for (const id of ids) {
|
|
@@ -76,6 +132,7 @@ export function runHarnessBench(
|
|
|
76
132
|
declared: "supported",
|
|
77
133
|
observed: "supported",
|
|
78
134
|
verdict: "supported",
|
|
135
|
+
basis: "adapter",
|
|
79
136
|
note: `${adapter.profile.integrationMode}; ${adapter.profile.authModel}`,
|
|
80
137
|
});
|
|
81
138
|
|
|
@@ -86,38 +143,125 @@ export function runHarnessBench(
|
|
|
86
143
|
declared: "supported",
|
|
87
144
|
observed: version ? "supported" : "skipped",
|
|
88
145
|
verdict: version ? "supported" : "skipped",
|
|
146
|
+
// A skip establishes nothing about the vendor, so it is not an attestation.
|
|
147
|
+
basis: version ? "attested" : "declared",
|
|
89
148
|
note: version ?? `${adapter.profile.binary} not found on PATH`,
|
|
90
149
|
});
|
|
91
150
|
|
|
151
|
+
results.push(contractResult(id, adapter, version));
|
|
152
|
+
|
|
92
153
|
const observations = observeAdapter(adapter);
|
|
154
|
+
const attestation = loadAttestation(id, version, adapter, opts);
|
|
93
155
|
for (const dimension of dimensions) {
|
|
94
156
|
const claim = adapter.profile.capabilities[dimension];
|
|
95
|
-
const
|
|
157
|
+
const live = attestation?.observations[dimension as AttestableDimension];
|
|
158
|
+
// A live observation of the installed CLI outranks a fixture check of
|
|
159
|
+
// Harnery's own normalizer. Everything else keeps its adapter basis.
|
|
160
|
+
const { observed, basis }: DimensionObservation = live
|
|
161
|
+
? { observed: live, basis: "attested" }
|
|
162
|
+
: observations[dimension];
|
|
96
163
|
results.push({
|
|
97
164
|
harness: id,
|
|
98
165
|
dimension,
|
|
99
166
|
declared: claim.support,
|
|
100
167
|
observed,
|
|
101
168
|
verdict: reconcile(claim.support, observed),
|
|
102
|
-
|
|
169
|
+
basis,
|
|
170
|
+
note: live ? `attested on ${attestation?.binary_version}` : claim.note,
|
|
103
171
|
});
|
|
104
172
|
}
|
|
105
173
|
}
|
|
106
174
|
|
|
107
175
|
const summary = { ...EMPTY_SUMMARY };
|
|
108
|
-
|
|
176
|
+
const basisSummary = { ...EMPTY_BASIS_SUMMARY };
|
|
177
|
+
for (const result of results) {
|
|
178
|
+
summary[result.verdict]++;
|
|
179
|
+
basisSummary[result.basis]++;
|
|
180
|
+
}
|
|
109
181
|
return {
|
|
110
182
|
generatedAt: new Date().toISOString(),
|
|
111
183
|
mode: "offline",
|
|
112
184
|
harnesses: ids,
|
|
113
185
|
results,
|
|
114
186
|
summary,
|
|
187
|
+
basisSummary,
|
|
115
188
|
drift: summary.drift > 0,
|
|
116
189
|
skipped: summary.skipped > 0,
|
|
117
190
|
};
|
|
118
191
|
}
|
|
119
192
|
|
|
120
|
-
|
|
193
|
+
/** Compare the vendor contract a declaration was validated against with the
|
|
194
|
+
* one actually installed (ADR 0037). This is the only dimension attested
|
|
195
|
+
* without a model turn, because a version string costs nothing to read.
|
|
196
|
+
*
|
|
197
|
+
* A mismatch is `drift`, not failure: a newer vendor CLI is not automatically
|
|
198
|
+
* broken, but the declaration is no longer backed by an observation. An
|
|
199
|
+
* unrecorded or unparseable declared version stays `unknown` rather than being
|
|
200
|
+
* inferred from the installed one. */
|
|
201
|
+
function contractResult(
|
|
202
|
+
id: HarnessId,
|
|
203
|
+
adapter: HarnessAdapter,
|
|
204
|
+
version: string | null,
|
|
205
|
+
): BenchResult {
|
|
206
|
+
const recorded = adapter.profile.verified;
|
|
207
|
+
const base = { harness: id, dimension: "contract" as const, declared: "supported" as const };
|
|
208
|
+
|
|
209
|
+
if (!version) {
|
|
210
|
+
return {
|
|
211
|
+
...base,
|
|
212
|
+
observed: "skipped",
|
|
213
|
+
verdict: "skipped",
|
|
214
|
+
basis: "declared",
|
|
215
|
+
note: `${adapter.profile.binary} not found on PATH; installed contract unobservable`,
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
const declaredToken = versionToken(recorded?.version);
|
|
219
|
+
if (!declaredToken) {
|
|
220
|
+
return {
|
|
221
|
+
...base,
|
|
222
|
+
observed: "unknown",
|
|
223
|
+
verdict: "unknown",
|
|
224
|
+
basis: "declared",
|
|
225
|
+
note: recorded
|
|
226
|
+
? `verified.version "${recorded.version}" is not a version; installed ${version}`
|
|
227
|
+
: `no verified vendor contract recorded; installed ${version}`,
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
const installedToken = versionToken(version) ?? version.trim().toLowerCase();
|
|
231
|
+
if (installedToken === declaredToken) {
|
|
232
|
+
return {
|
|
233
|
+
...base,
|
|
234
|
+
observed: "supported",
|
|
235
|
+
verdict: "supported",
|
|
236
|
+
basis: "attested",
|
|
237
|
+
note: `declaration validated against ${recorded?.version} on ${recorded?.date}`,
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
return {
|
|
241
|
+
...base,
|
|
242
|
+
observed: "drift",
|
|
243
|
+
verdict: "drift",
|
|
244
|
+
basis: "attested",
|
|
245
|
+
note: `declaration validated against ${recorded?.version} (${recorded?.date}); installed ${version}`,
|
|
246
|
+
};
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
interface DimensionObservation {
|
|
250
|
+
observed: BenchVerdict;
|
|
251
|
+
basis: BenchBasis;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/** Checked against Harnery's planner, normalizer, or fixture. */
|
|
255
|
+
function fromAdapter(observed: BenchVerdict): DimensionObservation {
|
|
256
|
+
return { observed, basis: "adapter" };
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/** No check exists for this dimension yet. */
|
|
260
|
+
const NOT_CHECKED: DimensionObservation = { observed: "unknown", basis: "declared" };
|
|
261
|
+
|
|
262
|
+
function observeAdapter(
|
|
263
|
+
adapter: HarnessAdapter,
|
|
264
|
+
): Record<HarnessCapabilityDimension, DimensionObservation> {
|
|
121
265
|
const profile = adapter.profile;
|
|
122
266
|
const effort = profile.effortValues[0];
|
|
123
267
|
const request: SpawnRequest = {
|
|
@@ -154,49 +298,57 @@ function observeAdapter(adapter: HarnessAdapter): Record<HarnessCapabilityDimens
|
|
|
154
298
|
const hookSubcommands = new Set(hookSpec?.events.map((event) => event.subcommand) ?? []);
|
|
155
299
|
|
|
156
300
|
return {
|
|
157
|
-
invocation:
|
|
301
|
+
invocation: fromAdapter(
|
|
158
302
|
!planningFailed && argv[0] === profile.binary && argv.includes(request.prompt)
|
|
159
303
|
? "supported"
|
|
160
304
|
: "unsupported",
|
|
161
|
-
|
|
305
|
+
),
|
|
306
|
+
modelSelection: fromAdapter(
|
|
162
307
|
!planningFailed && argv.includes(request.model ?? "") ? "supported" : "unsupported",
|
|
163
|
-
|
|
308
|
+
),
|
|
309
|
+
effortSelection: fromAdapter(
|
|
164
310
|
effort === undefined
|
|
165
311
|
? "unsupported"
|
|
166
312
|
: !planningFailed && argv.some((arg) => arg.includes(effort))
|
|
167
313
|
? "supported"
|
|
168
314
|
: "unsupported",
|
|
169
|
-
|
|
315
|
+
),
|
|
316
|
+
maxTurns: fromAdapter(
|
|
170
317
|
!planningFailed && argv.includes(String(request.maxTurns)) ? "supported" : "unsupported",
|
|
171
|
-
|
|
172
|
-
|
|
318
|
+
),
|
|
319
|
+
finalResult: fromAdapter(finalMatches ? "supported" : "unsupported"),
|
|
320
|
+
sessionId: fromAdapter(
|
|
173
321
|
sessionObserved === "supported" && normalized?.sessionId !== fixture.sessionId
|
|
174
322
|
? "unsupported"
|
|
175
323
|
: sessionObserved,
|
|
176
|
-
|
|
324
|
+
),
|
|
325
|
+
cost: fromAdapter(
|
|
177
326
|
costObserved === "supported" && normalized?.costUsd !== fixture.costUsd
|
|
178
327
|
? "unsupported"
|
|
179
328
|
: costObserved,
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
329
|
+
),
|
|
330
|
+
toolEvidence: fromAdapter(
|
|
331
|
+
normalized && "toolEvidence" in normalized ? "supported" : "unsupported",
|
|
332
|
+
),
|
|
333
|
+
policyMapping: NOT_CHECKED,
|
|
334
|
+
interruption: NOT_CHECKED,
|
|
335
|
+
streaming: NOT_CHECKED,
|
|
336
|
+
steering: NOT_CHECKED,
|
|
337
|
+
resume: NOT_CHECKED,
|
|
338
|
+
images: NOT_CHECKED,
|
|
339
|
+
contextTelemetry: NOT_CHECKED,
|
|
188
340
|
preCompactionSignal: hookSpec
|
|
189
|
-
? hookSubcommands.has("pre-compact")
|
|
190
|
-
|
|
191
|
-
: "unsupported"
|
|
192
|
-
: "unknown",
|
|
341
|
+
? fromAdapter(hookSubcommands.has("pre-compact") ? "supported" : "unsupported")
|
|
342
|
+
: NOT_CHECKED,
|
|
193
343
|
postCompactionSignal: hookSpec
|
|
194
|
-
?
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
344
|
+
? fromAdapter(
|
|
345
|
+
hookSubcommands.has("post-compact") ||
|
|
346
|
+
(adapter.profile.id === "claude-code" && hookSubcommands.has("session-start"))
|
|
347
|
+
? "supported"
|
|
348
|
+
: "unsupported",
|
|
349
|
+
)
|
|
350
|
+
: NOT_CHECKED,
|
|
351
|
+
compaction: NOT_CHECKED,
|
|
200
352
|
};
|
|
201
353
|
}
|
|
202
354
|
|
|
@@ -207,7 +359,10 @@ function reconcile(declared: CapabilitySupport, observed: BenchVerdict): BenchVe
|
|
|
207
359
|
return declared === observed ? observed : "drift";
|
|
208
360
|
}
|
|
209
361
|
|
|
210
|
-
|
|
362
|
+
/** Ask an installed vendor CLI what version it is. Null when it is absent or
|
|
363
|
+
* refuses to answer. Shared with the live attestation probe so both halves of
|
|
364
|
+
* the capability story key on the same string. */
|
|
365
|
+
export function probeBinaryVersion(binary: string): string | null {
|
|
211
366
|
const result = spawnSync(binary, ["--version"], { encoding: "utf8", timeout: 5_000 });
|
|
212
367
|
if (result.error || result.status !== 0) return null;
|
|
213
368
|
return (result.stdout || result.stderr).trim().split("\n")[0] || "installed";
|
|
@@ -1,12 +1,37 @@
|
|
|
1
1
|
export type { Spawner, SpawnRequest, SpawnResult } from "../workflow/types.ts";
|
|
2
2
|
export type {
|
|
3
|
+
AttestationOutcome,
|
|
4
|
+
HarnessAttestationReport,
|
|
5
|
+
HarnessAttestationResult,
|
|
6
|
+
RunHarnessAttestationOptions,
|
|
7
|
+
} from "./attest.ts";
|
|
8
|
+
export { ATTESTATION_PROMPT, runHarnessAttestation } from "./attest.ts";
|
|
9
|
+
export type {
|
|
10
|
+
AttestableDimension,
|
|
11
|
+
AttestationStoreOptions,
|
|
12
|
+
HarnessAttestation,
|
|
13
|
+
} from "./attestation.ts";
|
|
14
|
+
export {
|
|
15
|
+
ATTESTABLE_DIMENSIONS,
|
|
16
|
+
ATTESTATION_SCHEMA_VERSION,
|
|
17
|
+
attestationsDir,
|
|
18
|
+
harnessProofInputs,
|
|
19
|
+
isAttestationCurrent,
|
|
20
|
+
listAttestations,
|
|
21
|
+
profileDigest,
|
|
22
|
+
readAttestation,
|
|
23
|
+
validateAttestation,
|
|
24
|
+
writeAttestation,
|
|
25
|
+
} from "./attestation.ts";
|
|
26
|
+
export type {
|
|
27
|
+
BenchBasis,
|
|
3
28
|
BenchDimension,
|
|
4
29
|
BenchResult,
|
|
5
30
|
BenchVerdict,
|
|
6
31
|
HarnessBenchOptions,
|
|
7
32
|
HarnessBenchReport,
|
|
8
33
|
} from "./bench.ts";
|
|
9
|
-
export { runHarnessBench } from "./bench.ts";
|
|
34
|
+
export { probeBinaryVersion, runHarnessBench } from "./bench.ts";
|
|
10
35
|
export type { BuiltinHarnessId } from "./profiles.ts";
|
|
11
36
|
export {
|
|
12
37
|
BUILTIN_HARNESS_IDS,
|
|
@@ -316,6 +316,7 @@ async function executeWorkflow(
|
|
|
316
316
|
agents: [],
|
|
317
317
|
evidence: [],
|
|
318
318
|
harnessEvidence: opts.harnessEvidence,
|
|
319
|
+
harnessAttestations: opts.harnessAttestations,
|
|
319
320
|
policy: policy
|
|
320
321
|
? {
|
|
321
322
|
config: policy,
|
|
@@ -1054,6 +1055,7 @@ async function executeWorkflow(
|
|
|
1054
1055
|
agents: Array.from(agentProofs.values()),
|
|
1055
1056
|
evidence: evidenceRecords,
|
|
1056
1057
|
harnessEvidence: opts.harnessEvidence,
|
|
1058
|
+
harnessAttestations: opts.harnessAttestations,
|
|
1057
1059
|
policy: policy
|
|
1058
1060
|
? {
|
|
1059
1061
|
config: policy,
|
|
@@ -1133,6 +1135,7 @@ async function executeWorkflow(
|
|
|
1133
1135
|
agents: Array.from(agentProofs.values()),
|
|
1134
1136
|
evidence: evidenceRecords,
|
|
1135
1137
|
harnessEvidence: opts.harnessEvidence,
|
|
1138
|
+
harnessAttestations: opts.harnessAttestations,
|
|
1136
1139
|
policy: policy
|
|
1137
1140
|
? {
|
|
1138
1141
|
config: policy,
|
|
@@ -16,6 +16,7 @@ import type {
|
|
|
16
16
|
AcceptanceCriterion,
|
|
17
17
|
AcceptanceResult,
|
|
18
18
|
AcceptanceSummary,
|
|
19
|
+
HarnessAttestationCitation,
|
|
19
20
|
HarnessEvidenceCapability,
|
|
20
21
|
HarnessEvidenceCoverage,
|
|
21
22
|
ResultDigest,
|
|
@@ -81,6 +82,9 @@ export interface BuildWorkflowProofInput {
|
|
|
81
82
|
agents: WorkflowAgentProof[];
|
|
82
83
|
evidence: WorkflowEvidenceRecord[];
|
|
83
84
|
harnessEvidence?: Readonly<Record<string, HarnessEvidenceCapability | undefined>>;
|
|
85
|
+
/** Live attestations backing each harness's claims (ADR 0038). Injected by
|
|
86
|
+
* the caller so proof stays free of filesystem lookups. */
|
|
87
|
+
harnessAttestations?: Readonly<Record<string, HarnessAttestationCitation | undefined>>;
|
|
84
88
|
policy?: {
|
|
85
89
|
config: Readonly<NormalizedPolicy>;
|
|
86
90
|
decisions: readonly PolicyDecision[];
|
|
@@ -222,7 +226,7 @@ export function buildWorkflowProof(input: BuildWorkflowProofInput): WorkflowProo
|
|
|
222
226
|
session_id: clippedOptional(agent.session_id, MAX_REF_CHARS),
|
|
223
227
|
error: clippedOptional(agent.error, MAX_SUMMARY_CHARS),
|
|
224
228
|
}));
|
|
225
|
-
const harnesses = buildHarnessCoverage(agents, input.harnessEvidence);
|
|
229
|
+
const harnesses = buildHarnessCoverage(agents, input.harnessEvidence, input.harnessAttestations);
|
|
226
230
|
const unknowns = buildUnknowns(agents, harnesses, repository);
|
|
227
231
|
const journal = readFileSync(input.journalPath);
|
|
228
232
|
return {
|
|
@@ -455,6 +459,7 @@ function normalizeRepoSnapshot(snapshot: RepoSnapshot): WorkflowRepoSnapshot {
|
|
|
455
459
|
function buildHarnessCoverage(
|
|
456
460
|
agents: WorkflowAgentProof[],
|
|
457
461
|
claims: Readonly<Record<string, HarnessEvidenceCapability | undefined>> | undefined,
|
|
462
|
+
attestations: Readonly<Record<string, HarnessAttestationCitation | undefined>> | undefined,
|
|
458
463
|
): HarnessEvidenceCoverage[] {
|
|
459
464
|
return [...new Set(agents.map((agent) => agent.harness))].map((harness) => {
|
|
460
465
|
const harnessAgents = agents.filter((agent) => agent.harness === harness);
|
|
@@ -469,6 +474,7 @@ function buildHarnessCoverage(
|
|
|
469
474
|
session_ids: harnessAgents.filter((agent) => agent.session_id).length,
|
|
470
475
|
costs: harnessAgents.filter((agent) => agent.cost_usd !== undefined).length,
|
|
471
476
|
},
|
|
477
|
+
...(attestations?.[harness] ? { attestation: attestations[harness] } : {}),
|
|
472
478
|
};
|
|
473
479
|
});
|
|
474
480
|
}
|