onto-mcp 0.4.12 → 0.4.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.onto/authority/core-lexicon.yaml +1 -1
- package/.onto/authority/supported-models.yaml +137 -0
- package/.onto/domains/accounting-kr/competency_qs.md +153 -153
- package/.onto/processes/reconstruct/obligation-coverage-ledger.yaml +842 -0
- package/.onto/processes/reconstruct/obligation-coverage-recorded.yaml +225 -0
- package/.onto/processes/reconstruct/ontology-seeding-and-maturation-design.md +93 -54
- package/.onto/processes/reconstruct/reconstruct-contract-registry.yaml +169 -8
- package/.onto/processes/reconstruct/reconstruct-execution-ux-contract.md +72 -1
- package/.onto/processes/reconstruct/source-profiles/spreadsheet.md +35 -0
- package/.onto/processes/review/productized-live-path.md +9 -5
- package/.onto/processes/review/prompt-execution-runner-contract.md +32 -8
- package/.onto/processes/review/review-target-profile-contract.md +47 -4
- package/.onto/processes/shared/pipeline-execution-ledger-contract.md +11 -0
- package/.onto/processes/shared/target-material-kind-contract.md +1 -1
- package/README.md +64 -1
- package/dist/cli.js +11 -1
- package/dist/core-api/reconstruct-api.js +575 -24
- package/dist/core-api/review-api.js +70 -5
- package/dist/core-api/review-progress.js +7 -0
- package/dist/core-api/runtime-observation.js +7 -0
- package/dist/core-runtime/artifact-io.js +75 -3
- package/dist/core-runtime/cli/claude-code-review-unit-executor.js +19 -8
- package/dist/core-runtime/cli/materialize-review-prompt-packets.js +68 -14
- package/dist/core-runtime/cli/prepare-review-session.js +34 -1
- package/dist/core-runtime/cli/render-review-final-output.js +2 -1
- package/dist/core-runtime/cli/run-review-prompt-execution.js +758 -48
- package/dist/core-runtime/cli/run-seats.js +18 -0
- package/dist/core-runtime/cli/runtime-submit-context.js +10 -2
- package/dist/core-runtime/cli/unit-resubmit.js +269 -0
- package/dist/core-runtime/cli/worker-structured-output.js +80 -2
- package/dist/core-runtime/discovery/host-detection.js +71 -22
- package/dist/core-runtime/discovery/review-cert-assemble.js +208 -0
- package/dist/core-runtime/discovery/review-cert-record.js +442 -0
- package/dist/core-runtime/discovery/review-execution-units.js +22 -0
- package/dist/core-runtime/discovery/seat-inventory.js +167 -0
- package/dist/core-runtime/discovery/settings-chain.js +311 -52
- package/dist/core-runtime/discovery/supported-models.js +282 -29
- package/dist/core-runtime/discovery/synthesize-cert-assemble.js +197 -0
- package/dist/core-runtime/discovery/synthesize-cert-capsule.js +399 -0
- package/dist/core-runtime/discovery/synthesize-cert-judge.js +340 -0
- package/dist/core-runtime/discovery/synthesize-cert-loop.js +234 -0
- package/dist/core-runtime/discovery/synthesize-cert-mutation.js +117 -0
- package/dist/core-runtime/discovery/synthesize-cert-packet.js +182 -0
- package/dist/core-runtime/discovery/synthesize-cert-record.js +831 -0
- package/dist/core-runtime/discovery/synthesize-cert-sampler.js +336 -0
- package/dist/core-runtime/effort-calibration-ingest.js +135 -0
- package/dist/core-runtime/llm/dispatch-breaker.js +412 -0
- package/dist/core-runtime/llm/dispatch-fallback-adapter-capabilities.js +33 -0
- package/dist/core-runtime/llm/llm-caller.js +188 -47
- package/dist/core-runtime/llm/mock-llm-realization.js +35 -4
- package/dist/core-runtime/llm/model-switcher.js +1 -1
- package/dist/core-runtime/llm/openai-responses-incomplete-error.js +46 -0
- package/dist/core-runtime/llm/sealed-dispatch-capability.js +359 -0
- package/dist/core-runtime/llm/structured-dispatch-error.js +15 -0
- package/dist/core-runtime/onboard/claude-profile-scan.js +65 -0
- package/dist/core-runtime/onboard/cli-host.js +5 -2
- package/dist/core-runtime/onboard/configure-provider.js +33 -1
- package/dist/core-runtime/onboard/host-target.js +28 -3
- package/dist/core-runtime/onboard/register.js +54 -12
- package/dist/core-runtime/pipeline-execution-ledger.js +10 -0
- package/dist/core-runtime/reconstruct/artifact-types.js +35 -0
- package/dist/core-runtime/reconstruct/claim-projection-validation.js +17 -0
- package/dist/core-runtime/reconstruct/competency-projection-contract.js +63 -0
- package/dist/core-runtime/reconstruct/comprehension-artifact.js +321 -0
- package/dist/core-runtime/reconstruct/comprehension-reduce.js +367 -0
- package/dist/core-runtime/reconstruct/comprehension-semantic-map.js +771 -0
- package/dist/core-runtime/reconstruct/contract-registry.js +20 -0
- package/dist/core-runtime/reconstruct/dispatch-fallback-artifacts.js +640 -0
- package/dist/core-runtime/reconstruct/execution-telemetry.js +83 -9
- package/dist/core-runtime/reconstruct/final-output-sections.js +89 -0
- package/dist/core-runtime/reconstruct/governing-snapshot.js +36 -1
- package/dist/core-runtime/reconstruct/leaf-reader.js +305 -0
- package/dist/core-runtime/reconstruct/llm-dispatch-failure.js +270 -0
- package/dist/core-runtime/reconstruct/llm-touch-fingerprint.js +94 -0
- package/dist/core-runtime/reconstruct/material-admission-validation.js +18 -0
- package/dist/core-runtime/reconstruct/material-profile-validation.js +14 -0
- package/dist/core-runtime/reconstruct/materialize-preparation.js +304 -18
- package/dist/core-runtime/reconstruct/maturation-validation.js +1231 -53
- package/dist/core-runtime/reconstruct/mock-llm-realization.js +107 -3
- package/dist/core-runtime/reconstruct/obligation-assertion.js +18 -0
- package/dist/core-runtime/reconstruct/ontology-seed-validation.js +27 -0
- package/dist/core-runtime/reconstruct/output-budget.js +6 -0
- package/dist/core-runtime/reconstruct/pipeline-execution-ledger.js +87 -3
- package/dist/core-runtime/reconstruct/post-seed-validation.js +118 -5
- package/dist/core-runtime/reconstruct/purpose-authority-validation.js +117 -2
- package/dist/core-runtime/reconstruct/record.js +42 -0
- package/dist/core-runtime/reconstruct/registry-verification-validation.js +16 -0
- package/dist/core-runtime/reconstruct/run-control-validation.js +1011 -34
- package/dist/core-runtime/reconstruct/run.js +5198 -536
- package/dist/core-runtime/reconstruct/seed-authoring-readiness-validation.js +54 -3
- package/dist/core-runtime/reconstruct/semantic-quality-gate.js +105 -2
- package/dist/core-runtime/reconstruct/source-observation-delta-validation.js +41 -0
- package/dist/core-runtime/reconstruct/source-observations.js +166 -0
- package/dist/core-runtime/reconstruct/source-safety-validation.js +71 -183
- package/dist/core-runtime/reconstruct/source-scout-pack-validation.js +37 -1
- package/dist/core-runtime/reconstruct/terminal-validation.js +132 -2
- package/dist/core-runtime/review/continuation-plan.js +12 -5
- package/dist/core-runtime/review/issue-artifact-runtime.js +4 -1
- package/dist/core-runtime/review/materializers.js +100 -11
- package/dist/core-runtime/review/nesting-batch.js +6 -2
- package/dist/core-runtime/review/obligation-shardability.js +138 -0
- package/dist/core-runtime/review/pipeline-execution-ledger.js +42 -2
- package/dist/core-runtime/review/review-artifact-utils.js +217 -5
- package/dist/core-runtime/review/review-execution-steps.js +32 -8
- package/dist/core-runtime/review/review-progress-contract.js +8 -0
- package/dist/core-runtime/review/review-prompt-budget.js +97 -0
- package/dist/core-runtime/review/semantic-quality-gate.js +33 -7
- package/dist/core-runtime/review/spreadsheet-review-disposition.js +94 -0
- package/dist/core-runtime/route-identity.js +182 -0
- package/dist/core-runtime/spreadsheet-header-escalation.js +204 -0
- package/dist/core-runtime/spreadsheet-structure-observer.js +2689 -0
- package/dist/core-runtime/target-material-kind.js +58 -0
- package/dist/mcp/server.js +12 -5
- package/dist/tui/app.js +167 -0
- package/dist/tui/data/event-follower.js +116 -0
- package/dist/tui/data/node-detail.js +42 -0
- package/dist/tui/data/projection-poll.js +26 -0
- package/dist/tui/data/session-discovery.js +65 -0
- package/dist/tui/index.js +108 -0
- package/dist/tui/screens/log.js +25 -0
- package/dist/tui/screens/node-detail.js +17 -0
- package/dist/tui/screens/session-selector.js +26 -0
- package/dist/tui/screens/workflow-tree.js +90 -0
- package/dist/tui/view-model/reconstruct-adapter.js +162 -0
- package/dist/tui/view-model/review-adapter.js +270 -0
- package/dist/tui/view-model/tree-view-model.js +19 -0
- package/package.json +18 -3
- package/settings.example.json +77 -16
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
import { computeReviewCertAggregates, REVIEW_CERT_ARMS, REVIEW_CERT_CONTRACT, REVIEW_CERT_CORE_CHECK_FLOOR, } from "./review-cert-record.js";
|
|
2
|
+
import { SEMANTIC_QUALITY_GATE_CHECK_IDS } from "../review/semantic-quality-gate.js";
|
|
3
|
+
/** Non-dispatch worker-CLI invocations the shim also witnesses. The claude
|
|
4
|
+
* availability probe (host-detection.ts detectClaudeBinaryAvailable) runs
|
|
5
|
+
* `claude auth status` through the same resolveClaudeBin the shim overrides,
|
|
6
|
+
* so it lands in the capture — in BOTH arms (route resolution probes hosts
|
|
7
|
+
* regardless of the arm's provider). Probes are classified by EXACT argv and
|
|
8
|
+
* skipped; every other model-less line still fails loud, and an arm whose
|
|
9
|
+
* capture holds only probes is a witness violation (availability checking is
|
|
10
|
+
* not dispatch). The codex availability probe reads ~/.codex/auth.json
|
|
11
|
+
* directly (no exec), so it never reaches the capture. */
|
|
12
|
+
const WORKER_PROBE_ARGVS = [["auth", "status"]];
|
|
13
|
+
function isWorkerProbeArgv(argv) {
|
|
14
|
+
return WORKER_PROBE_ARGVS.some((probe) => probe.length === argv.length && probe.every((item, i) => item === argv[i]));
|
|
15
|
+
}
|
|
16
|
+
/** True when a raw capture line is a witnessable worker dispatch: string
|
|
17
|
+
* argv, not an availability probe, and carrying a model knob. The cert
|
|
18
|
+
* harness uses this to decide whether a rehearsal capture already witnessed
|
|
19
|
+
* a dispatch or still needs its synthetic declaration-derived line (probes
|
|
20
|
+
* alone are not dispatch). */
|
|
21
|
+
export function isWitnessableWorkerDispatchLine(raw) {
|
|
22
|
+
const argv = typeof raw === "object" && raw !== null &&
|
|
23
|
+
Array.isArray(raw.argv)
|
|
24
|
+
? raw.argv
|
|
25
|
+
: null;
|
|
26
|
+
if (argv === null || argv.some((item) => typeof item !== "string"))
|
|
27
|
+
return false;
|
|
28
|
+
if (isWorkerProbeArgv(argv))
|
|
29
|
+
return false;
|
|
30
|
+
return parseWorkerArgv(argv).model !== null;
|
|
31
|
+
}
|
|
32
|
+
/** Extracts (model, reasoning_effort) from one shim-captured argv, covering
|
|
33
|
+
* both worker CLI flag families. codex: `-m <model>` and config overrides as
|
|
34
|
+
* `-c model_reasoning_effort=<value>` (value may be TOML-quoted). claude:
|
|
35
|
+
* `--model <model>` and `--effort <value>`
|
|
36
|
+
* (claude-code-review-unit-executor.ts runClaudeWorker). */
|
|
37
|
+
function parseWorkerArgv(argv) {
|
|
38
|
+
let model = null;
|
|
39
|
+
let effort = null;
|
|
40
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
41
|
+
const arg = argv[index];
|
|
42
|
+
if (arg === "-m" || arg === "--model") {
|
|
43
|
+
model = argv[index + 1] ?? null;
|
|
44
|
+
}
|
|
45
|
+
else if (arg === "--effort") {
|
|
46
|
+
effort = argv[index + 1] ?? null;
|
|
47
|
+
}
|
|
48
|
+
else if (arg === "-c") {
|
|
49
|
+
const override = argv[index + 1];
|
|
50
|
+
const match = override === undefined
|
|
51
|
+
? null
|
|
52
|
+
: /^model_reasoning_effort=(.*)$/.exec(override);
|
|
53
|
+
if (match)
|
|
54
|
+
effort = match[1].replace(/^"(.*)"$/, "$1");
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return { model, reasoning_effort: effort };
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Pure capture→witness projection. `captureLinesByArm` holds the PARSED JSON
|
|
61
|
+
* lines of each arm's capture file (the file-per-arm split is the arm
|
|
62
|
+
* attribution — the shim knows nothing about arms). Fail-loud on: an arm with
|
|
63
|
+
* zero invocations (declared dispatch without witness), a line without a
|
|
64
|
+
* usable argv/model, or within-arm inconsistency of model or effort.
|
|
65
|
+
*/
|
|
66
|
+
export function projectReviewCertWitness(captureLinesByArm) {
|
|
67
|
+
const violations = [];
|
|
68
|
+
const witnessed = {};
|
|
69
|
+
for (const arm of REVIEW_CERT_ARMS) {
|
|
70
|
+
const lines = captureLinesByArm[arm];
|
|
71
|
+
if (lines.length === 0) {
|
|
72
|
+
violations.push(`arm ${arm}: no captured worker invocations — a declared dispatch without witness is not certifiable`);
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
let armWitness = null;
|
|
76
|
+
let count = 0;
|
|
77
|
+
let probeCount = 0;
|
|
78
|
+
for (const [index, raw] of lines.entries()) {
|
|
79
|
+
const argv = typeof raw === "object" && raw !== null &&
|
|
80
|
+
Array.isArray(raw.argv)
|
|
81
|
+
? raw.argv
|
|
82
|
+
: null;
|
|
83
|
+
if (argv === null || argv.some((item) => typeof item !== "string")) {
|
|
84
|
+
violations.push(`arm ${arm}: capture line ${index} carries no string argv`);
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
if (isWorkerProbeArgv(argv)) {
|
|
88
|
+
probeCount += 1;
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
const parsed = parseWorkerArgv(argv);
|
|
92
|
+
if (parsed.model === null) {
|
|
93
|
+
violations.push(`arm ${arm}: capture line ${index} has no -m/--model argument — not a witnessable worker dispatch`);
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
count += 1;
|
|
97
|
+
if (armWitness === null) {
|
|
98
|
+
armWitness = { model: parsed.model, reasoning_effort: parsed.reasoning_effort };
|
|
99
|
+
}
|
|
100
|
+
else if (armWitness.model !== parsed.model ||
|
|
101
|
+
armWitness.reasoning_effort !== parsed.reasoning_effort) {
|
|
102
|
+
violations.push(`arm ${arm}: inconsistent dispatch within one arm (${JSON.stringify(armWitness)} vs ${JSON.stringify(parsed)} at line ${index})`);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
if (armWitness !== null) {
|
|
106
|
+
witnessed[arm] = {
|
|
107
|
+
model: armWitness.model,
|
|
108
|
+
...(armWitness.reasoning_effort !== null
|
|
109
|
+
? { reasoning_effort: armWitness.reasoning_effort }
|
|
110
|
+
: {}),
|
|
111
|
+
invocation_count: count,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
else {
|
|
115
|
+
// Also covers the all-probe capture, which pushes no per-line violation
|
|
116
|
+
// — without this the arm would silently vanish from the witness.
|
|
117
|
+
violations.push(`arm ${arm}: no witnessable worker dispatch in ${lines.length} capture line(s) (${probeCount} availability probe(s) skipped) — a declared dispatch without witness is not certifiable`);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
if (violations.length > 0)
|
|
121
|
+
return { witnessed: null, violations };
|
|
122
|
+
return {
|
|
123
|
+
witnessed: witnessed,
|
|
124
|
+
violations: [],
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* Declared-vs-witnessed guard (B4 effort-witness precedent): the certified
|
|
129
|
+
* dispatch must BE the dispatched dispatch, knob by knob per arm. Rule (a) of
|
|
130
|
+
* the synthesize guard carries over: an openai(codex-route) arm must witness a
|
|
131
|
+
* PRESENT reasoning_effort — a knobless codex dispatch inherits an
|
|
132
|
+
* unobservable host-TOML effort and is not certifiable as any effort.
|
|
133
|
+
*/
|
|
134
|
+
export function reviewCertWitnessGuardViolations(args) {
|
|
135
|
+
const violations = [];
|
|
136
|
+
for (const arm of REVIEW_CERT_ARMS) {
|
|
137
|
+
const declared = args.declared[arm];
|
|
138
|
+
const witnessed = args.witnessed[arm];
|
|
139
|
+
if (declared.model !== witnessed.model) {
|
|
140
|
+
violations.push(`arm ${arm}: declared model=${declared.model} but the workers dispatched ${witnessed.model}`);
|
|
141
|
+
}
|
|
142
|
+
if (declared.reasoning_effort !== witnessed.reasoning_effort) {
|
|
143
|
+
violations.push(`arm ${arm}: declared reasoning_effort=${declared.reasoning_effort ?? "(absent)"} but witnessed ${witnessed.reasoning_effort ?? "(absent)"}`);
|
|
144
|
+
}
|
|
145
|
+
if (declared.provider === "openai" && witnessed.reasoning_effort === undefined) {
|
|
146
|
+
violations.push(`arm ${arm}: openai(codex-route) arm witnessed NO reasoning_effort — the worker would inherit an unobservable host-TOML effort; declare and dispatch an explicit effort`);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
return violations;
|
|
150
|
+
}
|
|
151
|
+
/** Assembles a review-cert/v1 record: witness-projects the capture, guards it
|
|
152
|
+
* against the declaration, and declares aggregates via the SAME computation
|
|
153
|
+
* the validator recomputes with (single rate authority). */
|
|
154
|
+
export function assembleReviewCertRecord(args) {
|
|
155
|
+
const projection = projectReviewCertWitness(args.captureLinesByArm);
|
|
156
|
+
if (projection.witnessed === null) {
|
|
157
|
+
return { record: null, violations: projection.violations };
|
|
158
|
+
}
|
|
159
|
+
const guardViolations = reviewCertWitnessGuardViolations({
|
|
160
|
+
declared: args.declared,
|
|
161
|
+
witnessed: projection.witnessed,
|
|
162
|
+
});
|
|
163
|
+
if (guardViolations.length > 0) {
|
|
164
|
+
return { record: null, violations: guardViolations };
|
|
165
|
+
}
|
|
166
|
+
const aggregates = computeReviewCertAggregates(args.runs, args.fixtures.map((fixture) => fixture.fixture_id));
|
|
167
|
+
const armDispatch = (arm) => {
|
|
168
|
+
const effort = projection.witnessed[arm].reasoning_effort;
|
|
169
|
+
return effort !== undefined ? { reasoning_effort: effort } : {};
|
|
170
|
+
};
|
|
171
|
+
return {
|
|
172
|
+
record: {
|
|
173
|
+
record_contract: REVIEW_CERT_CONTRACT,
|
|
174
|
+
created_at: args.createdAt,
|
|
175
|
+
provider: args.declared.candidate.provider,
|
|
176
|
+
model: args.declared.candidate.model,
|
|
177
|
+
arm_model: {
|
|
178
|
+
baseline: {
|
|
179
|
+
provider: args.declared.baseline.provider,
|
|
180
|
+
model: args.declared.baseline.model,
|
|
181
|
+
},
|
|
182
|
+
candidate: {
|
|
183
|
+
provider: args.declared.candidate.provider,
|
|
184
|
+
model: args.declared.candidate.model,
|
|
185
|
+
},
|
|
186
|
+
},
|
|
187
|
+
arm_dispatch: {
|
|
188
|
+
baseline: armDispatch("baseline"),
|
|
189
|
+
candidate: armDispatch("candidate"),
|
|
190
|
+
},
|
|
191
|
+
declared_reps: args.declaredReps,
|
|
192
|
+
fixtures: args.fixtures,
|
|
193
|
+
gate_pin: {
|
|
194
|
+
check_universe: [...SEMANTIC_QUALITY_GATE_CHECK_IDS],
|
|
195
|
+
issue_artifacts_provided: args.issueArtifactsProvided,
|
|
196
|
+
},
|
|
197
|
+
run_controls: args.runControls,
|
|
198
|
+
runs: [...args.runs],
|
|
199
|
+
declared_aggregates: {
|
|
200
|
+
per_fixture_check: aggregates.per_fixture_check,
|
|
201
|
+
core_check_floor: REVIEW_CERT_CORE_CHECK_FLOOR,
|
|
202
|
+
quality_pass: aggregates.quality_pass,
|
|
203
|
+
},
|
|
204
|
+
reproduction: { command: args.reproductionCommand },
|
|
205
|
+
},
|
|
206
|
+
violations: [],
|
|
207
|
+
};
|
|
208
|
+
}
|
|
@@ -0,0 +1,442 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { SEMANTIC_QUALITY_GATE_CHECK_IDS, } from "../review/semantic-quality-gate.js";
|
|
3
|
+
import { SynthesizeCertDispatchConfigSchema } from "./synthesize-cert-record.js";
|
|
4
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
5
|
+
// review-cert/v1 — the evidence contract behind the `review` role
|
|
6
|
+
// (review-role registration design 2026-07-11 §4; structural precedent: B5
|
|
7
|
+
// synthesize-cert/v1). A registry entry listing `roles: [review]` must cite a
|
|
8
|
+
// record of this contract that RECOMPUTES to zero violations for the entry's
|
|
9
|
+
// (provider, model).
|
|
10
|
+
//
|
|
11
|
+
// Axes (design §4):
|
|
12
|
+
// - support: the candidate completes the review pipeline — at least
|
|
13
|
+
// `declared_reps` fully-completed runs per fixture per arm (transport
|
|
14
|
+
// losses are recorded honestly as `not_run` and do not count);
|
|
15
|
+
// rescue channels (salvage transcription / resubmit) are PINNED OFF so no
|
|
16
|
+
// other model can contribute to the candidate's completion evidence.
|
|
17
|
+
// - quality: the recall spine must pass on every completed candidate run so
|
|
18
|
+
// a known material issue is not silently lost before the main context can
|
|
19
|
+
// re-verify it. Other candidate<baseline regressions remain visible as
|
|
20
|
+
// non-blocking quality disclosures; false-positive restraint is useful but
|
|
21
|
+
// does not own registration authority.
|
|
22
|
+
// - universe pin: every completed run must emit the FULL gate check universe
|
|
23
|
+
// (the gate emits a subset when issue artifacts are absent — a shrunken
|
|
24
|
+
// universe is a violation, not a smaller comparison).
|
|
25
|
+
// Deterministic recompute stops here; candidate quality/discrimination/honesty
|
|
26
|
+
// judgment beyond these aggregates is R7 human curation, not runtime logic.
|
|
27
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
28
|
+
/** v2 (20260712-review-cert-v2-design.md): measures the PRODUCT path —
|
|
29
|
+
* run_controls must pin resubmit ON (error-spec corrective retry) and salvage
|
|
30
|
+
* OFF; per-row resubmit usage is recorded and surfaced as a NON-blocking
|
|
31
|
+
* disclosure (reviewCertResubmitDisclosure). v1 records (raw-measurement pin,
|
|
32
|
+
* both channels OFF) no longer parse as cert candidates — the only two v1
|
|
33
|
+
* records on disk are uncited failure evidence. */
|
|
34
|
+
export const REVIEW_CERT_CONTRACT = "review-cert/v2";
|
|
35
|
+
export const REVIEW_CERT_ARMS = ["baseline", "candidate"];
|
|
36
|
+
/** INV-BENCH-1 floors: a decision-grade claim needs >=3 reps and >=2 fixtures. */
|
|
37
|
+
export const REVIEW_CERT_MIN_REPS = 3;
|
|
38
|
+
export const REVIEW_CERT_MIN_FIXTURES = 2;
|
|
39
|
+
/** Recall-first core (owner revision 2026-07-12): the candidate must preserve
|
|
40
|
+
* the known issue in the source artifact, ReviewRecord, and final projection,
|
|
41
|
+
* with enough grounding for the main context to re-verify it. */
|
|
42
|
+
export const REVIEW_CERT_CORE_CHECKS = [
|
|
43
|
+
"material_issue_recall",
|
|
44
|
+
"artifact_material_issue_recall",
|
|
45
|
+
"final_result_material_issue_recall",
|
|
46
|
+
"grounding",
|
|
47
|
+
];
|
|
48
|
+
export const REVIEW_CERT_CORE_CHECK_FLOOR = 1;
|
|
49
|
+
const IdSchema = z.string().min(1).regex(/^\S+$/, "id must not contain whitespace");
|
|
50
|
+
const Sha256Schema = z.string().regex(/^[0-9a-f]{64}$/, "expected a lowercase sha256 hex");
|
|
51
|
+
const ArmModelSchema = z
|
|
52
|
+
.object({ provider: IdSchema, model: IdSchema })
|
|
53
|
+
.strict();
|
|
54
|
+
const FixtureManifestEntrySchema = z
|
|
55
|
+
.object({
|
|
56
|
+
fixture_id: IdSchema,
|
|
57
|
+
target_anchor: z.string().min(1),
|
|
58
|
+
content_sha256: Sha256Schema,
|
|
59
|
+
})
|
|
60
|
+
.strict();
|
|
61
|
+
const CheckIdSchema = z.enum(SEMANTIC_QUALITY_GATE_CHECK_IDS);
|
|
62
|
+
const RunCheckSchema = z
|
|
63
|
+
.object({
|
|
64
|
+
check_id: CheckIdSchema,
|
|
65
|
+
status: z.enum(["passed", "failed"]),
|
|
66
|
+
})
|
|
67
|
+
.strict();
|
|
68
|
+
const ReviewCertRunSchema = z
|
|
69
|
+
.object({
|
|
70
|
+
arm: z.enum(REVIEW_CERT_ARMS),
|
|
71
|
+
fixture_id: IdSchema,
|
|
72
|
+
rep: z.number().int().min(1),
|
|
73
|
+
/** `ok` = the arm completed EVERY review unit for this run; anything less
|
|
74
|
+
* (transport loss, worker death, partial units) is an honest `not_run` and
|
|
75
|
+
* carries no checks. */
|
|
76
|
+
completion: z.enum(["ok", "not_run"]),
|
|
77
|
+
units_total: z.number().int().min(1),
|
|
78
|
+
units_completed: z.number().int().min(0),
|
|
79
|
+
/** Units in this run whose dispatch used the resubmit error-spec channel
|
|
80
|
+
* (corrective retry after a whitelist rejection). Recorded on ok AND
|
|
81
|
+
* not_run rows (diagnostic); never participates in ok/floor judgments —
|
|
82
|
+
* surfaced only via the reviewCertResubmitDisclosure projection. */
|
|
83
|
+
resubmit_applied_unit_count: z.number().int().min(0),
|
|
84
|
+
checks: z.array(RunCheckSchema),
|
|
85
|
+
})
|
|
86
|
+
.strict();
|
|
87
|
+
const AggregateRateSchema = z
|
|
88
|
+
.object({
|
|
89
|
+
fixture_id: IdSchema,
|
|
90
|
+
check_id: CheckIdSchema,
|
|
91
|
+
baseline_pass_rate: z.number().min(0).max(1),
|
|
92
|
+
candidate_pass_rate: z.number().min(0).max(1),
|
|
93
|
+
})
|
|
94
|
+
.strict();
|
|
95
|
+
const ReviewCertRecordSchema = z
|
|
96
|
+
.object({
|
|
97
|
+
record_contract: z.literal(REVIEW_CERT_CONTRACT),
|
|
98
|
+
created_at: z.string().min(1),
|
|
99
|
+
/** The certified candidate — must equal the citing registry entry. */
|
|
100
|
+
provider: IdSchema,
|
|
101
|
+
model: IdSchema,
|
|
102
|
+
arm_model: z
|
|
103
|
+
.object({ baseline: ArmModelSchema, candidate: ArmModelSchema })
|
|
104
|
+
.strict(),
|
|
105
|
+
/** Witnessed dispatch knobs per arm, projected from the harness's
|
|
106
|
+
* spawn-argument capture (design §4 — B4's in-process capture does not
|
|
107
|
+
* reach the review worker path, so the review harness owns its capture). */
|
|
108
|
+
arm_dispatch: z
|
|
109
|
+
.object({
|
|
110
|
+
baseline: SynthesizeCertDispatchConfigSchema,
|
|
111
|
+
candidate: SynthesizeCertDispatchConfigSchema,
|
|
112
|
+
})
|
|
113
|
+
.strict(),
|
|
114
|
+
declared_reps: z.number().int().min(1),
|
|
115
|
+
fixtures: z.array(FixtureManifestEntrySchema).min(1),
|
|
116
|
+
gate_pin: z
|
|
117
|
+
.object({
|
|
118
|
+
check_universe: z.array(CheckIdSchema).min(1),
|
|
119
|
+
issue_artifacts_provided: z.boolean(),
|
|
120
|
+
})
|
|
121
|
+
.strict(),
|
|
122
|
+
run_controls: z
|
|
123
|
+
.object({
|
|
124
|
+
salvage_enabled: z.boolean(),
|
|
125
|
+
resubmit_enabled: z.boolean(),
|
|
126
|
+
})
|
|
127
|
+
.strict(),
|
|
128
|
+
runs: z.array(ReviewCertRunSchema).min(1),
|
|
129
|
+
declared_aggregates: z
|
|
130
|
+
.object({
|
|
131
|
+
per_fixture_check: z.array(AggregateRateSchema).min(1),
|
|
132
|
+
core_check_floor: z.number().min(0).max(1),
|
|
133
|
+
quality_pass: z.boolean(),
|
|
134
|
+
})
|
|
135
|
+
.strict(),
|
|
136
|
+
reproduction: z.object({ command: z.string().min(1) }).strict(),
|
|
137
|
+
})
|
|
138
|
+
.strict();
|
|
139
|
+
/** True when `raw` even CLAIMS to be a review-cert record — used by G7 to pick
|
|
140
|
+
* the cert record out of an entry's evidence refs (other refs may be generic
|
|
141
|
+
* benchmark files and are not parsed against this schema). */
|
|
142
|
+
export function isReviewCertCandidate(raw) {
|
|
143
|
+
return (typeof raw === "object" &&
|
|
144
|
+
raw !== null &&
|
|
145
|
+
!Array.isArray(raw) &&
|
|
146
|
+
raw.record_contract === REVIEW_CERT_CONTRACT);
|
|
147
|
+
}
|
|
148
|
+
export function parseReviewCertRecord(raw) {
|
|
149
|
+
const parsed = ReviewCertRecordSchema.safeParse(raw);
|
|
150
|
+
if (!parsed.success) {
|
|
151
|
+
return {
|
|
152
|
+
record: null,
|
|
153
|
+
violations: parsed.error.issues.map((issue) => ({
|
|
154
|
+
code: "schema_shape_invalid",
|
|
155
|
+
message: `${issue.path.join(".") || "(root)"}: ${issue.message}`,
|
|
156
|
+
subject_id: issue.path.length > 0 ? String(issue.path[0]) : null,
|
|
157
|
+
})),
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
return { record: parsed.data, violations: [] };
|
|
161
|
+
}
|
|
162
|
+
function ratesEqual(a, b) {
|
|
163
|
+
return Math.abs(a - b) < 1e-9;
|
|
164
|
+
}
|
|
165
|
+
function runKey(run) {
|
|
166
|
+
return `${run.arm}\u0000${run.fixture_id}\u0000${run.rep}`;
|
|
167
|
+
}
|
|
168
|
+
/** Pass-rate for one arm × fixture × check over COMPLETED runs only. Null when
|
|
169
|
+
* the arm×fixture has no completed run (rep_floor reports that separately). */
|
|
170
|
+
function passRate(runs, arm, fixtureId, checkId) {
|
|
171
|
+
const completed = runs.filter((run) => run.arm === arm && run.fixture_id === fixtureId && run.completion === "ok");
|
|
172
|
+
if (completed.length === 0)
|
|
173
|
+
return null;
|
|
174
|
+
const passed = completed.filter((run) => run.checks.some((check) => check.check_id === checkId && check.status === "passed")).length;
|
|
175
|
+
return passed / completed.length;
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* The ONE aggregate computation both the validator (recompute/compare) and the
|
|
179
|
+
* harness assembler (declare) consume — a second rate implementation could
|
|
180
|
+
* silently disagree with the gate. Fixture×check pairs where either arm has no
|
|
181
|
+
* completed run are OMITTED (rep_floor reports that state separately).
|
|
182
|
+
* `quality_pass` = every recall-first core check meets the absolute floor.
|
|
183
|
+
* Candidate<baseline regressions outside that decisive spine are retained in
|
|
184
|
+
* the aggregate rows and projected by {@link reviewCertQualityDisclosures}.
|
|
185
|
+
*/
|
|
186
|
+
export function computeReviewCertAggregates(runs, fixtureIds) {
|
|
187
|
+
const rows = [];
|
|
188
|
+
let qualityPass = true;
|
|
189
|
+
for (const fixtureId of fixtureIds) {
|
|
190
|
+
for (const checkId of SEMANTIC_QUALITY_GATE_CHECK_IDS) {
|
|
191
|
+
const baselineRate = passRate(runs, "baseline", fixtureId, checkId);
|
|
192
|
+
const candidateRate = passRate(runs, "candidate", fixtureId, checkId);
|
|
193
|
+
if (baselineRate === null || candidateRate === null)
|
|
194
|
+
continue;
|
|
195
|
+
rows.push({
|
|
196
|
+
fixture_id: fixtureId,
|
|
197
|
+
check_id: checkId,
|
|
198
|
+
baseline_pass_rate: baselineRate,
|
|
199
|
+
candidate_pass_rate: candidateRate,
|
|
200
|
+
});
|
|
201
|
+
if (REVIEW_CERT_CORE_CHECKS.includes(checkId) &&
|
|
202
|
+
candidateRate < REVIEW_CERT_CORE_CHECK_FLOOR &&
|
|
203
|
+
!ratesEqual(candidateRate, REVIEW_CERT_CORE_CHECK_FLOOR)) {
|
|
204
|
+
qualityPass = false;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
return { per_fixture_check: rows, quality_pass: qualityPass };
|
|
209
|
+
}
|
|
210
|
+
/** Non-blocking comparison surface for R7. These rows remain useful signals,
|
|
211
|
+
* but they cannot reject a model whose recall spine is intact; the main
|
|
212
|
+
* context re-verifies surfaced issues before acting on them. */
|
|
213
|
+
/** v2 non-blocking resubmit-usage disclosure — same regime as
|
|
214
|
+
* reviewCertQualityDisclosures: NEVER stored, NEVER part of
|
|
215
|
+
* validateReviewCertRecord; consumers surface it as WARN/info. ok rows and
|
|
216
|
+
* not_run rows are aggregated SEPARATELY: the ok-row rate is usage that
|
|
217
|
+
* contributed to real completions (denominator = ok-row units_total sum — no
|
|
218
|
+
* not_run placeholder pollution); the not_run count is diagnostic "fired but
|
|
219
|
+
* did not save the run" volume. Always emits one entry per arm, including
|
|
220
|
+
* zero usage (a zero is a claim, not an omission). */
|
|
221
|
+
export function reviewCertResubmitDisclosure(record) {
|
|
222
|
+
return REVIEW_CERT_ARMS.map((arm) => {
|
|
223
|
+
const rows = record.runs.filter((run) => run.arm === arm);
|
|
224
|
+
const okRows = rows.filter((run) => run.completion === "ok");
|
|
225
|
+
const okApplied = okRows.reduce((sum, run) => sum + run.resubmit_applied_unit_count, 0);
|
|
226
|
+
const okUnits = okRows.reduce((sum, run) => sum + run.units_total, 0);
|
|
227
|
+
const notRunApplied = rows
|
|
228
|
+
.filter((run) => run.completion === "not_run")
|
|
229
|
+
.reduce((sum, run) => sum + run.resubmit_applied_unit_count, 0);
|
|
230
|
+
return {
|
|
231
|
+
code: "resubmit_usage",
|
|
232
|
+
message: `arm ${arm}: resubmit applied on ${okApplied}/${okUnits} units across ${okRows.length} ok run(s)` +
|
|
233
|
+
` (not_run runs additionally fired ${notRunApplied} unit(s))`,
|
|
234
|
+
subject_id: arm,
|
|
235
|
+
};
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
export function reviewCertQualityDisclosures(record) {
|
|
239
|
+
const fixtureIds = [...new Set(record.fixtures.map((fixture) => fixture.fixture_id))];
|
|
240
|
+
return computeReviewCertAggregates(record.runs, fixtureIds).per_fixture_check
|
|
241
|
+
.filter((row) => row.candidate_pass_rate < row.baseline_pass_rate &&
|
|
242
|
+
!ratesEqual(row.candidate_pass_rate, row.baseline_pass_rate))
|
|
243
|
+
.map((row) => {
|
|
244
|
+
const subject = `${row.fixture_id}/${row.check_id}`;
|
|
245
|
+
return {
|
|
246
|
+
code: "metric_regression",
|
|
247
|
+
message: `${subject}: candidate pass-rate ${row.candidate_pass_rate.toFixed(4)} < baseline ${row.baseline_pass_rate.toFixed(4)}`,
|
|
248
|
+
subject_id: subject,
|
|
249
|
+
};
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* G7 recompute — every design §4 clause re-derived from the atomic runs.
|
|
254
|
+
* Returns ALL violations (never short-circuits) so a cert failure report is
|
|
255
|
+
* complete. Baseline anchoring (registry membership / self-baseline) lives in
|
|
256
|
+
* {@link reviewCertBindingViolations}, which sees the registry.
|
|
257
|
+
*/
|
|
258
|
+
export function validateReviewCertRecord(record) {
|
|
259
|
+
const violations = [];
|
|
260
|
+
const push = (code, message, subject = null) => {
|
|
261
|
+
violations.push({ code, message, subject_id: subject });
|
|
262
|
+
};
|
|
263
|
+
// The certified identity is the candidate arm — a record whose top-level
|
|
264
|
+
// (provider, model) diverges from arm_model.candidate could launder one
|
|
265
|
+
// model's runs into another entry's certification.
|
|
266
|
+
if (record.arm_model.candidate.provider !== record.provider ||
|
|
267
|
+
record.arm_model.candidate.model !== record.model) {
|
|
268
|
+
push("arm_model_mismatch", `record certifies ${record.provider}/${record.model} but the candidate arm ran ${record.arm_model.candidate.provider}/${record.arm_model.candidate.model}`);
|
|
269
|
+
}
|
|
270
|
+
// INV-BENCH-1 floors.
|
|
271
|
+
if (record.declared_reps < REVIEW_CERT_MIN_REPS) {
|
|
272
|
+
push("declared_reps_floor", `declared_reps=${record.declared_reps} is below the INV-BENCH-1 floor (${REVIEW_CERT_MIN_REPS})`);
|
|
273
|
+
}
|
|
274
|
+
const fixtureIds = record.fixtures.map((fixture) => fixture.fixture_id);
|
|
275
|
+
if (new Set(fixtureIds).size !== fixtureIds.length) {
|
|
276
|
+
push("duplicate_manifest_input", "fixtures list a duplicate fixture_id");
|
|
277
|
+
}
|
|
278
|
+
if (new Set(fixtureIds).size < REVIEW_CERT_MIN_FIXTURES) {
|
|
279
|
+
push("fixture_floor", `fixtures=${new Set(fixtureIds).size} is below the INV-BENCH-1 floor (${REVIEW_CERT_MIN_FIXTURES})`);
|
|
280
|
+
}
|
|
281
|
+
// v2 run_controls pin (20260712-review-cert-v2-design.md §1): salvage stays
|
|
282
|
+
// OFF (another model must not contribute to completion evidence), resubmit
|
|
283
|
+
// must be ON (the cert measures the product path — error-spec corrective
|
|
284
|
+
// retries by the SAME candidate model).
|
|
285
|
+
if (record.run_controls.salvage_enabled || !record.run_controls.resubmit_enabled) {
|
|
286
|
+
push("rescue_channel_not_pinned", `run_controls must pin salvage_enabled=false and resubmit_enabled=true (got salvage=${record.run_controls.salvage_enabled}, resubmit=${record.run_controls.resubmit_enabled})`);
|
|
287
|
+
}
|
|
288
|
+
// Check-universe pin (design §4 M-2): the record must pin the FULL gate
|
|
289
|
+
// universe — a shrunken pin (e.g. an artifacts-less harness) is a violation.
|
|
290
|
+
const canonical = [...SEMANTIC_QUALITY_GATE_CHECK_IDS].sort();
|
|
291
|
+
const pinned = [...new Set(record.gate_pin.check_universe)].sort();
|
|
292
|
+
if (pinned.length !== canonical.length ||
|
|
293
|
+
pinned.some((id, index) => id !== canonical[index])) {
|
|
294
|
+
push("check_universe_mismatch", `gate_pin.check_universe must equal the gate's full check universe (${canonical.length} checks); got ${pinned.length}`);
|
|
295
|
+
}
|
|
296
|
+
if (!record.gate_pin.issue_artifacts_provided) {
|
|
297
|
+
push("check_universe_mismatch", "gate_pin.issue_artifacts_provided must be true — without issue artifacts the gate emits a subset universe");
|
|
298
|
+
}
|
|
299
|
+
// Run rows: unique coordinates, manifest membership, completion honesty,
|
|
300
|
+
// full universe emission on every completed run.
|
|
301
|
+
const seen = new Set();
|
|
302
|
+
const fixtureSet = new Set(fixtureIds);
|
|
303
|
+
for (const run of record.runs) {
|
|
304
|
+
const coordinate = `${run.arm}/${run.fixture_id}/rep${run.rep}`;
|
|
305
|
+
const key = runKey(run);
|
|
306
|
+
if (seen.has(key))
|
|
307
|
+
push("duplicate_row", `duplicate run row ${coordinate}`, coordinate);
|
|
308
|
+
seen.add(key);
|
|
309
|
+
if (!fixtureSet.has(run.fixture_id)) {
|
|
310
|
+
push("row_outside_manifest", `run ${coordinate} names a fixture outside the manifest`, coordinate);
|
|
311
|
+
}
|
|
312
|
+
if (run.completion === "ok") {
|
|
313
|
+
if (run.units_completed !== run.units_total) {
|
|
314
|
+
push("units_incomplete", `run ${coordinate} claims completion=ok with ${run.units_completed}/${run.units_total} units — a partial run must be not_run`, coordinate);
|
|
315
|
+
}
|
|
316
|
+
const emitted = [...new Set(run.checks.map((check) => check.check_id))].sort();
|
|
317
|
+
if (run.checks.length !== canonical.length ||
|
|
318
|
+
emitted.length !== canonical.length ||
|
|
319
|
+
emitted.some((id, index) => id !== canonical[index])) {
|
|
320
|
+
push("check_emission_incomplete", `run ${coordinate} must emit the full check universe exactly once (${canonical.length} checks); got ${run.checks.length}`, coordinate);
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
else if (run.checks.length > 0) {
|
|
324
|
+
push("check_emission_incomplete", `run ${coordinate} is not_run but carries checks — a lost run has no check evidence`, coordinate);
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
// Support axis: enough COMPLETED runs per arm × fixture.
|
|
328
|
+
for (const arm of REVIEW_CERT_ARMS) {
|
|
329
|
+
for (const fixtureId of fixtureSet) {
|
|
330
|
+
const completed = record.runs.filter((run) => run.arm === arm && run.fixture_id === fixtureId &&
|
|
331
|
+
run.completion === "ok").length;
|
|
332
|
+
if (completed < record.declared_reps) {
|
|
333
|
+
push("rep_floor", `${arm}/${fixtureId} has ${completed} completed runs, below declared_reps=${record.declared_reps}`, `${arm}/${fixtureId}`);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
// Quality axis: recompute rates; the recall-first core must meet its absolute
|
|
338
|
+
// floor. All other candidate-vs-baseline rates stay in the record for R7
|
|
339
|
+
// disclosure but do not own deterministic registration authority.
|
|
340
|
+
const declaredByKey = new Map(record.declared_aggregates.per_fixture_check.map((row) => [
|
|
341
|
+
`${row.fixture_id}\u0000${row.check_id}`,
|
|
342
|
+
row,
|
|
343
|
+
]));
|
|
344
|
+
if (!ratesEqual(record.declared_aggregates.core_check_floor, REVIEW_CERT_CORE_CHECK_FLOOR)) {
|
|
345
|
+
push("aggregate_mismatch", `declared core_check_floor=${record.declared_aggregates.core_check_floor} does not match the contract floor ${REVIEW_CERT_CORE_CHECK_FLOOR}`);
|
|
346
|
+
}
|
|
347
|
+
const computed = computeReviewCertAggregates(record.runs, [...fixtureSet]);
|
|
348
|
+
for (const row of computed.per_fixture_check) {
|
|
349
|
+
const subject = `${row.fixture_id}/${row.check_id}`;
|
|
350
|
+
const declared = declaredByKey.get(`${row.fixture_id}\u0000${row.check_id}`);
|
|
351
|
+
if (!declared ||
|
|
352
|
+
!ratesEqual(declared.baseline_pass_rate, row.baseline_pass_rate) ||
|
|
353
|
+
!ratesEqual(declared.candidate_pass_rate, row.candidate_pass_rate)) {
|
|
354
|
+
push("aggregate_mismatch", `declared aggregate for ${subject} is missing or does not recompute (baseline=${row.baseline_pass_rate.toFixed(4)}, candidate=${row.candidate_pass_rate.toFixed(4)})`, subject);
|
|
355
|
+
}
|
|
356
|
+
if (REVIEW_CERT_CORE_CHECKS.includes(row.check_id) &&
|
|
357
|
+
row.candidate_pass_rate < REVIEW_CERT_CORE_CHECK_FLOOR &&
|
|
358
|
+
!ratesEqual(row.candidate_pass_rate, REVIEW_CERT_CORE_CHECK_FLOOR)) {
|
|
359
|
+
push("core_check_floor", `${subject}: candidate pass-rate ${row.candidate_pass_rate.toFixed(4)} is below the absolute core-check floor ${REVIEW_CERT_CORE_CHECK_FLOOR.toFixed(4)}`, subject);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
if (record.declared_aggregates.quality_pass !== computed.quality_pass) {
|
|
363
|
+
push("aggregate_mismatch", `declared quality_pass=${record.declared_aggregates.quality_pass} does not recompute (recomputed ${computed.quality_pass})`);
|
|
364
|
+
}
|
|
365
|
+
return violations;
|
|
366
|
+
}
|
|
367
|
+
/**
|
|
368
|
+
* G7 binding for the `review` role — the B5 semantics replicated verbatim
|
|
369
|
+
* (design §4): a registry entry listing `roles: [review]` must cite, among its
|
|
370
|
+
* benchmark_evidence_refs, AT LEAST ONE record that (a) self-identifies as
|
|
371
|
+
* `review-cert/v1`, (b) parses, (c) recomputes to zero violations, and
|
|
372
|
+
* (d) certifies THIS entry's (provider, model). Existential: a co-cited
|
|
373
|
+
* failing record does not veto a passing one (the G7 script surfaces it as a
|
|
374
|
+
* WARN, mirroring the synthesize binding's laxness-lens honesty note).
|
|
375
|
+
*
|
|
376
|
+
* Pure: the caller (G7 script / tests) reads the refs and passes parsed JSON.
|
|
377
|
+
*/
|
|
378
|
+
export function reviewCertBindingViolations(args) {
|
|
379
|
+
const { entry, evidenceByRef, supportedModelKeys } = args;
|
|
380
|
+
if (!entry.roles?.includes("review"))
|
|
381
|
+
return [];
|
|
382
|
+
const entryId = `${entry.provider}/${entry.model}`;
|
|
383
|
+
const candidates = entry.benchmark_evidence_refs.filter((ref) => isReviewCertCandidate(evidenceByRef.get(ref)));
|
|
384
|
+
if (candidates.length === 0) {
|
|
385
|
+
return [{
|
|
386
|
+
code: "schema_shape_invalid",
|
|
387
|
+
message: `${entryId} lists role review but cites no ${REVIEW_CERT_CONTRACT} record among benchmark_evidence_refs`,
|
|
388
|
+
subject_id: entryId,
|
|
389
|
+
}];
|
|
390
|
+
}
|
|
391
|
+
const violations = [];
|
|
392
|
+
let bound = false;
|
|
393
|
+
for (const ref of candidates) {
|
|
394
|
+
const { record, violations: parseViolations } = parseReviewCertRecord(evidenceByRef.get(ref));
|
|
395
|
+
if (!record) {
|
|
396
|
+
violations.push(...parseViolations.map((item) => ({
|
|
397
|
+
...item,
|
|
398
|
+
message: `${ref}: ${item.message}`,
|
|
399
|
+
})));
|
|
400
|
+
continue;
|
|
401
|
+
}
|
|
402
|
+
if (record.provider !== entry.provider || record.model !== entry.model) {
|
|
403
|
+
violations.push({
|
|
404
|
+
code: "aggregate_mismatch",
|
|
405
|
+
message: `${ref} certifies ${record.provider}/${record.model}, not the citing entry ${entryId}`,
|
|
406
|
+
subject_id: entryId,
|
|
407
|
+
});
|
|
408
|
+
continue;
|
|
409
|
+
}
|
|
410
|
+
const baselineKey = `${record.arm_model.baseline.provider}/${record.arm_model.baseline.model}`;
|
|
411
|
+
const candidateKey = `${record.arm_model.candidate.provider}/${record.arm_model.candidate.model}`;
|
|
412
|
+
if (baselineKey === candidateKey) {
|
|
413
|
+
violations.push({
|
|
414
|
+
code: "baseline_is_candidate",
|
|
415
|
+
message: `${ref} baseline arm ran ${baselineKey}, the SAME model as the candidate — quality disclosure would be a model-vs-itself comparison`,
|
|
416
|
+
subject_id: entryId,
|
|
417
|
+
});
|
|
418
|
+
continue;
|
|
419
|
+
}
|
|
420
|
+
if (!supportedModelKeys.has(baselineKey)) {
|
|
421
|
+
violations.push({
|
|
422
|
+
code: "baseline_not_supported",
|
|
423
|
+
message: `${ref} baseline arm ran ${baselineKey}, which is not a certified supported model — quality disclosure would rest on an unanchored baseline`,
|
|
424
|
+
subject_id: entryId,
|
|
425
|
+
});
|
|
426
|
+
continue;
|
|
427
|
+
}
|
|
428
|
+
const recomputeViolations = validateReviewCertRecord(record);
|
|
429
|
+
if (recomputeViolations.length === 0) {
|
|
430
|
+
bound = true;
|
|
431
|
+
}
|
|
432
|
+
else {
|
|
433
|
+
violations.push(...recomputeViolations.map((item) => ({
|
|
434
|
+
...item,
|
|
435
|
+
message: `${ref}: ${item.message}`,
|
|
436
|
+
})));
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
if (bound)
|
|
440
|
+
return [];
|
|
441
|
+
return violations;
|
|
442
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Single source of the review execution unit-id vocabulary. Lives in its own
|
|
3
|
+
* leaf module because TWO discovery authorities consume it without an import
|
|
4
|
+
* cycle between them: settings-chain (the zod unit schema + unit→actor
|
|
5
|
+
* inheritance) and supported-models (the dispatch-role resolver, which must
|
|
6
|
+
* bound `review.execution.units.<id>.llm` paths to KNOWN unit ids so an
|
|
7
|
+
* unknown unit path keeps the fail-closed `author` requirement — G7 walks raw
|
|
8
|
+
* parsed YAML without the strict zod layer, so unknown keys do reach it).
|
|
9
|
+
*/
|
|
10
|
+
export const REVIEW_EXECUTION_UNIT_IDS = [
|
|
11
|
+
"lens",
|
|
12
|
+
"finding_ledger",
|
|
13
|
+
"finding_relation_graph",
|
|
14
|
+
"issue_ledger",
|
|
15
|
+
"issue_stance_matrix",
|
|
16
|
+
"deliberation_plan",
|
|
17
|
+
"problem_framing",
|
|
18
|
+
"issue_stance_response",
|
|
19
|
+
"deliberation_response",
|
|
20
|
+
"deliberation_resolution",
|
|
21
|
+
"synthesis_response",
|
|
22
|
+
];
|