onto-mcp 0.4.12 → 0.4.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.onto/authority/core-lexicon.yaml +1 -1
- package/.onto/authority/supported-models.yaml +137 -0
- package/.onto/domains/accounting-kr/competency_qs.md +153 -153
- package/.onto/processes/reconstruct/obligation-coverage-ledger.yaml +842 -0
- package/.onto/processes/reconstruct/obligation-coverage-recorded.yaml +225 -0
- package/.onto/processes/reconstruct/ontology-seeding-and-maturation-design.md +93 -54
- package/.onto/processes/reconstruct/reconstruct-contract-registry.yaml +169 -8
- package/.onto/processes/reconstruct/reconstruct-execution-ux-contract.md +72 -1
- package/.onto/processes/reconstruct/source-profiles/spreadsheet.md +35 -0
- package/.onto/processes/review/productized-live-path.md +9 -5
- package/.onto/processes/review/prompt-execution-runner-contract.md +32 -8
- package/.onto/processes/review/review-target-profile-contract.md +47 -4
- package/.onto/processes/shared/pipeline-execution-ledger-contract.md +11 -0
- package/.onto/processes/shared/target-material-kind-contract.md +1 -1
- package/README.md +64 -1
- package/dist/cli.js +11 -1
- package/dist/core-api/reconstruct-api.js +575 -24
- package/dist/core-api/review-api.js +70 -5
- package/dist/core-api/review-progress.js +7 -0
- package/dist/core-api/runtime-observation.js +7 -0
- package/dist/core-runtime/artifact-io.js +75 -3
- package/dist/core-runtime/cli/claude-code-review-unit-executor.js +19 -8
- package/dist/core-runtime/cli/materialize-review-prompt-packets.js +68 -14
- package/dist/core-runtime/cli/prepare-review-session.js +34 -1
- package/dist/core-runtime/cli/render-review-final-output.js +2 -1
- package/dist/core-runtime/cli/run-review-prompt-execution.js +758 -48
- package/dist/core-runtime/cli/run-seats.js +18 -0
- package/dist/core-runtime/cli/runtime-submit-context.js +10 -2
- package/dist/core-runtime/cli/unit-resubmit.js +269 -0
- package/dist/core-runtime/cli/worker-structured-output.js +80 -2
- package/dist/core-runtime/discovery/host-detection.js +71 -22
- package/dist/core-runtime/discovery/review-cert-assemble.js +208 -0
- package/dist/core-runtime/discovery/review-cert-record.js +442 -0
- package/dist/core-runtime/discovery/review-execution-units.js +22 -0
- package/dist/core-runtime/discovery/seat-inventory.js +167 -0
- package/dist/core-runtime/discovery/settings-chain.js +311 -52
- package/dist/core-runtime/discovery/supported-models.js +282 -29
- package/dist/core-runtime/discovery/synthesize-cert-assemble.js +197 -0
- package/dist/core-runtime/discovery/synthesize-cert-capsule.js +399 -0
- package/dist/core-runtime/discovery/synthesize-cert-judge.js +340 -0
- package/dist/core-runtime/discovery/synthesize-cert-loop.js +234 -0
- package/dist/core-runtime/discovery/synthesize-cert-mutation.js +117 -0
- package/dist/core-runtime/discovery/synthesize-cert-packet.js +182 -0
- package/dist/core-runtime/discovery/synthesize-cert-record.js +831 -0
- package/dist/core-runtime/discovery/synthesize-cert-sampler.js +336 -0
- package/dist/core-runtime/effort-calibration-ingest.js +135 -0
- package/dist/core-runtime/llm/dispatch-breaker.js +412 -0
- package/dist/core-runtime/llm/dispatch-fallback-adapter-capabilities.js +33 -0
- package/dist/core-runtime/llm/llm-caller.js +188 -47
- package/dist/core-runtime/llm/mock-llm-realization.js +35 -4
- package/dist/core-runtime/llm/model-switcher.js +1 -1
- package/dist/core-runtime/llm/openai-responses-incomplete-error.js +46 -0
- package/dist/core-runtime/llm/sealed-dispatch-capability.js +359 -0
- package/dist/core-runtime/llm/structured-dispatch-error.js +15 -0
- package/dist/core-runtime/onboard/claude-profile-scan.js +65 -0
- package/dist/core-runtime/onboard/cli-host.js +5 -2
- package/dist/core-runtime/onboard/configure-provider.js +33 -1
- package/dist/core-runtime/onboard/host-target.js +28 -3
- package/dist/core-runtime/onboard/register.js +54 -12
- package/dist/core-runtime/pipeline-execution-ledger.js +10 -0
- package/dist/core-runtime/reconstruct/artifact-types.js +35 -0
- package/dist/core-runtime/reconstruct/claim-projection-validation.js +17 -0
- package/dist/core-runtime/reconstruct/competency-projection-contract.js +63 -0
- package/dist/core-runtime/reconstruct/comprehension-artifact.js +321 -0
- package/dist/core-runtime/reconstruct/comprehension-reduce.js +367 -0
- package/dist/core-runtime/reconstruct/comprehension-semantic-map.js +771 -0
- package/dist/core-runtime/reconstruct/contract-registry.js +20 -0
- package/dist/core-runtime/reconstruct/dispatch-fallback-artifacts.js +640 -0
- package/dist/core-runtime/reconstruct/execution-telemetry.js +83 -9
- package/dist/core-runtime/reconstruct/final-output-sections.js +89 -0
- package/dist/core-runtime/reconstruct/governing-snapshot.js +36 -1
- package/dist/core-runtime/reconstruct/leaf-reader.js +305 -0
- package/dist/core-runtime/reconstruct/llm-dispatch-failure.js +270 -0
- package/dist/core-runtime/reconstruct/llm-touch-fingerprint.js +94 -0
- package/dist/core-runtime/reconstruct/material-admission-validation.js +18 -0
- package/dist/core-runtime/reconstruct/material-profile-validation.js +14 -0
- package/dist/core-runtime/reconstruct/materialize-preparation.js +304 -18
- package/dist/core-runtime/reconstruct/maturation-validation.js +1231 -53
- package/dist/core-runtime/reconstruct/mock-llm-realization.js +107 -3
- package/dist/core-runtime/reconstruct/obligation-assertion.js +18 -0
- package/dist/core-runtime/reconstruct/ontology-seed-validation.js +27 -0
- package/dist/core-runtime/reconstruct/output-budget.js +6 -0
- package/dist/core-runtime/reconstruct/pipeline-execution-ledger.js +87 -3
- package/dist/core-runtime/reconstruct/post-seed-validation.js +118 -5
- package/dist/core-runtime/reconstruct/purpose-authority-validation.js +117 -2
- package/dist/core-runtime/reconstruct/record.js +42 -0
- package/dist/core-runtime/reconstruct/registry-verification-validation.js +16 -0
- package/dist/core-runtime/reconstruct/run-control-validation.js +1011 -34
- package/dist/core-runtime/reconstruct/run.js +5198 -536
- package/dist/core-runtime/reconstruct/seed-authoring-readiness-validation.js +54 -3
- package/dist/core-runtime/reconstruct/semantic-quality-gate.js +105 -2
- package/dist/core-runtime/reconstruct/source-observation-delta-validation.js +41 -0
- package/dist/core-runtime/reconstruct/source-observations.js +166 -0
- package/dist/core-runtime/reconstruct/source-safety-validation.js +71 -183
- package/dist/core-runtime/reconstruct/source-scout-pack-validation.js +37 -1
- package/dist/core-runtime/reconstruct/terminal-validation.js +132 -2
- package/dist/core-runtime/review/continuation-plan.js +12 -5
- package/dist/core-runtime/review/issue-artifact-runtime.js +4 -1
- package/dist/core-runtime/review/materializers.js +100 -11
- package/dist/core-runtime/review/nesting-batch.js +6 -2
- package/dist/core-runtime/review/obligation-shardability.js +138 -0
- package/dist/core-runtime/review/pipeline-execution-ledger.js +42 -2
- package/dist/core-runtime/review/review-artifact-utils.js +217 -5
- package/dist/core-runtime/review/review-execution-steps.js +32 -8
- package/dist/core-runtime/review/review-progress-contract.js +8 -0
- package/dist/core-runtime/review/review-prompt-budget.js +97 -0
- package/dist/core-runtime/review/semantic-quality-gate.js +33 -7
- package/dist/core-runtime/review/spreadsheet-review-disposition.js +94 -0
- package/dist/core-runtime/route-identity.js +182 -0
- package/dist/core-runtime/spreadsheet-header-escalation.js +204 -0
- package/dist/core-runtime/spreadsheet-structure-observer.js +2689 -0
- package/dist/core-runtime/target-material-kind.js +58 -0
- package/dist/mcp/server.js +12 -5
- package/dist/tui/app.js +167 -0
- package/dist/tui/data/event-follower.js +116 -0
- package/dist/tui/data/node-detail.js +42 -0
- package/dist/tui/data/projection-poll.js +26 -0
- package/dist/tui/data/session-discovery.js +65 -0
- package/dist/tui/index.js +108 -0
- package/dist/tui/screens/log.js +25 -0
- package/dist/tui/screens/node-detail.js +17 -0
- package/dist/tui/screens/session-selector.js +26 -0
- package/dist/tui/screens/workflow-tree.js +90 -0
- package/dist/tui/view-model/reconstruct-adapter.js +162 -0
- package/dist/tui/view-model/review-adapter.js +270 -0
- package/dist/tui/view-model/tree-view-model.js +19 -0
- package/package.json +18 -3
- package/settings.example.json +77 -16
|
@@ -0,0 +1,831 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `synthesize-cert/v1` evidence record — shared schema, parser, and G7 recompute
|
|
3
|
+
* validator (INV-MODEL-1 role-aware design §6.3/§6.4/§6.4a).
|
|
4
|
+
*
|
|
5
|
+
* A supported-model registry entry that lists the `semantic_map_synthesize` role
|
|
6
|
+
* must cite a record of this contract among its benchmark_evidence_refs; the G7
|
|
7
|
+
* guard (scripts/check-supported-models.ts) re-derives every §6.2 clause from the
|
|
8
|
+
* record's ATOMIC judgement rows and fails on any mismatch — declared aggregates
|
|
9
|
+
* are never trusted, only compared. This module is the SINGLE owner of the record
|
|
10
|
+
* schema and its recomputation (design §6.3 "parser 소유": the G7 script, the B4
|
|
11
|
+
* bench harness, and tests all consume this module; no local parsers).
|
|
12
|
+
*
|
|
13
|
+
* Everything here is pure and deterministic — callers do I/O and hand in parsed
|
|
14
|
+
* JSON. The B4 harness (not yet built) must PRODUCE records that pass
|
|
15
|
+
* {@link validateSynthesizeCertRecord} with zero violations; building the
|
|
16
|
+
* validator first pins the contract before any bench spend (B5-first order).
|
|
17
|
+
*/
|
|
18
|
+
import { z } from "zod";
|
|
19
|
+
export const SYNTHESIZE_CERT_CONTRACT = "synthesize-cert/v1";
|
|
20
|
+
/** Bench arms (design §6.5). The negative-control arm is defined by INPUT
|
|
21
|
+
* mutation only — every arm runs the identical system prompt (§6.2-4). */
|
|
22
|
+
export const SYNTHESIZE_CERT_ARMS = [
|
|
23
|
+
"baseline",
|
|
24
|
+
"candidate",
|
|
25
|
+
"negative_control",
|
|
26
|
+
];
|
|
27
|
+
/** Judged metrics (§6.2-3). A negative-control mutation must target BOTH. */
|
|
28
|
+
export const SYNTHESIZE_CERT_METRICS = ["grounding", "boundary"];
|
|
29
|
+
/** Input stratum flags (seam/no-seam × leaf/merge → 4 combinations, §6.2-2). */
|
|
30
|
+
const StratumSchema = z
|
|
31
|
+
.object({ seam: z.boolean(), merge: z.boolean() })
|
|
32
|
+
.strict();
|
|
33
|
+
/** Coverage floors (§6.2-1/2 · §6.4a). Named here so the recompute and its
|
|
34
|
+
* tests share one authority; changing a floor is an evidence-contract change
|
|
35
|
+
* (INVARIANT-CHANGE: INV-MODEL-1). */
|
|
36
|
+
export const SYNTHESIZE_CERT_FLOORS = {
|
|
37
|
+
minFixtures: 2,
|
|
38
|
+
minRepsPerFixtureArm: 3,
|
|
39
|
+
minDecisivePerStratumArm: 5,
|
|
40
|
+
};
|
|
41
|
+
// NOTE (§13.3 boundary re-anchor): the discrimination-delta and decisiveness-
|
|
42
|
+
// ratio constants were REMOVED — two rounds of cross-validation proved no
|
|
43
|
+
// deterministic threshold on self-declared row verdicts can judge negative-arm
|
|
44
|
+
// efficacy or selective-exclusion honesty without either over-blocking honest
|
|
45
|
+
// benches or opening a new laundering channel. Those semantic judgements now
|
|
46
|
+
// live in the R7 human-curation checklist (§6.5). The recompute keeps only the
|
|
47
|
+
// structural absolute floors above.
|
|
48
|
+
/** Ids join into space-separated coordinate keys, so whitespace inside an id
|
|
49
|
+
* could alias distinct coordinates; honest ids are sha256-derived and never
|
|
50
|
+
* contain whitespace — reject at the schema (producer-lens LOW-8 hardening). */
|
|
51
|
+
const IdSchema = z.string().min(1).regex(/^\S+$/, "id must not contain whitespace");
|
|
52
|
+
const InputManifestEntrySchema = z
|
|
53
|
+
.object({
|
|
54
|
+
fixture_id: IdSchema,
|
|
55
|
+
input_id: IdSchema,
|
|
56
|
+
input_sha256: z.string().min(1),
|
|
57
|
+
stratum: StratumSchema,
|
|
58
|
+
})
|
|
59
|
+
.strict();
|
|
60
|
+
const MetricVerdictSchema = z.enum(["pass", "fail", "not_judged"]);
|
|
61
|
+
const JudgementRowSchema = z
|
|
62
|
+
.object({
|
|
63
|
+
row_id: z.string().min(1),
|
|
64
|
+
fixture_id: IdSchema,
|
|
65
|
+
input_id: IdSchema,
|
|
66
|
+
input_sha256: z.string().min(1),
|
|
67
|
+
rep: z.number().int().positive(),
|
|
68
|
+
arm: z.enum(SYNTHESIZE_CERT_ARMS),
|
|
69
|
+
stratum: StratumSchema,
|
|
70
|
+
// Failure-plane separation (§6.3): the synthesize OUTPUT plane and the judge
|
|
71
|
+
// EXECUTION plane are recorded independently so a failed candidate is a row,
|
|
72
|
+
// never a silent drop (R8 — the legacy judge script's pre-judging drop is
|
|
73
|
+
// exactly what this contract forbids). "not_run" = the synthesize call never
|
|
74
|
+
// produced output (transport loss / crash / resume) — an honestly
|
|
75
|
+
// representable, non-decisive loss like the judge-plane losses, NOT a
|
|
76
|
+
// §6.2-1 parse/structural failure (producer-lens HIGH-1: without it, every
|
|
77
|
+
// synthesize-plane loss forced either an unpassable record or the
|
|
78
|
+
// validator-invisible {ok, not_run} lie).
|
|
79
|
+
candidate_output_status: z.enum([
|
|
80
|
+
"ok",
|
|
81
|
+
"parse_fail",
|
|
82
|
+
"structural_fail",
|
|
83
|
+
"not_run",
|
|
84
|
+
]),
|
|
85
|
+
judge_status: z.enum(["ok", "judge_error", "timeout", "not_run"]),
|
|
86
|
+
metrics: z
|
|
87
|
+
.object({
|
|
88
|
+
grounding: MetricVerdictSchema,
|
|
89
|
+
boundary: MetricVerdictSchema,
|
|
90
|
+
})
|
|
91
|
+
.strict(),
|
|
92
|
+
// Optional honesty fields (producer-lens HIGH-1/MED-3): the sha of the
|
|
93
|
+
// synthesize output an "ok" status refers to (future hardening may require
|
|
94
|
+
// it), and how many attempts this coordinate consumed (re-runs overwrite
|
|
95
|
+
// the coordinate — duplicate rows are violations — so without this count
|
|
96
|
+
// the published judge_failure_rate is residual-only by construction).
|
|
97
|
+
output_sha256: z.string().min(1).optional(),
|
|
98
|
+
attempts: z.number().int().min(1).optional(),
|
|
99
|
+
// negative arm only: which ORIGINAL manifest input this mutated input
|
|
100
|
+
// derives from (mutation lineage, §6.3 / onto issue-014).
|
|
101
|
+
source_input_id: z.string().min(1).optional(),
|
|
102
|
+
})
|
|
103
|
+
.strict()
|
|
104
|
+
.refine((row) => row.arm === "negative_control"
|
|
105
|
+
? row.source_input_id !== undefined
|
|
106
|
+
: row.source_input_id === undefined, {
|
|
107
|
+
message: "source_input_id is required on negative_control rows and forbidden elsewhere",
|
|
108
|
+
path: ["source_input_id"],
|
|
109
|
+
});
|
|
110
|
+
const NegativeArmSchema = z
|
|
111
|
+
.object({
|
|
112
|
+
arm: z.literal("negative_control"),
|
|
113
|
+
// The mutation is realized ONLY as a deterministic named transform of the
|
|
114
|
+
// bench harness; the record cites kind+params, the transform implementation
|
|
115
|
+
// owns boundedness (§6.3 / onto issue-002/007).
|
|
116
|
+
mutation_kind: z.string().min(1),
|
|
117
|
+
mutation_params: z.record(z.string(), z.unknown()),
|
|
118
|
+
targeted_metrics: z.array(z.enum(SYNTHESIZE_CERT_METRICS)).min(1),
|
|
119
|
+
})
|
|
120
|
+
.strict();
|
|
121
|
+
// One arm's WITNESSED dispatch config (effort-witness design 2026-07-10/11).
|
|
122
|
+
// Exactly the dispatch-affecting knobs the LlmCallConfig carries per provider
|
|
123
|
+
// route: `reasoning_effort` (codex/openai) and `thinking_mode` (anthropic SDK).
|
|
124
|
+
// A knob absent from the dispatched config is OMITTED (never null) — an empty
|
|
125
|
+
// cell is the witness "dispatched with no knob on the config", which the
|
|
126
|
+
// producer guard treats as certifiable only where the provider route makes it
|
|
127
|
+
// observable (it rejects a knobless codex arm: that shape means an
|
|
128
|
+
// unobservable host-TOML inherit, not a provider default).
|
|
129
|
+
const DispatchConfigSchema = z
|
|
130
|
+
.object({
|
|
131
|
+
reasoning_effort: z.string().min(1).optional(),
|
|
132
|
+
thinking_mode: z.enum(["disabled"]).optional(),
|
|
133
|
+
})
|
|
134
|
+
.strict();
|
|
135
|
+
/** Exported for the capture→arm_dispatch projection (assemble module): one
|
|
136
|
+
* schema owns the cell shape on both the record and the capture-line side. */
|
|
137
|
+
export const SynthesizeCertDispatchConfigSchema = DispatchConfigSchema;
|
|
138
|
+
const ArmMetricMeansSchema = z
|
|
139
|
+
.object({
|
|
140
|
+
// Mean = pass ratio over the arm's decisive rows; null iff the arm has no
|
|
141
|
+
// decisive rows (the recompute compares the null state too).
|
|
142
|
+
grounding: z.number().min(0).max(1).nullable(),
|
|
143
|
+
boundary: z.number().min(0).max(1).nullable(),
|
|
144
|
+
})
|
|
145
|
+
.strict();
|
|
146
|
+
const DeclaredAggregatesSchema = z
|
|
147
|
+
.object({
|
|
148
|
+
decisive_row_count: z
|
|
149
|
+
.object({
|
|
150
|
+
baseline: z.number().int().nonnegative(),
|
|
151
|
+
candidate: z.number().int().nonnegative(),
|
|
152
|
+
negative_control: z.number().int().nonnegative(),
|
|
153
|
+
})
|
|
154
|
+
.strict(),
|
|
155
|
+
metric_means: z
|
|
156
|
+
.object({
|
|
157
|
+
baseline: ArmMetricMeansSchema,
|
|
158
|
+
candidate: ArmMetricMeansSchema,
|
|
159
|
+
negative_control: ArmMetricMeansSchema,
|
|
160
|
+
})
|
|
161
|
+
.strict(),
|
|
162
|
+
// §6.2-1 promises 평균/표준편차/n and §6.2-5 makes 분산 a public field
|
|
163
|
+
// (spec-lens F1: with .strict(), omitting it would even FORBID publishing
|
|
164
|
+
// it). Bernoulli population std dev sqrt(m·(1-m)) — derivable, but the
|
|
165
|
+
// frozen contract wants it co-published; same null semantics as the mean.
|
|
166
|
+
metric_stddev: z
|
|
167
|
+
.object({
|
|
168
|
+
baseline: ArmMetricMeansSchema,
|
|
169
|
+
candidate: ArmMetricMeansSchema,
|
|
170
|
+
negative_control: ArmMetricMeansSchema,
|
|
171
|
+
})
|
|
172
|
+
.strict(),
|
|
173
|
+
// Honest co-publication (§6.2-6): judge failure rate AND its attribution
|
|
174
|
+
// (spec-lens F8 — a scalar rate alone attributes nothing), plus the
|
|
175
|
+
// per-condition repetition matrix.
|
|
176
|
+
judge_failure_rate: z.number().min(0).max(1),
|
|
177
|
+
judge_status_counts: z
|
|
178
|
+
.object({
|
|
179
|
+
ok: z.number().int().nonnegative(),
|
|
180
|
+
judge_error: z.number().int().nonnegative(),
|
|
181
|
+
timeout: z.number().int().nonnegative(),
|
|
182
|
+
not_run: z.number().int().nonnegative(),
|
|
183
|
+
})
|
|
184
|
+
.strict(),
|
|
185
|
+
reps_matrix: z
|
|
186
|
+
.array(z
|
|
187
|
+
.object({
|
|
188
|
+
fixture_id: z.string().min(1),
|
|
189
|
+
arm: z.enum(SYNTHESIZE_CERT_ARMS),
|
|
190
|
+
distinct_reps: z.number().int().nonnegative(),
|
|
191
|
+
})
|
|
192
|
+
.strict())
|
|
193
|
+
.min(1),
|
|
194
|
+
})
|
|
195
|
+
.strict();
|
|
196
|
+
const SynthesizeCertRecordSchema = z
|
|
197
|
+
.object({
|
|
198
|
+
record_contract: z.literal(SYNTHESIZE_CERT_CONTRACT),
|
|
199
|
+
created_at: z.string().min(1),
|
|
200
|
+
// The candidate under certification — G7 binds this to the registry entry
|
|
201
|
+
// (a record for another model must not certify this entry).
|
|
202
|
+
provider: z.string().min(1),
|
|
203
|
+
model: z.string().min(1),
|
|
204
|
+
// Declared rep count per (input × arm) condition; the expected row-key
|
|
205
|
+
// universe is DERIVED as manifest × declared_reps × arms (§6.3 — the
|
|
206
|
+
// producer-declared scalar `expected_judgements` is abolished). Capped so a
|
|
207
|
+
// hostile record cannot make the universe enumeration itself a DoS on the
|
|
208
|
+
// CI gate (laxness-lens F7); real benches run single-digit reps.
|
|
209
|
+
declared_reps: z.number().int().min(1).max(1000),
|
|
210
|
+
// Per-arm prompt sha256 — kept per-arm so the "every arm ran the identical
|
|
211
|
+
// prompt" clause (§6.2-4) is a falsifiable comparison, not a tautology.
|
|
212
|
+
arm_prompt_sha256: z
|
|
213
|
+
.object({
|
|
214
|
+
baseline: z.string().min(1),
|
|
215
|
+
candidate: z.string().min(1),
|
|
216
|
+
negative_control: z.string().min(1),
|
|
217
|
+
})
|
|
218
|
+
.strict(),
|
|
219
|
+
// Per-arm MODEL identity (producer-lens HIGH-2): the prompt sha proves the
|
|
220
|
+
// same prompt, but without this nothing records which model ran each arm —
|
|
221
|
+
// a baseline run on a weak model would make §6.2-5 (candidate >= baseline)
|
|
222
|
+
// unfalsifiable. The candidate cell must equal the record's top-level
|
|
223
|
+
// (provider, model) (recompute-checked); the baseline cell is a falsifiable
|
|
224
|
+
// declaration (§6.5 pins baseline = the current production model; a
|
|
225
|
+
// registry-pinning policy can harden this at B5 registration time).
|
|
226
|
+
arm_model: z
|
|
227
|
+
.object({
|
|
228
|
+
baseline: z
|
|
229
|
+
.object({ provider: z.string().min(1), model: z.string().min(1) })
|
|
230
|
+
.strict(),
|
|
231
|
+
candidate: z
|
|
232
|
+
.object({ provider: z.string().min(1), model: z.string().min(1) })
|
|
233
|
+
.strict(),
|
|
234
|
+
negative_control: z
|
|
235
|
+
.object({ provider: z.string().min(1), model: z.string().min(1) })
|
|
236
|
+
.strict(),
|
|
237
|
+
})
|
|
238
|
+
.strict(),
|
|
239
|
+
// Per-arm dispatch-config WITNESS (effort-witness design 2026-07-10/11):
|
|
240
|
+
// projected from the run's live-call capture, never copied from a
|
|
241
|
+
// declaration — the declared-vs-witnessed comparison is the producer-side
|
|
242
|
+
// guard (synthesize-cert-assemble.ts), not a validator recompute (the
|
|
243
|
+
// validator has no capture access; witness consistency is owned at
|
|
244
|
+
// assembly). BOUNDARY: this witnesses the CONFIG each arm call was
|
|
245
|
+
// dispatched WITH, not the provider-side realization (an anthropic route
|
|
246
|
+
// ignores effort once thinking is disabled; a codex route inherits a host
|
|
247
|
+
// TOML effort when the config carries none — the producer guard rejects
|
|
248
|
+
// those shapes instead of recording them). Optional: absent on records
|
|
249
|
+
// predating the field and on rejudge records of legacy runs.
|
|
250
|
+
arm_dispatch: z
|
|
251
|
+
.object({
|
|
252
|
+
baseline: DispatchConfigSchema,
|
|
253
|
+
candidate: DispatchConfigSchema,
|
|
254
|
+
negative_control: DispatchConfigSchema,
|
|
255
|
+
})
|
|
256
|
+
.strict()
|
|
257
|
+
.optional(),
|
|
258
|
+
negative_arm: NegativeArmSchema,
|
|
259
|
+
// Frozen at ORIGINAL enumeration time (§6.3): re-runs/resumes stay bound to
|
|
260
|
+
// this universe. The outer join below machine-checks the orphan-row and
|
|
261
|
+
// missing-coordinate directions ONLY — a CONSISTENT regeneration (manifest
|
|
262
|
+
// and rows shrunk together, floors still met) is not detectable from a
|
|
263
|
+
// single record; that residual binding is owned by the B4 harness plus R7
|
|
264
|
+
// human curation (spec-lens F3 — do not read this field as fully
|
|
265
|
+
// machine-enforced scope-freeze). NOTE for producers: input_id must be
|
|
266
|
+
// GLOBALLY unique across fixtures (namespace it, e.g. "<fixture>:<slice>")
|
|
267
|
+
// — per-row lookups key on bare input_id, stricter than §6.3's composite
|
|
268
|
+
// row key (fail-closed; spec-lens F5).
|
|
269
|
+
input_manifest: z.array(InputManifestEntrySchema).min(1),
|
|
270
|
+
judgement_rows: z.array(JudgementRowSchema),
|
|
271
|
+
declared_aggregates: DeclaredAggregatesSchema,
|
|
272
|
+
reproduction: z
|
|
273
|
+
.object({
|
|
274
|
+
command: z.string().min(1),
|
|
275
|
+
source_paths: z.array(z.string().min(1)).min(1),
|
|
276
|
+
limitations: z.string().min(1),
|
|
277
|
+
})
|
|
278
|
+
.strict(),
|
|
279
|
+
})
|
|
280
|
+
.strict();
|
|
281
|
+
/** True when `raw` even CLAIMS to be a synthesize-cert record — used by G7 to
|
|
282
|
+
* pick the cert record out of an entry's evidence refs (other refs may be
|
|
283
|
+
* generic benchmark files and are not parsed against this schema). */
|
|
284
|
+
export function isSynthesizeCertCandidate(raw) {
|
|
285
|
+
return (typeof raw === "object" &&
|
|
286
|
+
raw !== null &&
|
|
287
|
+
!Array.isArray(raw) &&
|
|
288
|
+
raw.record_contract === SYNTHESIZE_CERT_CONTRACT);
|
|
289
|
+
}
|
|
290
|
+
export function parseSynthesizeCertRecord(raw) {
|
|
291
|
+
const parsed = SynthesizeCertRecordSchema.safeParse(raw);
|
|
292
|
+
if (!parsed.success) {
|
|
293
|
+
return {
|
|
294
|
+
record: null,
|
|
295
|
+
violations: parsed.error.issues.map((issue) => ({
|
|
296
|
+
code: "schema_shape_invalid",
|
|
297
|
+
message: `${issue.path.join(".") || "(root)"}: ${issue.message}`,
|
|
298
|
+
subject_id: issue.path.length > 0 ? String(issue.path[0]) : null,
|
|
299
|
+
})),
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
return { record: parsed.data, violations: [] };
|
|
303
|
+
}
|
|
304
|
+
/** Decisive row (§6.3 정의 핀): the synthesize output parsed AND the judge
|
|
305
|
+
* completed — only these carry metric evidence. */
|
|
306
|
+
export function isDecisiveRow(row) {
|
|
307
|
+
return row.candidate_output_status === "ok" && row.judge_status === "ok";
|
|
308
|
+
}
|
|
309
|
+
const STRATUM_COMBOS = [
|
|
310
|
+
{ seam: false, merge: false },
|
|
311
|
+
{ seam: false, merge: true },
|
|
312
|
+
{ seam: true, merge: false },
|
|
313
|
+
{ seam: true, merge: true },
|
|
314
|
+
];
|
|
315
|
+
function stratumKey(stratum) {
|
|
316
|
+
return `seam=${stratum.seam}|merge=${stratum.merge}`;
|
|
317
|
+
}
|
|
318
|
+
function rowCoordinate(row) {
|
|
319
|
+
return `${row.fixture_id} ${row.input_id} ${row.rep} ${row.arm}`;
|
|
320
|
+
}
|
|
321
|
+
/** Pass-ratio mean over decisive rows for one arm+metric; null when the arm
|
|
322
|
+
* has no decisive rows. (A decisive row left `not_judged` is policed by its
|
|
323
|
+
* own dedicated check, not here.) */
|
|
324
|
+
function metricMean(rows, metric) {
|
|
325
|
+
const decisive = rows.filter(isDecisiveRow);
|
|
326
|
+
if (decisive.length === 0)
|
|
327
|
+
return null;
|
|
328
|
+
const passCount = decisive.filter((row) => row.metrics[metric] === "pass").length;
|
|
329
|
+
return passCount / decisive.length;
|
|
330
|
+
}
|
|
331
|
+
function meansEqual(a, b) {
|
|
332
|
+
if (a === null || b === null)
|
|
333
|
+
return a === b;
|
|
334
|
+
return Math.abs(a - b) < 1e-9;
|
|
335
|
+
}
|
|
336
|
+
/**
|
|
337
|
+
* G7 recompute (§6.4 table + §6.4a) — every §6.2 clause re-derived from the
|
|
338
|
+
* atomic rows and the frozen input manifest. Returns ALL violations (never
|
|
339
|
+
* short-circuits) so a bench failure report is complete.
|
|
340
|
+
*/
|
|
341
|
+
export function validateSynthesizeCertRecord(record) {
|
|
342
|
+
const violations = [];
|
|
343
|
+
const violation = (code, message, subjectId) => violations.push({ code, message, subject_id: subjectId });
|
|
344
|
+
// --- manifest indexes -----------------------------------------------------
|
|
345
|
+
const manifestByInputId = new Map();
|
|
346
|
+
const shaSeenByFixture = new Map();
|
|
347
|
+
for (const entry of record.input_manifest) {
|
|
348
|
+
if (manifestByInputId.has(entry.input_id)) {
|
|
349
|
+
violation("duplicate_manifest_input", `input_manifest lists input_id ${entry.input_id} more than once`, entry.input_id);
|
|
350
|
+
}
|
|
351
|
+
manifestByInputId.set(entry.input_id, entry);
|
|
352
|
+
// Same content under two ids within one fixture is an input-count inflation
|
|
353
|
+
// path (laxness-lens F3: 1 real input × 2 ids × reps clears a stratum floor
|
|
354
|
+
// one input cannot). Intra-fixture duplicate content is near-certainly a
|
|
355
|
+
// harness bug; cross-fixture collisions stay legal (a shared slice can
|
|
356
|
+
// legitimately appear in two workbooks).
|
|
357
|
+
const shaKey = `${entry.fixture_id} ${entry.input_sha256}`;
|
|
358
|
+
const priorId = shaSeenByFixture.get(shaKey);
|
|
359
|
+
if (priorId !== undefined) {
|
|
360
|
+
violation("duplicate_manifest_input", `fixture ${entry.fixture_id} lists the same input_sha256 under ids ${priorId} and ${entry.input_id}`, entry.input_id);
|
|
361
|
+
}
|
|
362
|
+
else {
|
|
363
|
+
shaSeenByFixture.set(shaKey, entry.input_id);
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
const fixtureIds = [...new Set(record.input_manifest.map((m) => m.fixture_id))];
|
|
367
|
+
const manifestInputsByFixture = new Map();
|
|
368
|
+
for (const entry of record.input_manifest) {
|
|
369
|
+
const bucket = manifestInputsByFixture.get(entry.fixture_id) ?? [];
|
|
370
|
+
bucket.push(entry.input_id);
|
|
371
|
+
manifestInputsByFixture.set(entry.fixture_id, bucket);
|
|
372
|
+
}
|
|
373
|
+
// --- §6.4 row 1: floors + output plane ------------------------------------
|
|
374
|
+
if (fixtureIds.length < SYNTHESIZE_CERT_FLOORS.minFixtures) {
|
|
375
|
+
violation("fixture_floor", `manifest has ${fixtureIds.length} distinct fixture(s); >= ${SYNTHESIZE_CERT_FLOORS.minFixtures} required`, null);
|
|
376
|
+
}
|
|
377
|
+
if (record.declared_reps < SYNTHESIZE_CERT_FLOORS.minRepsPerFixtureArm) {
|
|
378
|
+
violation("declared_reps_floor", `declared_reps ${record.declared_reps} is below the per-condition floor ${SYNTHESIZE_CERT_FLOORS.minRepsPerFixtureArm}`, null);
|
|
379
|
+
}
|
|
380
|
+
const repsByFixtureArm = new Map();
|
|
381
|
+
for (const row of record.judgement_rows) {
|
|
382
|
+
const key = `${row.fixture_id} ${row.arm}`;
|
|
383
|
+
const reps = repsByFixtureArm.get(key) ?? new Set();
|
|
384
|
+
reps.add(row.rep);
|
|
385
|
+
repsByFixtureArm.set(key, reps);
|
|
386
|
+
}
|
|
387
|
+
for (const fixtureId of fixtureIds) {
|
|
388
|
+
for (const arm of SYNTHESIZE_CERT_ARMS) {
|
|
389
|
+
const reps = repsByFixtureArm.get(`${fixtureId} ${arm}`) ?? new Set();
|
|
390
|
+
if (reps.size < SYNTHESIZE_CERT_FLOORS.minRepsPerFixtureArm) {
|
|
391
|
+
violation("rep_floor", `fixture ${fixtureId} arm ${arm} has ${reps.size} distinct rep(s); >= ${SYNTHESIZE_CERT_FLOORS.minRepsPerFixtureArm} required`, fixtureId);
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
for (const row of record.judgement_rows) {
|
|
396
|
+
// §6.2-1 zero-tolerance is scoped to parse/structural FAILURES on the
|
|
397
|
+
// certifying arms; "not_run" is a synthesize-plane LOSS (non-decisive,
|
|
398
|
+
// honestly representable — HIGH-1), bounded by the decisive floors.
|
|
399
|
+
if (row.arm !== "negative_control" &&
|
|
400
|
+
(row.candidate_output_status === "parse_fail" ||
|
|
401
|
+
row.candidate_output_status === "structural_fail")) {
|
|
402
|
+
violation("output_status_not_ok", `${row.arm} row ${row.row_id} has candidate_output_status ${row.candidate_output_status}; §6.2-1 requires 0 parse/structural failures on baseline/candidate arms`, row.row_id);
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
// --- §6.4 row 2: outer join over the expected universe ---------------------
|
|
406
|
+
const expected = new Set();
|
|
407
|
+
for (const entry of record.input_manifest) {
|
|
408
|
+
for (const arm of SYNTHESIZE_CERT_ARMS) {
|
|
409
|
+
for (let rep = 1; rep <= record.declared_reps; rep += 1) {
|
|
410
|
+
expected.add(rowCoordinate({
|
|
411
|
+
fixture_id: entry.fixture_id,
|
|
412
|
+
input_id: entry.input_id,
|
|
413
|
+
rep,
|
|
414
|
+
arm,
|
|
415
|
+
}));
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
const seenCoordinates = new Map();
|
|
420
|
+
for (const row of record.judgement_rows) {
|
|
421
|
+
const coordinate = rowCoordinate(row);
|
|
422
|
+
const prior = seenCoordinates.get(coordinate);
|
|
423
|
+
if (prior) {
|
|
424
|
+
violation("duplicate_row", `rows ${prior.row_id} and ${row.row_id} share coordinate (${row.fixture_id}, ${row.input_id}, ${row.rep}, ${row.arm})`, row.row_id);
|
|
425
|
+
}
|
|
426
|
+
else {
|
|
427
|
+
seenCoordinates.set(coordinate, row);
|
|
428
|
+
}
|
|
429
|
+
if (!expected.has(coordinate)) {
|
|
430
|
+
violation("row_outside_manifest", `row ${row.row_id} sits outside the expected universe (manifest x declared_reps x arms) — orphan row / manifest scope-shrink`, row.row_id);
|
|
431
|
+
}
|
|
432
|
+
const manifestEntry = manifestByInputId.get(row.input_id);
|
|
433
|
+
if (manifestEntry && stratumKey(manifestEntry.stratum) !== stratumKey(row.stratum)) {
|
|
434
|
+
violation("stratum_row_mismatch", `row ${row.row_id} stratum disagrees with the manifest stratum of input ${row.input_id}`, row.row_id);
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
for (const coordinate of expected) {
|
|
438
|
+
if (!seenCoordinates.has(coordinate)) {
|
|
439
|
+
const [fixtureId, inputId, rep, arm] = coordinate.split(" ");
|
|
440
|
+
violation("expected_row_missing", `expected coordinate (${fixtureId}, ${inputId}, rep ${rep}, ${arm}) has no row — silent drop`, inputId ?? null);
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
// --- §6.4a: per-fixture stratum×arm decisive coverage ----------------------
|
|
444
|
+
// Absolute decisive floor only (§13.3 boundary re-anchor): "enough judged
|
|
445
|
+
// sample per cell" is a structural quantity. The RATIO of exclusion (owner
|
|
446
|
+
// decision A / fix ③) was a semantic honesty judgement — every deterministic
|
|
447
|
+
// denominator (all-rows vs judge-attempted) either self-tripped honest losses
|
|
448
|
+
// or opened a self-declared not_run exclusion channel — so WHICH rows were
|
|
449
|
+
// excluded and whether the losses are honest is delegated to R7 (§6.5).
|
|
450
|
+
const decisiveByFixtureStratumArm = new Map();
|
|
451
|
+
for (const row of record.judgement_rows) {
|
|
452
|
+
if (!isDecisiveRow(row))
|
|
453
|
+
continue;
|
|
454
|
+
const key = `${row.fixture_id} ${stratumKey(row.stratum)} ${row.arm}`;
|
|
455
|
+
decisiveByFixtureStratumArm.set(key, (decisiveByFixtureStratumArm.get(key) ?? 0) + 1);
|
|
456
|
+
}
|
|
457
|
+
const strataMetPerCombo = new Map();
|
|
458
|
+
for (const fixtureId of fixtureIds) {
|
|
459
|
+
const possessed = new Map();
|
|
460
|
+
for (const inputId of manifestInputsByFixture.get(fixtureId) ?? []) {
|
|
461
|
+
const entry = manifestByInputId.get(inputId);
|
|
462
|
+
if (entry)
|
|
463
|
+
possessed.set(stratumKey(entry.stratum), entry.stratum);
|
|
464
|
+
}
|
|
465
|
+
for (const [key] of possessed) {
|
|
466
|
+
let fixtureMeetsFloor = true;
|
|
467
|
+
for (const arm of SYNTHESIZE_CERT_ARMS) {
|
|
468
|
+
const n = decisiveByFixtureStratumArm.get(`${fixtureId} ${key} ${arm}`) ?? 0;
|
|
469
|
+
if (n < SYNTHESIZE_CERT_FLOORS.minDecisivePerStratumArm) {
|
|
470
|
+
fixtureMeetsFloor = false;
|
|
471
|
+
violation("stratum_coverage", `fixture ${fixtureId} possesses stratum ${key} but arm ${arm} has ${n} decisive row(s); >= ${SYNTHESIZE_CERT_FLOORS.minDecisivePerStratumArm} required (fixture-possessed strata cannot dodge the floor)`, fixtureId);
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
if (fixtureMeetsFloor) {
|
|
475
|
+
strataMetPerCombo.set(key, (strataMetPerCombo.get(key) ?? 0) + 1);
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
for (const combo of STRATUM_COMBOS) {
|
|
480
|
+
const key = stratumKey(combo);
|
|
481
|
+
if ((strataMetPerCombo.get(key) ?? 0) < 1) {
|
|
482
|
+
violation("stratum_global_floor", `stratum ${key} meets the decisive floor in no fixture — every stratum must be floor-met by at least one fixture (§6.4a; a token fixture cannot stand in)`, null);
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
// --- §6.4 row 3: negative-arm STRUCTURE + lineage (§13.3 boundary re-anchor) -
|
|
486
|
+
// The negative arm's actual DISCRIMINATION EFFICACY (does the mutation really
|
|
487
|
+
// degrade the metric?) is a semantic judgement — two rounds of cross-validation
|
|
488
|
+
// proved every deterministic threshold (relative, absolute-fallback) either
|
|
489
|
+
// over-blocks honest low-baseline metrics or admits a rubber-stamp negative.
|
|
490
|
+
// So the recompute enforces only the STRUCTURE (both metrics targeted, mutation
|
|
491
|
+
// actually applied via lineage identity); efficacy is delegated to R7 (§6.5).
|
|
492
|
+
const negativeRows = record.judgement_rows.filter((row) => row.arm === "negative_control");
|
|
493
|
+
const targeted = new Set(record.negative_arm.targeted_metrics);
|
|
494
|
+
for (const metric of SYNTHESIZE_CERT_METRICS) {
|
|
495
|
+
if (!targeted.has(metric)) {
|
|
496
|
+
violation("negative_targets_incomplete", `negative_arm.targeted_metrics omits ${metric}; every judged metric must be targeted (§6.2-3)`, metric);
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
// Lineage identity (laxness-lens F1 ≡ spec-lens F4, independently converged):
|
|
500
|
+
// the expected universe already places every negative row on an ORIGINAL
|
|
501
|
+
// manifest input coordinate, so the only coherent lineage is
|
|
502
|
+
// source_input_id === input_id. The earlier per-(fixture, rep) multiset
|
|
503
|
+
// bijection admitted PERMUTED lineage — swapped slots let unmutated original
|
|
504
|
+
// content pass the slot-anchored mutation check with zero violations.
|
|
505
|
+
for (const row of negativeRows) {
|
|
506
|
+
if (!row.source_input_id)
|
|
507
|
+
continue; // schema-enforced; guard for safety
|
|
508
|
+
if (row.source_input_id !== row.input_id) {
|
|
509
|
+
violation("negative_lineage", `negative row ${row.row_id} cites source_input_id ${row.source_input_id} but sits on coordinate input ${row.input_id}; a negative row mutates its OWN coordinate input`, row.row_id);
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
// --- §6.4 row 4: prompt/input axis separation ------------------------------
|
|
513
|
+
const promptShas = new Set(Object.values(record.arm_prompt_sha256));
|
|
514
|
+
if (promptShas.size !== 1) {
|
|
515
|
+
violation("prompt_sha_mismatch", "arm_prompt_sha256 values differ across arms; every arm must run the identical system prompt (§6.2-4)", null);
|
|
516
|
+
}
|
|
517
|
+
// arm_model internal consistency (HIGH-2): the candidate cell IS the model
|
|
518
|
+
// under certification — a divergence means the record certifies one model
|
|
519
|
+
// while its rows ran another.
|
|
520
|
+
if (record.arm_model.candidate.provider !== record.provider ||
|
|
521
|
+
record.arm_model.candidate.model !== record.model) {
|
|
522
|
+
violation("arm_model_mismatch", `arm_model.candidate is ${record.arm_model.candidate.provider}/${record.arm_model.candidate.model} but the record certifies ${record.provider}/${record.model}`, null);
|
|
523
|
+
}
|
|
524
|
+
const allManifestShas = new Set(record.input_manifest.map((entry) => entry.input_sha256));
|
|
525
|
+
for (const row of record.judgement_rows) {
|
|
526
|
+
const manifestEntry = manifestByInputId.get(row.input_id);
|
|
527
|
+
if (!manifestEntry)
|
|
528
|
+
continue; // reported as row_outside_manifest above
|
|
529
|
+
if (row.arm === "negative_control") {
|
|
530
|
+
// N16 + laxness-lens F1: mutated content must differ not only from its
|
|
531
|
+
// OWN original but from EVERY manifest original — a negative sha equal to
|
|
532
|
+
// any manifest sha means unmutated original content was smuggled into
|
|
533
|
+
// the negative arm (e.g. a permuted/copy-paste harness bug).
|
|
534
|
+
if (allManifestShas.has(row.input_sha256)) {
|
|
535
|
+
violation("negative_mutation_not_applied", `negative row ${row.row_id} input sha equals a manifest original sha — the declared mutation was not applied (N16)`, row.row_id);
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
else if (row.input_sha256 !== manifestEntry.input_sha256) {
|
|
539
|
+
violation("input_sha_mismatch", `${row.arm} row ${row.row_id} input sha differs from the manifest sha of ${row.input_id}; baseline/candidate must run the original inputs (§6.2-4)`, row.row_id);
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
// --- metric verdict completeness on decisive rows ---------------------------
|
|
543
|
+
for (const row of record.judgement_rows) {
|
|
544
|
+
if (!isDecisiveRow(row))
|
|
545
|
+
continue;
|
|
546
|
+
for (const metric of SYNTHESIZE_CERT_METRICS) {
|
|
547
|
+
if (row.metrics[metric] === "not_judged") {
|
|
548
|
+
violation("metric_not_judged_on_decisive", `decisive row ${row.row_id} carries not_judged for ${metric}; a completed judgement must verdict every metric`, row.row_id);
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
// --- §6.4 row 5: candidate >= baseline per metric ---------------------------
|
|
553
|
+
// Computed deterministically, but its MEANING rests on the baseline/candidate
|
|
554
|
+
// verdicts being genuine — that authenticity is R7 (§6.5/§13.3), not enforced
|
|
555
|
+
// here. baseline identity (≠ candidate, supported) is checked at G7 binding.
|
|
556
|
+
const baselineRows = record.judgement_rows.filter((row) => row.arm === "baseline");
|
|
557
|
+
const candidateRows = record.judgement_rows.filter((row) => row.arm === "candidate");
|
|
558
|
+
for (const metric of SYNTHESIZE_CERT_METRICS) {
|
|
559
|
+
const baselineMean = metricMean(baselineRows, metric);
|
|
560
|
+
const candidateMean = metricMean(candidateRows, metric);
|
|
561
|
+
if (baselineMean !== null &&
|
|
562
|
+
candidateMean !== null &&
|
|
563
|
+
candidateMean < baselineMean) {
|
|
564
|
+
violation("metric_regression", `candidate mean ${candidateMean} < baseline mean ${baselineMean} for ${metric} (§6.2-5)`, metric);
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
// --- schema row: declared aggregates vs recompute ---------------------------
|
|
568
|
+
// The recompute is the SAME exported helper the B4 harness must use to fill
|
|
569
|
+
// declared_aggregates (producer-lens LOW-7): the declared-vs-recomputed
|
|
570
|
+
// comparison then guards post-hoc row tampering, not logic divergence.
|
|
571
|
+
const computed = computeSynthesizeCertAggregates({
|
|
572
|
+
inputManifest: record.input_manifest,
|
|
573
|
+
judgementRows: record.judgement_rows,
|
|
574
|
+
});
|
|
575
|
+
for (const arm of SYNTHESIZE_CERT_ARMS) {
|
|
576
|
+
const declaredCount = record.declared_aggregates.decisive_row_count[arm];
|
|
577
|
+
const recomputedCount = computed.decisive_row_count[arm];
|
|
578
|
+
if (declaredCount !== recomputedCount) {
|
|
579
|
+
violation("aggregate_mismatch", `declared decisive_row_count.${arm}=${declaredCount} but rows recompute to ${recomputedCount}`, arm);
|
|
580
|
+
}
|
|
581
|
+
for (const metric of SYNTHESIZE_CERT_METRICS) {
|
|
582
|
+
const declaredMean = record.declared_aggregates.metric_means[arm][metric];
|
|
583
|
+
const recomputedMean = computed.metric_means[arm][metric];
|
|
584
|
+
if (!meansEqual(declaredMean, recomputedMean)) {
|
|
585
|
+
violation("aggregate_mismatch", `declared metric_means.${arm}.${metric}=${String(declaredMean)} but rows recompute to ${String(recomputedMean)}`, arm);
|
|
586
|
+
}
|
|
587
|
+
const declaredStddev = record.declared_aggregates.metric_stddev[arm][metric];
|
|
588
|
+
const recomputedStddev = computed.metric_stddev[arm][metric];
|
|
589
|
+
if (!meansEqual(declaredStddev, recomputedStddev)) {
|
|
590
|
+
violation("aggregate_mismatch", `declared metric_stddev.${arm}.${metric}=${String(declaredStddev)} but rows recompute to ${String(recomputedStddev)}`, arm);
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
if (!meansEqual(record.declared_aggregates.judge_failure_rate, computed.judge_failure_rate)) {
|
|
595
|
+
violation("aggregate_mismatch", `declared judge_failure_rate=${record.declared_aggregates.judge_failure_rate} but rows recompute to ${computed.judge_failure_rate}`, null);
|
|
596
|
+
}
|
|
597
|
+
for (const status of ["ok", "judge_error", "timeout", "not_run"]) {
|
|
598
|
+
if (record.declared_aggregates.judge_status_counts[status] !==
|
|
599
|
+
computed.judge_status_counts[status]) {
|
|
600
|
+
violation("aggregate_mismatch", `declared judge_status_counts.${status}=${record.declared_aggregates.judge_status_counts[status]} but rows recompute to ${computed.judge_status_counts[status]}`, status);
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
const declaredRepsMatrix = new Map();
|
|
604
|
+
for (const cell of record.declared_aggregates.reps_matrix) {
|
|
605
|
+
const key = `${cell.fixture_id} ${cell.arm}`;
|
|
606
|
+
if (declaredRepsMatrix.has(key)) {
|
|
607
|
+
// Last-wins Map collapse would silently ignore a contradictory declared
|
|
608
|
+
// cell (laxness-lens F6) — duplicates are violations.
|
|
609
|
+
violation("aggregate_mismatch", `reps_matrix declares cell (${cell.fixture_id}, ${cell.arm}) more than once`, cell.fixture_id);
|
|
610
|
+
}
|
|
611
|
+
declaredRepsMatrix.set(key, cell.distinct_reps);
|
|
612
|
+
}
|
|
613
|
+
const expectedRepsKeys = new Set();
|
|
614
|
+
for (const cell of computed.reps_matrix) {
|
|
615
|
+
const key = `${cell.fixture_id} ${cell.arm}`;
|
|
616
|
+
expectedRepsKeys.add(key);
|
|
617
|
+
const declared = declaredRepsMatrix.get(key);
|
|
618
|
+
if (declared === undefined || declared !== cell.distinct_reps) {
|
|
619
|
+
violation("aggregate_mismatch", `reps_matrix cell (${cell.fixture_id}, ${cell.arm}) declared=${String(declared)} recomputed=${cell.distinct_reps}`, cell.fixture_id);
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
for (const key of declaredRepsMatrix.keys()) {
|
|
623
|
+
if (!expectedRepsKeys.has(key)) {
|
|
624
|
+
const [fixtureId, arm] = key.split(" ");
|
|
625
|
+
violation("aggregate_mismatch", `reps_matrix cites (${fixtureId}, ${arm}) which is not a manifest fixture x arm cell`, fixtureId ?? null);
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
return violations;
|
|
629
|
+
}
|
|
630
|
+
/**
|
|
631
|
+
* Recomputes the declared_aggregates block from the atomic rows + manifest —
|
|
632
|
+
* the single shared implementation (§6.3 parser-ownership): the B4 harness
|
|
633
|
+
* fills declared_aggregates by CALLING this, and the validator compares the
|
|
634
|
+
* record's declared block against the same computation, so the comparison can
|
|
635
|
+
* only fail on post-hoc tampering (or a harness that refused to use it).
|
|
636
|
+
*/
|
|
637
|
+
export function computeSynthesizeCertAggregates(args) {
|
|
638
|
+
const armRows = {
|
|
639
|
+
baseline: [],
|
|
640
|
+
candidate: [],
|
|
641
|
+
negative_control: [],
|
|
642
|
+
};
|
|
643
|
+
for (const row of args.judgementRows)
|
|
644
|
+
armRows[row.arm].push(row);
|
|
645
|
+
const decisiveRowCount = {
|
|
646
|
+
baseline: armRows.baseline.filter(isDecisiveRow).length,
|
|
647
|
+
candidate: armRows.candidate.filter(isDecisiveRow).length,
|
|
648
|
+
negative_control: armRows.negative_control.filter(isDecisiveRow).length,
|
|
649
|
+
};
|
|
650
|
+
const metricMeans = Object.fromEntries(SYNTHESIZE_CERT_ARMS.map((arm) => [
|
|
651
|
+
arm,
|
|
652
|
+
Object.fromEntries(SYNTHESIZE_CERT_METRICS.map((metric) => [
|
|
653
|
+
metric,
|
|
654
|
+
metricMean(armRows[arm], metric),
|
|
655
|
+
])),
|
|
656
|
+
]));
|
|
657
|
+
// Bernoulli population std dev sqrt(m·(1-m)) — §6.2-1/§6.2-5 co-publication;
|
|
658
|
+
// null exactly when the mean is null.
|
|
659
|
+
const metricStddev = Object.fromEntries(SYNTHESIZE_CERT_ARMS.map((arm) => [
|
|
660
|
+
arm,
|
|
661
|
+
Object.fromEntries(SYNTHESIZE_CERT_METRICS.map((metric) => {
|
|
662
|
+
const mean = metricMeans[arm][metric];
|
|
663
|
+
return [metric, mean === null ? null : Math.sqrt(mean * (1 - mean))];
|
|
664
|
+
})),
|
|
665
|
+
]));
|
|
666
|
+
const totalRows = args.judgementRows.length;
|
|
667
|
+
const judgeStatusCounts = {
|
|
668
|
+
ok: 0,
|
|
669
|
+
judge_error: 0,
|
|
670
|
+
timeout: 0,
|
|
671
|
+
not_run: 0,
|
|
672
|
+
};
|
|
673
|
+
for (const row of args.judgementRows)
|
|
674
|
+
judgeStatusCounts[row.judge_status] += 1;
|
|
675
|
+
const judgeFailures = totalRows - judgeStatusCounts.ok;
|
|
676
|
+
const fixtureIds = [
|
|
677
|
+
...new Set(args.inputManifest.map((entry) => entry.fixture_id)),
|
|
678
|
+
];
|
|
679
|
+
const repsByFixtureArm = new Map();
|
|
680
|
+
for (const row of args.judgementRows) {
|
|
681
|
+
const key = `${row.fixture_id} ${row.arm}`;
|
|
682
|
+
const reps = repsByFixtureArm.get(key) ?? new Set();
|
|
683
|
+
reps.add(row.rep);
|
|
684
|
+
repsByFixtureArm.set(key, reps);
|
|
685
|
+
}
|
|
686
|
+
const repsMatrix = [];
|
|
687
|
+
for (const fixtureId of fixtureIds) {
|
|
688
|
+
for (const arm of SYNTHESIZE_CERT_ARMS) {
|
|
689
|
+
repsMatrix.push({
|
|
690
|
+
fixture_id: fixtureId,
|
|
691
|
+
arm,
|
|
692
|
+
distinct_reps: repsByFixtureArm.get(`${fixtureId} ${arm}`)?.size ?? 0,
|
|
693
|
+
});
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
return {
|
|
697
|
+
decisive_row_count: decisiveRowCount,
|
|
698
|
+
metric_means: metricMeans,
|
|
699
|
+
metric_stddev: metricStddev,
|
|
700
|
+
judge_failure_rate: totalRows === 0 ? 0 : judgeFailures / totalRows,
|
|
701
|
+
judge_status_counts: judgeStatusCounts,
|
|
702
|
+
reps_matrix: repsMatrix,
|
|
703
|
+
};
|
|
704
|
+
}
|
|
705
|
+
/**
|
|
706
|
+
* Pre-spend manifest lint (producer-lens MED-6): predicts, from the manifest
|
|
707
|
+
* and declared rep count ALONE, whether the §6.4a decisive floors are even
|
|
708
|
+
* reachable — every possessed stratum needs inputs x reps >= the floor with
|
|
709
|
+
* zero losses, so a violation here means the bench WILL fail no matter how
|
|
710
|
+
* well the runs go. The B4 harness must run this before any paid call.
|
|
711
|
+
*/
|
|
712
|
+
export function synthesizeCertManifestFloorViolations(args) {
|
|
713
|
+
const violations = [];
|
|
714
|
+
const fixtureIds = [
|
|
715
|
+
...new Set(args.inputManifest.map((entry) => entry.fixture_id)),
|
|
716
|
+
];
|
|
717
|
+
if (fixtureIds.length < SYNTHESIZE_CERT_FLOORS.minFixtures) {
|
|
718
|
+
violations.push({
|
|
719
|
+
code: "fixture_floor",
|
|
720
|
+
message: `manifest has ${fixtureIds.length} distinct fixture(s); >= ${SYNTHESIZE_CERT_FLOORS.minFixtures} required`,
|
|
721
|
+
subject_id: null,
|
|
722
|
+
});
|
|
723
|
+
}
|
|
724
|
+
if (args.declaredReps < SYNTHESIZE_CERT_FLOORS.minRepsPerFixtureArm) {
|
|
725
|
+
violations.push({
|
|
726
|
+
code: "declared_reps_floor",
|
|
727
|
+
message: `declared_reps ${args.declaredReps} is below the per-condition floor ${SYNTHESIZE_CERT_FLOORS.minRepsPerFixtureArm}`,
|
|
728
|
+
subject_id: null,
|
|
729
|
+
});
|
|
730
|
+
}
|
|
731
|
+
const inputsByFixtureStratum = new Map();
|
|
732
|
+
for (const entry of args.inputManifest) {
|
|
733
|
+
const key = `${entry.fixture_id} seam=${entry.stratum.seam}|merge=${entry.stratum.merge}`;
|
|
734
|
+
inputsByFixtureStratum.set(key, (inputsByFixtureStratum.get(key) ?? 0) + 1);
|
|
735
|
+
}
|
|
736
|
+
for (const [key, inputCount] of inputsByFixtureStratum) {
|
|
737
|
+
const ceiling = inputCount * args.declaredReps;
|
|
738
|
+
if (ceiling < SYNTHESIZE_CERT_FLOORS.minDecisivePerStratumArm) {
|
|
739
|
+
const [fixtureId] = key.split(" ");
|
|
740
|
+
violations.push({
|
|
741
|
+
code: "stratum_coverage",
|
|
742
|
+
message: `${key}: at most ${ceiling} decisive row(s) possible (inputs x reps) — below the floor ${SYNTHESIZE_CERT_FLOORS.minDecisivePerStratumArm}; add inputs or reps before spending`,
|
|
743
|
+
subject_id: fixtureId ?? null,
|
|
744
|
+
});
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
return violations;
|
|
748
|
+
}
|
|
749
|
+
/**
|
|
750
|
+
* G7 binding (design §11 B5 · onto 20260705-7e0e5263 issue-001/003/006): a
|
|
751
|
+
* registry entry listing the `semantic_map_synthesize` role must cite, among its
|
|
752
|
+
* benchmark_evidence_refs, at least one record that (a) self-identifies as
|
|
753
|
+
* `synthesize-cert/v1`, (b) parses, (c) recomputes to zero violations, and
|
|
754
|
+
* (d) certifies THIS entry's (provider, model). A role label without that proof
|
|
755
|
+
* is exactly the laundering the evidence contract exists to prevent.
|
|
756
|
+
*
|
|
757
|
+
* Pure: the caller (G7 script / tests) reads the refs and passes parsed JSON.
|
|
758
|
+
* `evidenceByRef` must contain an entry for every benchmark_evidence_refs path
|
|
759
|
+
* it could read; unreadable/unparseable refs are simply absent and cannot count.
|
|
760
|
+
*/
|
|
761
|
+
export function synthesizeCertBindingViolations(args) {
|
|
762
|
+
const { entry, evidenceByRef, supportedModelKeys } = args;
|
|
763
|
+
if (!entry.roles?.includes("semantic_map_synthesize"))
|
|
764
|
+
return [];
|
|
765
|
+
const entryId = `${entry.provider}/${entry.model}`;
|
|
766
|
+
const candidates = entry.benchmark_evidence_refs.filter((ref) => isSynthesizeCertCandidate(evidenceByRef.get(ref)));
|
|
767
|
+
if (candidates.length === 0) {
|
|
768
|
+
return [{
|
|
769
|
+
code: "schema_shape_invalid",
|
|
770
|
+
message: `${entryId} lists role semantic_map_synthesize but cites no ${SYNTHESIZE_CERT_CONTRACT} record among benchmark_evidence_refs`,
|
|
771
|
+
subject_id: entryId,
|
|
772
|
+
}];
|
|
773
|
+
}
|
|
774
|
+
const violations = [];
|
|
775
|
+
let bound = false;
|
|
776
|
+
for (const ref of candidates) {
|
|
777
|
+
const { record, violations: parseViolations } = parseSynthesizeCertRecord(evidenceByRef.get(ref));
|
|
778
|
+
if (!record) {
|
|
779
|
+
violations.push(...parseViolations.map((item) => ({
|
|
780
|
+
...item,
|
|
781
|
+
message: `${ref}: ${item.message}`,
|
|
782
|
+
})));
|
|
783
|
+
continue;
|
|
784
|
+
}
|
|
785
|
+
if (record.provider !== entry.provider || record.model !== entry.model) {
|
|
786
|
+
violations.push({
|
|
787
|
+
code: "aggregate_mismatch",
|
|
788
|
+
message: `${ref} certifies ${record.provider}/${record.model}, not the citing entry ${entryId}`,
|
|
789
|
+
subject_id: entryId,
|
|
790
|
+
});
|
|
791
|
+
continue;
|
|
792
|
+
}
|
|
793
|
+
const baselineKey = `${record.arm_model.baseline.provider}/${record.arm_model.baseline.model}`;
|
|
794
|
+
const candidateKey = `${record.arm_model.candidate.provider}/${record.arm_model.candidate.model}`;
|
|
795
|
+
// Baseline identity (§13.3 boundary re-anchor · crossval self-baseline HIGH):
|
|
796
|
+
// the candidate's own key is always in supportedModelKeys at gate time, so a
|
|
797
|
+
// membership check alone lets a record declare arm_model.baseline = candidate
|
|
798
|
+
// (SELF) — candidate>=baseline then compares the model to itself, vacuously.
|
|
799
|
+
// Reject baseline == candidate deterministically. (Whether the baseline is
|
|
800
|
+
// the RIGHT production model, and whether its verdicts are genuine, stays R7.)
|
|
801
|
+
if (baselineKey === candidateKey) {
|
|
802
|
+
violations.push({
|
|
803
|
+
code: "baseline_is_candidate",
|
|
804
|
+
message: `${ref} baseline arm ran ${baselineKey}, the SAME model as the candidate — candidate>=baseline would be a model-vs-itself comparison`,
|
|
805
|
+
subject_id: entryId,
|
|
806
|
+
});
|
|
807
|
+
continue;
|
|
808
|
+
}
|
|
809
|
+
if (!supportedModelKeys.has(baselineKey)) {
|
|
810
|
+
violations.push({
|
|
811
|
+
code: "baseline_not_supported",
|
|
812
|
+
message: `${ref} baseline arm ran ${baselineKey}, which is not a certified supported model — the candidate>=baseline check would rest on an unanchored baseline`,
|
|
813
|
+
subject_id: entryId,
|
|
814
|
+
});
|
|
815
|
+
continue;
|
|
816
|
+
}
|
|
817
|
+
const recomputeViolations = validateSynthesizeCertRecord(record);
|
|
818
|
+
if (recomputeViolations.length === 0) {
|
|
819
|
+
bound = true;
|
|
820
|
+
}
|
|
821
|
+
else {
|
|
822
|
+
violations.push(...recomputeViolations.map((item) => ({
|
|
823
|
+
...item,
|
|
824
|
+
message: `${ref}: ${item.message}`,
|
|
825
|
+
})));
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
if (bound)
|
|
829
|
+
return [];
|
|
830
|
+
return violations;
|
|
831
|
+
}
|