onto-mcp 0.4.11 → 0.4.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.onto/authority/core-lexicon.yaml +1 -1
- package/.onto/authority/supported-models.yaml +153 -0
- package/.onto/domains/accounting-kr/competency_qs.md +153 -153
- package/.onto/processes/reconstruct/obligation-coverage-ledger.yaml +842 -0
- package/.onto/processes/reconstruct/obligation-coverage-recorded.yaml +225 -0
- package/.onto/processes/reconstruct/ontology-seeding-and-maturation-design.md +93 -54
- package/.onto/processes/reconstruct/reconstruct-contract-registry.yaml +202 -49
- package/.onto/processes/reconstruct/reconstruct-execution-ux-contract.md +72 -1
- package/.onto/processes/reconstruct/source-profiles/spreadsheet.md +35 -0
- package/.onto/processes/review/nesting-batch-worker-contract.md +1 -1
- package/.onto/processes/review/productized-live-path.md +9 -5
- package/.onto/processes/review/prompt-execution-runner-contract.md +32 -8
- package/.onto/processes/review/review-target-profile-contract.md +47 -4
- package/.onto/processes/shared/pipeline-execution-ledger-contract.md +11 -0
- package/.onto/processes/shared/target-material-kind-contract.md +1 -1
- package/README.md +64 -1
- package/dist/cli.js +11 -1
- package/dist/core-api/reconstruct-api.js +679 -18
- package/dist/core-api/review-api.js +70 -5
- package/dist/core-api/review-progress.js +7 -0
- package/dist/core-api/runtime-observation.js +7 -0
- package/dist/core-runtime/artifact-io.js +131 -0
- package/dist/core-runtime/cli/claude-code-review-unit-executor.js +21 -9
- package/dist/core-runtime/cli/claude-nesting-batch-worker.js +4 -2
- package/dist/core-runtime/cli/materialize-review-prompt-packets.js +68 -14
- package/dist/core-runtime/cli/prepare-review-session.js +34 -1
- package/dist/core-runtime/cli/render-review-final-output.js +2 -1
- package/dist/core-runtime/cli/run-review-prompt-execution.js +758 -48
- package/dist/core-runtime/cli/run-seats.js +18 -0
- package/dist/core-runtime/cli/runtime-submit-context.js +10 -2
- package/dist/core-runtime/cli/unit-resubmit.js +269 -0
- package/dist/core-runtime/cli/worker-structured-output.js +80 -2
- package/dist/core-runtime/discovery/host-detection.js +71 -22
- package/dist/core-runtime/discovery/review-cert-assemble.js +208 -0
- package/dist/core-runtime/discovery/review-cert-record.js +442 -0
- package/dist/core-runtime/discovery/review-execution-units.js +22 -0
- package/dist/core-runtime/discovery/seat-inventory.js +167 -0
- package/dist/core-runtime/discovery/settings-chain.js +311 -52
- package/dist/core-runtime/discovery/supported-models.js +289 -25
- package/dist/core-runtime/discovery/synthesize-cert-assemble.js +197 -0
- package/dist/core-runtime/discovery/synthesize-cert-capsule.js +399 -0
- package/dist/core-runtime/discovery/synthesize-cert-judge.js +340 -0
- package/dist/core-runtime/discovery/synthesize-cert-loop.js +234 -0
- package/dist/core-runtime/discovery/synthesize-cert-mutation.js +117 -0
- package/dist/core-runtime/discovery/synthesize-cert-packet.js +182 -0
- package/dist/core-runtime/discovery/synthesize-cert-record.js +831 -0
- package/dist/core-runtime/discovery/synthesize-cert-sampler.js +336 -0
- package/dist/core-runtime/effort-calibration-ingest.js +326 -0
- package/dist/core-runtime/effort-calibration-reconstruct.js +47 -0
- package/dist/core-runtime/effort-calibration-review.js +22 -0
- package/dist/core-runtime/effort-calibration-sweep.js +110 -0
- package/dist/core-runtime/effort-frontier.js +134 -0
- package/dist/core-runtime/llm/claude-bin.js +77 -0
- package/dist/core-runtime/llm/dispatch-breaker.js +412 -0
- package/dist/core-runtime/llm/dispatch-fallback-adapter-capabilities.js +33 -0
- package/dist/core-runtime/llm/llm-caller.js +316 -35
- package/dist/core-runtime/llm/mock-llm-realization.js +35 -4
- package/dist/core-runtime/llm/model-switcher.js +1 -1
- package/dist/core-runtime/llm/openai-responses-incomplete-error.js +46 -0
- package/dist/core-runtime/llm/sealed-dispatch-capability.js +359 -0
- package/dist/core-runtime/llm/structured-dispatch-error.js +15 -0
- package/dist/core-runtime/onboard/claude-profile-scan.js +65 -0
- package/dist/core-runtime/onboard/cli-host.js +5 -2
- package/dist/core-runtime/onboard/configure-provider.js +33 -1
- package/dist/core-runtime/onboard/host-target.js +28 -3
- package/dist/core-runtime/onboard/register.js +54 -12
- package/dist/core-runtime/pipeline-execution-ledger.js +10 -0
- package/dist/core-runtime/reconstruct/artifact-types.js +37 -0
- package/dist/core-runtime/reconstruct/benchmark-evidence.js +16 -0
- package/dist/core-runtime/reconstruct/claim-projection-validation.js +19 -5
- package/dist/core-runtime/reconstruct/competency-projection-contract.js +63 -0
- package/dist/core-runtime/reconstruct/comprehension-artifact.js +321 -0
- package/dist/core-runtime/reconstruct/comprehension-reduce.js +367 -0
- package/dist/core-runtime/reconstruct/comprehension-semantic-map.js +771 -0
- package/dist/core-runtime/reconstruct/contract-registry.js +20 -0
- package/dist/core-runtime/reconstruct/directive-validation.js +4 -3
- package/dist/core-runtime/reconstruct/dispatch-fallback-artifacts.js +640 -0
- package/dist/core-runtime/reconstruct/execution-telemetry.js +84 -9
- package/dist/core-runtime/reconstruct/final-output-sections.js +89 -0
- package/dist/core-runtime/reconstruct/governing-snapshot.js +36 -1
- package/dist/core-runtime/reconstruct/leaf-reader.js +305 -0
- package/dist/core-runtime/reconstruct/llm-dispatch-failure.js +270 -0
- package/dist/core-runtime/reconstruct/llm-touch-fingerprint.js +94 -0
- package/dist/core-runtime/reconstruct/material-admission-validation.js +20 -5
- package/dist/core-runtime/reconstruct/material-profile-validation.js +24 -6
- package/dist/core-runtime/reconstruct/materialize-preparation.js +380 -23
- package/dist/core-runtime/reconstruct/maturation-validation.js +1522 -67
- package/dist/core-runtime/reconstruct/mock-llm-realization.js +120 -1
- package/dist/core-runtime/reconstruct/obligation-assertion.js +18 -0
- package/dist/core-runtime/reconstruct/ontology-seed-validation.js +39 -7
- package/dist/core-runtime/reconstruct/output-budget.js +6 -0
- package/dist/core-runtime/reconstruct/pipeline-execution-ledger.js +112 -5
- package/dist/core-runtime/reconstruct/post-seed-validation.js +122 -11
- package/dist/core-runtime/reconstruct/proof-authority-validation.js +2 -6
- package/dist/core-runtime/reconstruct/purpose-authority-validation.js +119 -7
- package/dist/core-runtime/reconstruct/record.js +49 -3
- package/dist/core-runtime/reconstruct/registry-verification-validation.js +41 -5
- package/dist/core-runtime/reconstruct/run-control-validation.js +1019 -37
- package/dist/core-runtime/reconstruct/run.js +5409 -517
- package/dist/core-runtime/reconstruct/seed-authoring-readiness-validation.js +56 -8
- package/dist/core-runtime/reconstruct/semantic-quality-gate.js +112 -7
- package/dist/core-runtime/reconstruct/source-observation-delta-validation.js +53 -5
- package/dist/core-runtime/reconstruct/source-observations.js +166 -0
- package/dist/core-runtime/reconstruct/source-safety-validation.js +77 -188
- package/dist/core-runtime/reconstruct/source-scout-pack-validation.js +50 -6
- package/dist/core-runtime/reconstruct/terminal-validation.js +145 -13
- package/dist/core-runtime/review/continuation-plan.js +12 -5
- package/dist/core-runtime/review/issue-artifact-runtime.js +4 -1
- package/dist/core-runtime/review/materializers.js +100 -11
- package/dist/core-runtime/review/nesting-batch.js +6 -2
- package/dist/core-runtime/review/obligation-shardability.js +138 -0
- package/dist/core-runtime/review/pipeline-execution-ledger.js +42 -2
- package/dist/core-runtime/review/review-artifact-utils.js +219 -7
- package/dist/core-runtime/review/review-execution-steps.js +32 -8
- package/dist/core-runtime/review/review-progress-contract.js +8 -0
- package/dist/core-runtime/review/review-prompt-budget.js +97 -0
- package/dist/core-runtime/review/semantic-quality-gate.js +33 -7
- package/dist/core-runtime/review/spreadsheet-review-disposition.js +94 -0
- package/dist/core-runtime/route-identity.js +182 -0
- package/dist/core-runtime/spreadsheet-header-escalation.js +204 -0
- package/dist/core-runtime/spreadsheet-structure-observer.js +2689 -0
- package/dist/core-runtime/target-material-kind.js +58 -0
- package/dist/mcp/server.js +27 -5
- package/dist/mcp/tool-schemas.js +8 -0
- package/dist/tui/app.js +167 -0
- package/dist/tui/data/event-follower.js +116 -0
- package/dist/tui/data/node-detail.js +42 -0
- package/dist/tui/data/projection-poll.js +26 -0
- package/dist/tui/data/session-discovery.js +65 -0
- package/dist/tui/index.js +108 -0
- package/dist/tui/screens/log.js +25 -0
- package/dist/tui/screens/node-detail.js +17 -0
- package/dist/tui/screens/session-selector.js +26 -0
- package/dist/tui/screens/workflow-tree.js +90 -0
- package/dist/tui/view-model/reconstruct-adapter.js +162 -0
- package/dist/tui/view-model/review-adapter.js +270 -0
- package/dist/tui/view-model/tree-view-model.js +19 -0
- package/package.json +20 -3
- package/settings.example.json +77 -16
|
@@ -0,0 +1,336 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* S1 — deterministic stratified sampler for the B4 `synthesize-cert/v1` bench
|
|
3
|
+
* (design 20260706-b4-r8-harness-design v3 §3/§4/§15.1).
|
|
4
|
+
*
|
|
5
|
+
* Everything here is pure and deterministic — the orchestrator script does the
|
|
6
|
+
* I/O (observe → buildColumnLeaves → reduceColumnLeavesWithTrace →
|
|
7
|
+
* classifyFrontier per column) and hands the per-fixture candidate universe in.
|
|
8
|
+
* Two responsibilities live here because they must not drift apart:
|
|
9
|
+
*
|
|
10
|
+
* 1. TWO-LAYER INPUT IDENTITY (§4 · round-2 R2-IND-1):
|
|
11
|
+
* - `deterministic_facts_sha256` — hash of the node's LLM-independent facts
|
|
12
|
+
* (node_ref + sorted format_clusters + canonical value_shape_seams,
|
|
13
|
+
* child_summaries EXCLUDED). Computable BEFORE the reference child
|
|
14
|
+
* authoring spend, so sampling/scope/floor pre-checks run pre-spend
|
|
15
|
+
* (§3 sequencing).
|
|
16
|
+
* - `input_sha256` — hash of the FULL frozen SemanticSynthesisInput packet
|
|
17
|
+
* (child_summaries included). The same-input comparison key: every arm
|
|
18
|
+
* runs the identical frozen packet, so baseline/candidate rows share it.
|
|
19
|
+
* Both hashes canonicalize through ONE serializer, and both run the shipped
|
|
20
|
+
* `assertSynthesisInputBounded` source-safe envelope first, so a smuggled
|
|
21
|
+
* extra field can never be silently hash-invisible.
|
|
22
|
+
*
|
|
23
|
+
* 2. HONEST DETERMINISTIC SELECTION (§3, cherry-pick 금지): per
|
|
24
|
+
* (fixture × possessed stratum), over-provision K inputs with a seeded,
|
|
25
|
+
* content-blind rule — merge strata order by subtree leaf count ascending
|
|
26
|
+
* (a DISCLOSED cost bias; seed tie-break), leaf strata stride-sample the
|
|
27
|
+
* stable-key order — and record provenance (pool size, seed, stride, rank,
|
|
28
|
+
* selected-vs-nearest-unselected) for the R7 representativeness audit.
|
|
29
|
+
*
|
|
30
|
+
* The pre-spend floor gate is part of the sample result (not a separate call a
|
|
31
|
+
* caller could forget): `floor_violations` re-uses the shipped
|
|
32
|
+
* `synthesizeCertManifestFloorViolations` plus a pre-spend prediction of the
|
|
33
|
+
* record validator's `stratum_global_floor`, so an under-provisioned manifest
|
|
34
|
+
* is rejected before any paid call. Structural integrity failures (a candidate
|
|
35
|
+
* whose key/hash lies, an input_id collision) THROW instead — those are
|
|
36
|
+
* harness bugs, not sampling outcomes.
|
|
37
|
+
*/
|
|
38
|
+
import { createHash } from "node:crypto";
|
|
39
|
+
import { reduceNodeKey, } from "../reconstruct/comprehension-reduce.js";
|
|
40
|
+
import { assertReduceTopologyIsTree, assertSynthesisInputBounded, buildSynthesisInputForNode, computeSubtreeLeafCounts, } from "../reconstruct/comprehension-semantic-map.js";
|
|
41
|
+
import { synthesizeCertManifestFloorViolations, SYNTHESIZE_CERT_FLOORS, } from "./synthesize-cert-record.js";
|
|
42
|
+
/** Folded into every stratum seed — editing the sampler's selection rule must
|
|
43
|
+
* ship with a version bump so a re-run cannot silently claim the old picks. */
|
|
44
|
+
export const SYNTHESIZE_CERT_SAMPLER_VERSION = "synthesize-cert-sampler/v1";
|
|
45
|
+
/** §3 over-provisioning: at a pessimistic decisive rate of 0.4, K×reps(3)×0.4 =
|
|
46
|
+
* 6 ≥ the per-stratum decisive floor (5), so a stratum survives real losses. */
|
|
47
|
+
export const SYNTHESIZE_CERT_DEFAULT_PER_STRATUM_K = 5;
|
|
48
|
+
const sha256Hex = (text) => createHash("sha256").update(text).digest("hex");
|
|
49
|
+
/** Same projection format as the record module's internal stratum key (kept a
|
|
50
|
+
* one-line local; the record module's helper is private and that file is the
|
|
51
|
+
* frozen evidence contract). */
|
|
52
|
+
const stratumKey = (s) => `seam=${s.seam}|merge=${s.merge}`;
|
|
53
|
+
const cmpStr = (x, y) => (x < y ? -1 : x > y ? 1 : 0);
|
|
54
|
+
/** Fixed-key-order canonical shape shared by BOTH identity hashes: sorted
|
|
55
|
+
* clusters, canonically-sorted re-keyed seams, re-keyed node_ref. Child
|
|
56
|
+
* summaries (when included) keep their packet order — they are positional
|
|
57
|
+
* prompt content, and the builder's order is already deterministic. */
|
|
58
|
+
function canonicalPacketShape(packet, includeChildren) {
|
|
59
|
+
const r = packet.node_ref;
|
|
60
|
+
const seams = packet.value_shape_seams
|
|
61
|
+
.map((s) => ({ row: s.row, prev_shape: s.prev_shape, new_shape: s.new_shape }))
|
|
62
|
+
.sort((x, y) => x.row - y.row || cmpStr(x.prev_shape, y.prev_shape) || cmpStr(x.new_shape, y.new_shape));
|
|
63
|
+
return {
|
|
64
|
+
node_ref: { sheet: r.sheet, column_index: r.column_index, row_start: r.row_start, row_end: r.row_end },
|
|
65
|
+
format_clusters: [...packet.format_clusters].sort(),
|
|
66
|
+
value_shape_seams: seams,
|
|
67
|
+
...(includeChildren
|
|
68
|
+
? { child_summaries: packet.child_summaries.map((c) => ({ key: c.key, summary: c.summary })) }
|
|
69
|
+
: {}),
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
/** `deterministic_facts_sha256` (§4): child-independent by construction — the
|
|
73
|
+
* facts are wrapped with child_summaries=[] to reuse the shipped source-safe
|
|
74
|
+
* envelope guard, then hashed WITHOUT the child field. */
|
|
75
|
+
export function synthesizeCertDeterministicFactsSha256(facts) {
|
|
76
|
+
const probe = {
|
|
77
|
+
node_ref: facts.node_ref,
|
|
78
|
+
format_clusters: facts.format_clusters,
|
|
79
|
+
value_shape_seams: facts.value_shape_seams,
|
|
80
|
+
child_summaries: [],
|
|
81
|
+
};
|
|
82
|
+
assertSynthesisInputBounded(probe);
|
|
83
|
+
return sha256Hex(JSON.stringify(canonicalPacketShape(probe, false)));
|
|
84
|
+
}
|
|
85
|
+
/** `input_sha256` (§4): identity of the FULL frozen packet (child_summaries
|
|
86
|
+
* included) — the same-input comparison key across arms. The exact-key envelope
|
|
87
|
+
* guard runs first so an extra/smuggled field fails closed instead of being
|
|
88
|
+
* silently excluded from the hash. */
|
|
89
|
+
export function synthesizeCertInputSha256(packet) {
|
|
90
|
+
assertSynthesisInputBounded(packet);
|
|
91
|
+
return sha256Hex(JSON.stringify(canonicalPacketShape(packet, true)));
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Enumerates the sampler universe for ONE column pipeline: every non-subsumed
|
|
95
|
+
* node, stratum-tagged (merge = accumulating mode; seam = the packet's canonical
|
|
96
|
+
* value_shape_seams are non-empty). The deterministic facts come through the
|
|
97
|
+
* SINGLE-SOURCE `buildSynthesisInputForNode` (with placeholder child summaries,
|
|
98
|
+
* discarded) so the facts hashed here are exactly the fields the frozen packet
|
|
99
|
+
* will carry — the two cannot drift.
|
|
100
|
+
*/
|
|
101
|
+
export function collectSynthesizeCertCandidates(args) {
|
|
102
|
+
const { trace, nodesByKey, modes, sheetIndex } = args;
|
|
103
|
+
if (!Number.isSafeInteger(sheetIndex) || sheetIndex < 0) {
|
|
104
|
+
throw new Error(`synthesize-cert-sampler: sheetIndex must be a non-negative safe integer, got ${sheetIndex}`);
|
|
105
|
+
}
|
|
106
|
+
assertReduceTopologyIsTree(trace);
|
|
107
|
+
const leafCounts = computeSubtreeLeafCounts(trace);
|
|
108
|
+
const out = [];
|
|
109
|
+
for (const [key, tnode] of trace.nodes) {
|
|
110
|
+
const mode = modes.get(key);
|
|
111
|
+
if (mode === undefined) {
|
|
112
|
+
throw new Error(`synthesize-cert-sampler: no frontier mode for ${key} — modes must cover the trace`);
|
|
113
|
+
}
|
|
114
|
+
if (mode === "subsumed")
|
|
115
|
+
continue;
|
|
116
|
+
// Placeholder child summaries: buildSynthesisInputForNode requires a summary
|
|
117
|
+
// per consumed child; the facts we keep exclude child_summaries entirely.
|
|
118
|
+
const consumed = tnode.child_keys.filter((k) => modes.get(k) !== "subsumed");
|
|
119
|
+
const packetShape = buildSynthesisInputForNode(trace, nodesByKey, modes, key, new Map(consumed.map((k) => [k, ""])));
|
|
120
|
+
const facts = {
|
|
121
|
+
node_ref: packetShape.node_ref,
|
|
122
|
+
format_clusters: packetShape.format_clusters,
|
|
123
|
+
value_shape_seams: packetShape.value_shape_seams,
|
|
124
|
+
};
|
|
125
|
+
const leafCount = leafCounts.get(key);
|
|
126
|
+
if (leafCount === undefined) {
|
|
127
|
+
throw new Error(`synthesize-cert-sampler: no leaf count for ${key} (impossible)`);
|
|
128
|
+
}
|
|
129
|
+
out.push({
|
|
130
|
+
node_key: key,
|
|
131
|
+
node_ref: packetShape.node_ref,
|
|
132
|
+
sheet_index: sheetIndex,
|
|
133
|
+
stratum: { seam: facts.value_shape_seams.length > 0, merge: mode === "accumulating" },
|
|
134
|
+
subtree_leaf_count: leafCount,
|
|
135
|
+
deterministic_facts: facts,
|
|
136
|
+
deterministic_facts_sha256: synthesizeCertDeterministicFactsSha256(facts),
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
out.sort((a, b) => cmpStr(a.node_key, b.node_key));
|
|
140
|
+
return out;
|
|
141
|
+
}
|
|
142
|
+
function buildInputId(fixtureId, c) {
|
|
143
|
+
const r = c.node_ref;
|
|
144
|
+
return `${fixtureId.slice(0, 8)}-s${c.sheet_index}-c${r.column_index}-r${r.row_start}_${r.row_end}`;
|
|
145
|
+
}
|
|
146
|
+
/** Nearest unselected pool index for a selected index (scan outward; ties →
|
|
147
|
+
* earlier index). Returns -1 when every pool member was selected. */
|
|
148
|
+
function nearestUnselectedIndex(selected, index, poolSize) {
|
|
149
|
+
for (let dist = 1; dist < poolSize; dist += 1) {
|
|
150
|
+
const lo = index - dist;
|
|
151
|
+
if (lo >= 0 && !selected.has(lo))
|
|
152
|
+
return lo;
|
|
153
|
+
const hi = index + dist;
|
|
154
|
+
if (hi < poolSize && !selected.has(hi))
|
|
155
|
+
return hi;
|
|
156
|
+
}
|
|
157
|
+
return -1;
|
|
158
|
+
}
|
|
159
|
+
export function sampleStratifiedManifest(fixtures, opts) {
|
|
160
|
+
const perStratumK = opts?.perStratumK ?? SYNTHESIZE_CERT_DEFAULT_PER_STRATUM_K;
|
|
161
|
+
const declaredReps = opts?.declaredReps ?? SYNTHESIZE_CERT_FLOORS.minRepsPerFixtureArm;
|
|
162
|
+
if (!Number.isSafeInteger(perStratumK) || perStratumK < 1) {
|
|
163
|
+
throw new Error(`synthesize-cert-sampler: perStratumK must be a positive safe integer, got ${perStratumK}`);
|
|
164
|
+
}
|
|
165
|
+
if (!Number.isSafeInteger(declaredReps) || declaredReps < 1) {
|
|
166
|
+
throw new Error(`synthesize-cert-sampler: declaredReps must be a positive safe integer, got ${declaredReps}`);
|
|
167
|
+
}
|
|
168
|
+
const manifest = [];
|
|
169
|
+
const provenance = [];
|
|
170
|
+
const seenFixtureIds = new Set();
|
|
171
|
+
for (const fixture of fixtures) {
|
|
172
|
+
// fixture_id must be the workbook's real content sha (64-hex): the input_id
|
|
173
|
+
// prefix and the whitespace-free id guarantee both derive from it.
|
|
174
|
+
if (!/^[0-9a-f]{64}$/.test(fixture.fixture_id)) {
|
|
175
|
+
throw new Error(`synthesize-cert-sampler: fixture_id must be a lowercase sha256 hex (workbook content identity), got '${fixture.fixture_id}'`);
|
|
176
|
+
}
|
|
177
|
+
if (seenFixtureIds.has(fixture.fixture_id)) {
|
|
178
|
+
throw new Error(`synthesize-cert-sampler: duplicate fixture_id ${fixture.fixture_id}`);
|
|
179
|
+
}
|
|
180
|
+
seenFixtureIds.add(fixture.fixture_id);
|
|
181
|
+
// Candidate integrity — a candidate whose declared identity lies is a
|
|
182
|
+
// harness bug, fail-closed (§3 정직 선정 rests on these being real).
|
|
183
|
+
const seenNodeKeys = new Set();
|
|
184
|
+
for (const c of fixture.candidates) {
|
|
185
|
+
if (reduceNodeKey(c.node_ref) !== c.node_key) {
|
|
186
|
+
throw new Error(`synthesize-cert-sampler: candidate node_key '${c.node_key}' != reduceNodeKey(node_ref) '${reduceNodeKey(c.node_ref)}'`);
|
|
187
|
+
}
|
|
188
|
+
if (seenNodeKeys.has(c.node_key)) {
|
|
189
|
+
throw new Error(`synthesize-cert-sampler: duplicate candidate node_key '${c.node_key}' in fixture ${fixture.fixture_id}`);
|
|
190
|
+
}
|
|
191
|
+
seenNodeKeys.add(c.node_key);
|
|
192
|
+
if (synthesizeCertDeterministicFactsSha256(c.deterministic_facts) !== c.deterministic_facts_sha256) {
|
|
193
|
+
throw new Error(`synthesize-cert-sampler: candidate ${c.node_key} deterministic_facts_sha256 does not recompute from its facts`);
|
|
194
|
+
}
|
|
195
|
+
if (c.stratum.seam !== (c.deterministic_facts.value_shape_seams.length > 0)) {
|
|
196
|
+
throw new Error(`synthesize-cert-sampler: candidate ${c.node_key} seam flag disagrees with its value_shape_seams`);
|
|
197
|
+
}
|
|
198
|
+
if (!Number.isSafeInteger(c.sheet_index) || c.sheet_index < 0) {
|
|
199
|
+
throw new Error(`synthesize-cert-sampler: candidate ${c.node_key} sheet_index invalid`);
|
|
200
|
+
}
|
|
201
|
+
if (!Number.isSafeInteger(c.subtree_leaf_count) || c.subtree_leaf_count < 1) {
|
|
202
|
+
throw new Error(`synthesize-cert-sampler: candidate ${c.node_key} subtree_leaf_count invalid`);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
// Group by possessed stratum, stable-key order (§3-1).
|
|
206
|
+
const pools = new Map();
|
|
207
|
+
for (const c of [...fixture.candidates].sort((a, b) => cmpStr(a.node_key, b.node_key))) {
|
|
208
|
+
const k = stratumKey(c.stratum);
|
|
209
|
+
const pool = pools.get(k) ?? [];
|
|
210
|
+
pool.push(c);
|
|
211
|
+
pools.set(k, pool);
|
|
212
|
+
}
|
|
213
|
+
for (const [poolKey, pool] of [...pools.entries()].sort((a, b) => cmpStr(a[0], b[0]))) {
|
|
214
|
+
const stratum = pool[0].stratum;
|
|
215
|
+
const seed = sha256Hex(`${fixture.fixture_id}|${poolKey}|${SYNTHESIZE_CERT_SAMPLER_VERSION}`);
|
|
216
|
+
const isMerge = stratum.merge;
|
|
217
|
+
// merge: order by subtree leaf count ascending (disclosed content-blind
|
|
218
|
+
// cost bias, §3-2), seed tie-break, node_key as the final total order.
|
|
219
|
+
const ordered = isMerge
|
|
220
|
+
? [...pool].sort((a, b) => a.subtree_leaf_count - b.subtree_leaf_count ||
|
|
221
|
+
cmpStr(sha256Hex(`${seed}|${a.node_key}`), sha256Hex(`${seed}|${b.node_key}`)) ||
|
|
222
|
+
cmpStr(a.node_key, b.node_key))
|
|
223
|
+
: pool; // leaf strata: stable-key order (already sorted).
|
|
224
|
+
const takeAll = ordered.length <= perStratumK;
|
|
225
|
+
let stride = null;
|
|
226
|
+
let pickedIndices;
|
|
227
|
+
if (takeAll) {
|
|
228
|
+
pickedIndices = ordered.map((_, i) => i);
|
|
229
|
+
}
|
|
230
|
+
else if (isMerge) {
|
|
231
|
+
pickedIndices = Array.from({ length: perStratumK }, (_, i) => i); // smallest-K cost bound
|
|
232
|
+
}
|
|
233
|
+
else {
|
|
234
|
+
// Seeded stride over the stable-key order: spreads picks across the
|
|
235
|
+
// pool instead of clustering at its head. offset+ (K-1)·stride ≤ N-1.
|
|
236
|
+
stride = Math.floor(ordered.length / perStratumK);
|
|
237
|
+
const offset = parseInt(seed.slice(0, 12), 16) % stride;
|
|
238
|
+
pickedIndices = Array.from({ length: perStratumK }, (_, i) => offset + i * stride);
|
|
239
|
+
}
|
|
240
|
+
const selectedSet = new Set(pickedIndices);
|
|
241
|
+
pickedIndices.forEach((poolIndex, rank) => {
|
|
242
|
+
const c = ordered[poolIndex];
|
|
243
|
+
const nearest = nearestUnselectedIndex(selectedSet, poolIndex, ordered.length);
|
|
244
|
+
manifest.push({
|
|
245
|
+
fixture_id: fixture.fixture_id,
|
|
246
|
+
input_id: buildInputId(fixture.fixture_id, c),
|
|
247
|
+
node_key: c.node_key,
|
|
248
|
+
sheet_index: c.sheet_index,
|
|
249
|
+
stratum: c.stratum,
|
|
250
|
+
subtree_leaf_count: c.subtree_leaf_count,
|
|
251
|
+
deterministic_facts: c.deterministic_facts,
|
|
252
|
+
deterministic_facts_sha256: c.deterministic_facts_sha256,
|
|
253
|
+
sampling_rank: rank,
|
|
254
|
+
nearest_unselected_id: nearest === -1 ? null : buildInputId(fixture.fixture_id, ordered[nearest]),
|
|
255
|
+
});
|
|
256
|
+
});
|
|
257
|
+
provenance.push({
|
|
258
|
+
fixture_id: fixture.fixture_id,
|
|
259
|
+
stratum,
|
|
260
|
+
pool_size: ordered.length,
|
|
261
|
+
selected_count: pickedIndices.length,
|
|
262
|
+
seed,
|
|
263
|
+
ordering: isMerge ? "subtree_leaf_count_asc" : "stable_key_stride",
|
|
264
|
+
stride,
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
// §3 E: whitespace-free + globally unique input_ids, asserted before freeze.
|
|
269
|
+
const seenInputIds = new Set();
|
|
270
|
+
for (const entry of manifest) {
|
|
271
|
+
if (!/^\S+$/.test(entry.input_id)) {
|
|
272
|
+
throw new Error(`synthesize-cert-sampler: input_id '${entry.input_id}' contains whitespace or is empty`);
|
|
273
|
+
}
|
|
274
|
+
if (seenInputIds.has(entry.input_id)) {
|
|
275
|
+
throw new Error(`synthesize-cert-sampler: input_id collision '${entry.input_id}' — ids must be globally unique`);
|
|
276
|
+
}
|
|
277
|
+
seenInputIds.add(entry.input_id);
|
|
278
|
+
}
|
|
279
|
+
// Pre-spend floor gate (§3 sequencing): deterministic-facts scope — the
|
|
280
|
+
// input_sha256 slot carries deterministic_facts_sha256 because the frozen
|
|
281
|
+
// packet sha does not exist yet; the floor lint never reads the sha.
|
|
282
|
+
const floorViolations = synthesizeCertManifestFloorViolations({
|
|
283
|
+
inputManifest: manifest.map((e) => ({
|
|
284
|
+
fixture_id: e.fixture_id,
|
|
285
|
+
input_id: e.input_id,
|
|
286
|
+
input_sha256: e.deterministic_facts_sha256,
|
|
287
|
+
stratum: e.stratum,
|
|
288
|
+
})),
|
|
289
|
+
declaredReps,
|
|
290
|
+
});
|
|
291
|
+
// Pre-spend prediction of the record validator's stratum_global_floor: every
|
|
292
|
+
// 2×2 stratum must be floor-reachable (inputs × reps ≥ decisive floor) in at
|
|
293
|
+
// least ONE fixture — a manifest missing a stratum entirely passes the
|
|
294
|
+
// possessed-strata lint above but is guaranteed to fail post-spend.
|
|
295
|
+
const selectedByFixtureStratum = new Map();
|
|
296
|
+
for (const e of manifest) {
|
|
297
|
+
const k = `${e.fixture_id} ${stratumKey(e.stratum)}`;
|
|
298
|
+
selectedByFixtureStratum.set(k, (selectedByFixtureStratum.get(k) ?? 0) + 1);
|
|
299
|
+
}
|
|
300
|
+
for (const combo of [
|
|
301
|
+
{ seam: false, merge: false },
|
|
302
|
+
{ seam: false, merge: true },
|
|
303
|
+
{ seam: true, merge: false },
|
|
304
|
+
{ seam: true, merge: true },
|
|
305
|
+
]) {
|
|
306
|
+
const reachable = fixtures.some((fx) => (selectedByFixtureStratum.get(`${fx.fixture_id} ${stratumKey(combo)}`) ?? 0) * declaredReps >=
|
|
307
|
+
SYNTHESIZE_CERT_FLOORS.minDecisivePerStratumArm);
|
|
308
|
+
if (!reachable) {
|
|
309
|
+
floorViolations.push({
|
|
310
|
+
code: "stratum_global_floor",
|
|
311
|
+
message: `stratum ${stratumKey(combo)} is floor-reachable in no fixture (inputs x reps < ${SYNTHESIZE_CERT_FLOORS.minDecisivePerStratumArm} everywhere) — the record WILL fail stratum_global_floor; do not spend`,
|
|
312
|
+
subject_id: null,
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
const manifestIdentitySha256 = sha256Hex(JSON.stringify([...manifest]
|
|
317
|
+
.sort((a, b) => cmpStr(a.input_id, b.input_id))
|
|
318
|
+
.map((e) => ({
|
|
319
|
+
fixture_id: e.fixture_id,
|
|
320
|
+
input_id: e.input_id,
|
|
321
|
+
node_key: e.node_key,
|
|
322
|
+
stratum: { seam: e.stratum.seam, merge: e.stratum.merge },
|
|
323
|
+
deterministic_facts_sha256: e.deterministic_facts_sha256,
|
|
324
|
+
sampling_rank: e.sampling_rank,
|
|
325
|
+
nearest_unselected_id: e.nearest_unselected_id,
|
|
326
|
+
}))));
|
|
327
|
+
return {
|
|
328
|
+
manifest,
|
|
329
|
+
provenance,
|
|
330
|
+
floor_violations: floorViolations,
|
|
331
|
+
manifest_identity_sha256: manifestIdentitySha256,
|
|
332
|
+
sampler_version: SYNTHESIZE_CERT_SAMPLER_VERSION,
|
|
333
|
+
per_stratum_k: perStratumK,
|
|
334
|
+
declared_reps: declaredReps,
|
|
335
|
+
};
|
|
336
|
+
}
|
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Effort-calibration ingestion (P4) — turn an existing benchmark report's JSON
|
|
3
|
+
* into the `EffortSweepRun[]` the sweep aggregator consumes. Pure: no LLM calls
|
|
4
|
+
* and no IO. The benchmark scripts already run the (paid, live) sweep and write
|
|
5
|
+
* the report; this module only re-reads that record, so the calibration report
|
|
6
|
+
* is produced deterministically and is unit-testable against captured output.
|
|
7
|
+
*
|
|
8
|
+
* Honesty rule (carried from the sweep aggregator): a run that produced no
|
|
9
|
+
* quality verdict — a failed run with no gate — is emitted as an unjudged
|
|
10
|
+
* (passed=null, qualityScore=null) sweep run, NOT dropped. Dropping failures
|
|
11
|
+
* would let an effort look viable on a single surviving pass; counting them as
|
|
12
|
+
* non-passing in the denominator keeps the quorum honest.
|
|
13
|
+
*
|
|
14
|
+
* Pipeline asymmetry, mirrored from the two benchmark harnesses:
|
|
15
|
+
* - review: one unit-sweep invocation varies one unit's effort internally, so
|
|
16
|
+
* each run self-describes its (unit, effort); the baseline run (all units at
|
|
17
|
+
* base_effort) is the shared base-effort point for every swept unit.
|
|
18
|
+
* - reconstruct: one invocation pins one effort knob (global author --effort or
|
|
19
|
+
* --judge-effort), so the (stage, effort) of a report is the knob that was
|
|
20
|
+
* pinned; the sweep is several invocations, one report per effort point.
|
|
21
|
+
*
|
|
22
|
+
* Cost is intentionally NOT populated here. Per-stage cost is reporting-only
|
|
23
|
+
* (the frontier decision ignores it) and correct attribution needs a verified
|
|
24
|
+
* mapping that this stage cannot prove: review's swept stage ids live in the
|
|
25
|
+
* execution namespace (e.g. `finding_ledger`, `synthesis_response`, `lens`)
|
|
26
|
+
* while per-unit telemetry uses runtime ids (`finding-ledger`, `synthesize`,
|
|
27
|
+
* per-lens ids), and a reconstruct report's `totals` is the whole-pipeline cost,
|
|
28
|
+
* not the swept stage's. The optional `cost` field stays in the sweep/frontier
|
|
29
|
+
* layer; populating it correctly is deferred to P4b, when real multi-effort
|
|
30
|
+
* sweep data exists to verify the per-stage unit-telemetry attribution.
|
|
31
|
+
*/
|
|
32
|
+
import { modelProviderFromRuntimeProvider, profileDerivedRouteIdentity, routeHintMatches, routeToken, witnessedReconstructRouteIdentity, worstRouteCompleteness, } from "./route-identity.js";
|
|
33
|
+
import { reviewRunGateSignal } from "./effort-calibration-review.js";
|
|
34
|
+
import { reconstructRunGateSignal } from "./effort-calibration-reconstruct.js";
|
|
35
|
+
const UNJUDGED = { passed: null, qualityScore: null };
|
|
36
|
+
/** Only a unit-sweep baseline case is the shared base-effort point per unit. */
|
|
37
|
+
const UNIT_SWEEP_BASE_PREFIX = "unit-sweep-base";
|
|
38
|
+
/**
|
|
39
|
+
* Ingest a review unit-sweep benchmark report into sweep runs. Candidate runs
|
|
40
|
+
* map to their swept unit at its varied effort; each unit-sweep BASELINE run
|
|
41
|
+
* becomes the base-effort point for EVERY swept unit (the units observed across
|
|
42
|
+
* candidate runs), since "unit X at base, others at base" is exactly the
|
|
43
|
+
* baseline. Only `unit-sweep-base-*` runs are treated as baselines: a report may
|
|
44
|
+
* also carry non-unit-sweep cases (e.g. `all-high`) that share `base_effort` but
|
|
45
|
+
* vary every unit at once, and replicating those would contaminate the
|
|
46
|
+
* single-variable frontier. Runs without a quality gate (failures) are emitted
|
|
47
|
+
* unjudged; runs that are neither a candidate nor a unit-sweep baseline are
|
|
48
|
+
* skipped (nothing single-variable to attribute).
|
|
49
|
+
*/
|
|
50
|
+
export function ingestReviewReport(report) {
|
|
51
|
+
const runs = report.runs ?? [];
|
|
52
|
+
const sweptUnits = [
|
|
53
|
+
...new Set(runs
|
|
54
|
+
.map((r) => r.varied_unit_id)
|
|
55
|
+
.filter((id) => typeof id === "string")),
|
|
56
|
+
];
|
|
57
|
+
// The base-effort sample is the frontier's reference point; a candidate-only
|
|
58
|
+
// report (e.g. run with --unit-sweep-candidate-only) omits it, so the frontier
|
|
59
|
+
// would recommend an effort purely because the baseline was never measured.
|
|
60
|
+
const hasBaseline = runs.some((r) => r.base_effort && r.case_id?.startsWith(UNIT_SWEEP_BASE_PREFIX));
|
|
61
|
+
if (sweptUnits.length > 0 && !hasBaseline) {
|
|
62
|
+
throw new Error("review report has unit-sweep candidates but no unit-sweep baseline (base-effort sample missing; was it run with --unit-sweep-candidate-only?)");
|
|
63
|
+
}
|
|
64
|
+
// Points (stage|effort) with at least one completed (gated) run. Failed runs
|
|
65
|
+
// carry no quality gate or route telemetry, so they are attributed only to a
|
|
66
|
+
// point that also has completed evidence — three failures alone must not stamp
|
|
67
|
+
// a unit-effort on a route no retained run proved executed there.
|
|
68
|
+
const completedPoints = new Set();
|
|
69
|
+
for (const run of runs) {
|
|
70
|
+
if (!run.semantic_quality_gate)
|
|
71
|
+
continue;
|
|
72
|
+
if (run.varied_unit_id && run.varied_effort) {
|
|
73
|
+
completedPoints.add(`${run.varied_unit_id}|${run.varied_effort}`);
|
|
74
|
+
}
|
|
75
|
+
else if (run.base_effort && run.case_id?.startsWith(UNIT_SWEEP_BASE_PREFIX)) {
|
|
76
|
+
for (const unit of sweptUnits)
|
|
77
|
+
completedPoints.add(`${unit}|${run.base_effort}`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
const out = [];
|
|
81
|
+
for (const run of runs) {
|
|
82
|
+
const gated = Boolean(run.semantic_quality_gate);
|
|
83
|
+
const gate = run.semantic_quality_gate
|
|
84
|
+
? reviewRunGateSignal(run.semantic_quality_gate)
|
|
85
|
+
: { ...UNJUDGED };
|
|
86
|
+
if (run.varied_unit_id && run.varied_effort) {
|
|
87
|
+
if (gated || completedPoints.has(`${run.varied_unit_id}|${run.varied_effort}`)) {
|
|
88
|
+
out.push({ stage: run.varied_unit_id, effort: run.varied_effort, gate });
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
else if (run.base_effort && run.case_id?.startsWith(UNIT_SWEEP_BASE_PREFIX)) {
|
|
92
|
+
for (const unit of sweptUnits) {
|
|
93
|
+
if (gated || completedPoints.has(`${unit}|${run.base_effort}`)) {
|
|
94
|
+
out.push({ stage: unit, effort: run.base_effort, gate });
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return out;
|
|
100
|
+
}
|
|
101
|
+
/** The answer-support judge's LLM step; no-call-exempt, so it can early-exit. */
|
|
102
|
+
export const JUDGE_STEP_ID = "answer_support_judgment";
|
|
103
|
+
/**
|
|
104
|
+
* Whether the answer-support judge actually ran an LLM call at `effort` in this
|
|
105
|
+
* run. The judge is no-call-exempt: it early-exits (no convergent evidence
|
|
106
|
+
* clusters) or degrades to an inherited config, in which case it leaves no
|
|
107
|
+
* `answer_support_judgment` telemetry at the requested effort — so a judge
|
|
108
|
+
* sample must be backed by real judge telemetry, not just the requested knob.
|
|
109
|
+
*/
|
|
110
|
+
function judgeExercisedAt(run, effort) {
|
|
111
|
+
return (run.units ?? []).some((u) => u.step_id === JUDGE_STEP_ID &&
|
|
112
|
+
u.effort === effort &&
|
|
113
|
+
(u.llm_call_count ?? 0) >= 1);
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Whether a completed run's telemetry shows the swept stage actually ran at
|
|
117
|
+
* `effort` — the requested knob is never trusted over telemetry. The author
|
|
118
|
+
* stage always runs, so its `metadata.applied_effort` must equal the point (a
|
|
119
|
+
* route that ignored the pin, or a recovery de-escalation, is not an `effort`
|
|
120
|
+
* sample). The judge stage must show an answer_support_judgment call at that
|
|
121
|
+
* effort. Runs that don't match are not evidence for this frontier point.
|
|
122
|
+
*/
|
|
123
|
+
function appliedEffortMatches(run, stage, effort) {
|
|
124
|
+
return stage === "judge"
|
|
125
|
+
? judgeExercisedAt(run, effort)
|
|
126
|
+
: run.metadata?.applied_effort === effort;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Derive the (stage, effort) a reconstruct report pinned. A judge override marks
|
|
130
|
+
* a judge-stage report ONLY when it pins a judge EFFORT; a model-only override
|
|
131
|
+
* (effort null/absent) varied the judge model, not an effort, so there is no
|
|
132
|
+
* effort point to calibrate and this returns null (the caller must pass an
|
|
133
|
+
* explicit tag) rather than mislabeling it as author. Without a judge override
|
|
134
|
+
* it is an author-stage report at the pinned `requested_effort`, falling back to
|
|
135
|
+
* the first run's telemetry `applied_effort` when the settings chain governed
|
|
136
|
+
* the effort. Returns null when no effort can be attributed.
|
|
137
|
+
*/
|
|
138
|
+
export function deriveReconstructTag(report) {
|
|
139
|
+
const judgeOverride = report.requested_judge_override;
|
|
140
|
+
if (judgeOverride) {
|
|
141
|
+
return judgeOverride.effort
|
|
142
|
+
? { stage: "judge", effort: judgeOverride.effort }
|
|
143
|
+
: null;
|
|
144
|
+
}
|
|
145
|
+
if (report.requested_effort) {
|
|
146
|
+
return { stage: "author", effort: report.requested_effort };
|
|
147
|
+
}
|
|
148
|
+
const applied = report.runs?.[0]?.metadata?.applied_effort;
|
|
149
|
+
if (applied)
|
|
150
|
+
return { stage: "author", effort: applied };
|
|
151
|
+
return null;
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Ingest a reconstruct benchmark report into sweep runs for one (stage, effort)
|
|
155
|
+
* point. The point is the explicit `tag` when given, else derived from the
|
|
156
|
+
* report's pinned knob. A completed run only contributes a sample when its
|
|
157
|
+
* telemetry shows the swept stage actually ran at that effort (author:
|
|
158
|
+
* applied_effort matches; judge: an answer_support_judgment call at that effort),
|
|
159
|
+
* so a pin the route ignored, a recovery de-escalation, or a judge early-exit
|
|
160
|
+
* does not fabricate a sample. Their golden gate is then distilled. Failed runs
|
|
161
|
+
* (no telemetry) are emitted unjudged for the author stage only — they cannot
|
|
162
|
+
* prove the judge ran. Throws when no tag can be determined.
|
|
163
|
+
*/
|
|
164
|
+
export function ingestReconstructReport(report, tag) {
|
|
165
|
+
const point = tag ?? deriveReconstructTag(report);
|
|
166
|
+
if (!point) {
|
|
167
|
+
throw new Error("reconstruct report pins no effort (no judge override, requested_effort, or applied_effort); pass an explicit stage:effort tag");
|
|
168
|
+
}
|
|
169
|
+
const out = [];
|
|
170
|
+
for (const run of report.runs ?? []) {
|
|
171
|
+
if (!appliedEffortMatches(run, point.stage, point.effort)) {
|
|
172
|
+
continue; // telemetry doesn't show this stage running at this effort
|
|
173
|
+
}
|
|
174
|
+
out.push({
|
|
175
|
+
stage: point.stage,
|
|
176
|
+
effort: point.effort,
|
|
177
|
+
gate: reconstructRunGateSignal(run.quality_gate),
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
// Failed runs carry no telemetry, so attribute them only to the author stage
|
|
181
|
+
// (which always runs) AND only when at least one completed run was retained at
|
|
182
|
+
// this point — that retained run's telemetry is what proves the model/route/
|
|
183
|
+
// effort. A source whose runs all failed proves no route/effort, so its
|
|
184
|
+
// failures are not turned into route-keyed samples.
|
|
185
|
+
if (point.stage === "author" && out.length > 0) {
|
|
186
|
+
const failedCount = report.reconstruct_extension?.failed_runs?.length ?? 0;
|
|
187
|
+
for (let i = 0; i < failedCount; i++) {
|
|
188
|
+
out.push({ stage: "author", effort: point.effort, gate: { ...UNJUDGED } });
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
return out;
|
|
192
|
+
}
|
|
193
|
+
// ---------------------------------------------------------------------------
|
|
194
|
+
// Route derivation (S1.3 — CLI consumer of the witnessed route identity)
|
|
195
|
+
// ---------------------------------------------------------------------------
|
|
196
|
+
/**
|
|
197
|
+
* Derive one reconstruct run's RouteIdentity. Prefers the harness-surfaced
|
|
198
|
+
* witnessed `route_identity` (S1.1/S1.2). A legacy report that predates route
|
|
199
|
+
* witnessing carries only the provider-only `provider_route`; that string is
|
|
200
|
+
* degraded through the witnessed constructor (adapter unknown → route_completeness
|
|
201
|
+
* `provider_only`, or `under_determined` for an unknown brand) so the CLI records
|
|
202
|
+
* an honest, degraded identity instead of failing (design §10). Returns null when
|
|
203
|
+
* neither is present (no route evidence on this run).
|
|
204
|
+
*/
|
|
205
|
+
export function reconstructRunRouteIdentity(run) {
|
|
206
|
+
const witnessed = run.metadata?.route_identity;
|
|
207
|
+
if (witnessed)
|
|
208
|
+
return witnessed;
|
|
209
|
+
const providerRoute = run.metadata?.provider_route;
|
|
210
|
+
if (providerRoute == null)
|
|
211
|
+
return null;
|
|
212
|
+
return witnessedReconstructRouteIdentity({
|
|
213
|
+
provider: providerRoute,
|
|
214
|
+
executionAdapter: null,
|
|
215
|
+
declaredBillingMode: null,
|
|
216
|
+
effectiveBaseUrl: null,
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Derive one review run's RouteIdentity from its (already structurally rich)
|
|
221
|
+
* runtime_route projection. Review has no result-level witness — the route is
|
|
222
|
+
* resolved from settings/profile — so provenance is profile_derived (design §4).
|
|
223
|
+
* A legacy report that carries only the provider-level `runtime_provider` token
|
|
224
|
+
* (no `model_provider`) falls back to the brand mapping, degrading to
|
|
225
|
+
* provider_only — symmetric with the reconstruct legacy path (design §10) rather
|
|
226
|
+
* than discarding the recoverable brand. Returns null when the run carries no
|
|
227
|
+
* route projection at all.
|
|
228
|
+
*/
|
|
229
|
+
export function reviewRunRouteIdentity(run) {
|
|
230
|
+
const route = run.review_profile?.runtime_route;
|
|
231
|
+
if (!route)
|
|
232
|
+
return null;
|
|
233
|
+
return profileDerivedRouteIdentity({
|
|
234
|
+
executionAdapter: route.execution_adapter ?? null,
|
|
235
|
+
modelProvider: route.model_provider ?? modelProviderFromRuntimeProvider(route.runtime_provider),
|
|
236
|
+
billingMode: route.billing_mode ?? null,
|
|
237
|
+
effectiveBaseUrl: route.base_url ?? null,
|
|
238
|
+
realization: route.artifact_generation_realization ?? null,
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
/**
|
|
242
|
+
* The route identities of the reconstruct runs that ACTUALLY feed the frontier —
|
|
243
|
+
* the same runs `ingestReconstructReport` retains (telemetry shows the swept
|
|
244
|
+
* stage ran at the point). Route evidence is collected from these retained
|
|
245
|
+
* samples only, never from runs whose applied effort / judge mismatch dropped
|
|
246
|
+
* them, so a discarded wrong-route run can't taint route_completeness or add a
|
|
247
|
+
* phantom route (finding: derive route evidence from retained samples). A
|
|
248
|
+
* retained run that carries no route witness contributes null (kept, not
|
|
249
|
+
* dropped, so the summary can downgrade for it). Failed runs carry no telemetry
|
|
250
|
+
* and assert no route, so they are excluded.
|
|
251
|
+
*/
|
|
252
|
+
export function reconstructRetainedRouteIdentities(report, tag) {
|
|
253
|
+
const point = tag ?? deriveReconstructTag(report);
|
|
254
|
+
if (!point)
|
|
255
|
+
return [];
|
|
256
|
+
const out = [];
|
|
257
|
+
for (const run of report.runs ?? []) {
|
|
258
|
+
if (!appliedEffortMatches(run, point.stage, point.effort))
|
|
259
|
+
continue;
|
|
260
|
+
out.push(retainedRunRouteIdentity(run, point));
|
|
261
|
+
}
|
|
262
|
+
return out;
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* The route identity that backs a retained sample at `point`. For an author
|
|
266
|
+
* point the run-level metadata route (= the first author unit) is the swept
|
|
267
|
+
* unit's route. For a JUDGE point the swept unit is the answer_support_judgment
|
|
268
|
+
* call, which may run on a different route than the author, so its OWN
|
|
269
|
+
* route_identity is read; a judge unit that ran without a route witness yields
|
|
270
|
+
* null (the route is undetermined, never silently the author's).
|
|
271
|
+
*/
|
|
272
|
+
function retainedRunRouteIdentity(run, point) {
|
|
273
|
+
if (point.stage === "judge") {
|
|
274
|
+
const judgeUnit = (run.units ?? []).find((u) => u.step_id === JUDGE_STEP_ID &&
|
|
275
|
+
u.effort === point.effort &&
|
|
276
|
+
(u.llm_call_count ?? 0) >= 1);
|
|
277
|
+
return judgeUnit?.route_identity ?? null;
|
|
278
|
+
}
|
|
279
|
+
return reconstructRunRouteIdentity(run);
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* The route identities of the review runs that feed the frontier — the gated
|
|
283
|
+
* (completed) candidate and unit-sweep baseline runs, mirroring
|
|
284
|
+
* `ingestReviewReport`'s attribution. Failed runs carry no quality gate or route
|
|
285
|
+
* telemetry, so only gated runs are sampled. A gated run missing its route
|
|
286
|
+
* projection contributes null (so the summary can downgrade for it).
|
|
287
|
+
*/
|
|
288
|
+
export function reviewRetainedRouteIdentities(report) {
|
|
289
|
+
const out = [];
|
|
290
|
+
for (const run of report.runs ?? []) {
|
|
291
|
+
if (!run.semantic_quality_gate)
|
|
292
|
+
continue;
|
|
293
|
+
const isCandidate = Boolean(run.varied_unit_id && run.varied_effort);
|
|
294
|
+
const isBaseline = Boolean(run.base_effort && run.case_id?.startsWith(UNIT_SWEEP_BASE_PREFIX));
|
|
295
|
+
if (!isCandidate && !isBaseline)
|
|
296
|
+
continue;
|
|
297
|
+
out.push(reviewRunRouteIdentity(run));
|
|
298
|
+
}
|
|
299
|
+
return out;
|
|
300
|
+
}
|
|
301
|
+
export function summarizeDerivedRoutes(derived, declaredRouteHint) {
|
|
302
|
+
// A null among contributing runs is a retained sample with no route witness —
|
|
303
|
+
// the profile cannot claim a complete route while including it (design §10).
|
|
304
|
+
const hasMissingWitness = derived.some((id) => id === null);
|
|
305
|
+
const byKey = new Map();
|
|
306
|
+
for (const identity of derived) {
|
|
307
|
+
if (!identity)
|
|
308
|
+
continue;
|
|
309
|
+
const key = JSON.stringify([routeToken(identity), identity.effective_base_url ?? null]);
|
|
310
|
+
if (!byKey.has(key))
|
|
311
|
+
byKey.set(key, identity);
|
|
312
|
+
}
|
|
313
|
+
const identities = [...byKey.values()];
|
|
314
|
+
const tokens = identities.map((id) => id.effective_base_url
|
|
315
|
+
? `${routeToken(id)}@${id.effective_base_url}`
|
|
316
|
+
: routeToken(id));
|
|
317
|
+
const completeness = identities.length === 0 || hasMissingWitness
|
|
318
|
+
? "under_determined"
|
|
319
|
+
: worstRouteCompleteness(identities.map((id) => id.route_completeness));
|
|
320
|
+
return {
|
|
321
|
+
identities,
|
|
322
|
+
tokens,
|
|
323
|
+
completeness,
|
|
324
|
+
hintCorroborated: identities.some((id) => routeHintMatches(declaredRouteHint, id)),
|
|
325
|
+
};
|
|
326
|
+
}
|