create-principles-disciple 1.145.42 → 1.146.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/_release/product-identity.json +2 -2
- package/console/dist/server/index.js +8 -0
- package/console/dist/server/routes/examiner-executor.d.ts +260 -0
- package/console/dist/server/routes/examiner-executor.js +628 -0
- package/console/dist/server/routes/examiner.d.ts +52 -0
- package/console/dist/server/routes/examiner.js +299 -0
- package/console/dist/ui/App.js +6 -3
- package/console/dist/ui/api.d.ts +138 -0
- package/console/dist/ui/api.js +56 -0
- package/console/dist/ui/components/layout/app-sidebar.d.ts +10 -0
- package/console/dist/ui/components/layout/app-sidebar.js +10 -2
- package/console/dist/ui/i18n/en.json +63 -0
- package/console/dist/ui/i18n/zh-CN.json +63 -0
- package/console/dist/ui/pages/examiner/ExaminerPage.d.ts +15 -0
- package/console/dist/ui/pages/examiner/ExaminerPage.js +158 -0
- package/console/dist/ui/pages/principles/PrincipleDetailPage.js +2 -1
- package/console/dist/ui/utils/runtime-mode.d.ts +11 -0
- package/console/dist/ui/utils/runtime-mode.js +11 -0
- package/console/dist/web/assets/app.css +44 -0
- package/console/dist/web/assets/app.js +33 -31
- package/console/lab/lib/challenge-actor.mjs +394 -0
- package/console/lab/lib/challenge-arms.mjs +102 -0
- package/console/lab/lib/challenge-behavior-report.mjs +2484 -0
- package/console/lab/lib/challenge-compile.mjs +211 -0
- package/console/lab/lib/challenge-freeze.mjs +215 -0
- package/console/lab/lib/challenge-isolation-probe.mjs +550 -0
- package/console/lab/lib/challenge-ledger.mjs +206 -0
- package/console/lab/lib/challenge-openclaw-host.mjs +630 -0
- package/console/lab/lib/challenge-runner.mjs +868 -0
- package/console/lab/lib/challenge-stage-r-gate.mjs +458 -0
- package/console/lab/lib/challenge-statistics.mjs +591 -0
- package/console/lab/lib/claim-evidence-env.mjs +652 -0
- package/console/lab/lib/claim-evidence-scorer.mjs +784 -0
- package/console/lab/lib/examiner-injection-evidence.mjs +367 -0
- package/console/lab/lib/examiner-study.mjs +1203 -0
- package/console/lab/lib/examiner-v07-cases.mjs +187 -0
- package/console/lab/lib/examiner-v2-cases.mjs +61 -0
- package/console/lab/lib/fixture.mjs +382 -0
- package/console/lab/lib/g2.mjs +146 -0
- package/console/lab/lib/host-actor.mjs +987 -0
- package/console/lab/lib/sidecar.mjs +273 -0
- package/package.json +1 -1
- package/pd-cli/package.json +1 -1
- package/release-manager/package.json +1 -1
|
@@ -0,0 +1,2484 @@
|
|
|
1
|
+
// SPEC-PD-EXAMINER-001 v0.4 — challenge study behavior report generator
|
|
2
|
+
// (SPEC §8 REPORT_READY lifecycle, §9 denominators/anomaly table/six report
|
|
3
|
+
// dimensions, §10 behavior-report.json + behavior-report.md, §12.4 / Phase 2b
|
|
4
|
+
// DoD "报告可从原始轨迹重算").
|
|
5
|
+
//
|
|
6
|
+
// Discipline (Owner-approved supplement 2026-10-26):
|
|
7
|
+
// - The ONLY truth sources are the study's append-only ledgers
|
|
8
|
+
// (budget-ledger.jsonl / events.jsonl / adjudications.jsonl), the frozen
|
|
9
|
+
// plan artifacts (preregistration.json, suite-manifest.json, hashes.json,
|
|
10
|
+
// run-authorization.json) and the per-attempt files under runs/. This
|
|
11
|
+
// module is a DERIVED read model — it never writes run state and never
|
|
12
|
+
// redefines scorer semantics: adjudication fields
|
|
13
|
+
// (behaviorAligned/forbiddenBehaviorOccurred/objectiveTaskOutcome/
|
|
14
|
+
// negativeTransfer/overCheck) are copied VERBATIM from
|
|
15
|
+
// claim-evidence-scorer output embedded in each attempt.
|
|
16
|
+
// - assignedDenominator comes from the PREREGISTERED plan (prereg arms ×
|
|
17
|
+
// frozen suite cases × planned repetitions). run-authorization.json only
|
|
18
|
+
// defines the authorized SUBSET and never shrinks the denominator.
|
|
19
|
+
// - Two-layer model: plan units are (caseId, armId, repetitionId);
|
|
20
|
+
// execution records are full (caseId, armId, repetitionId, attemptId).
|
|
21
|
+
// Nothing is deleted — an UNKNOWN followed by a recovered new attemptId
|
|
22
|
+
// keeps BOTH records; the primary analysis follows the ledger's own
|
|
23
|
+
// classifyUnit last-settlement authority, and any unit whose multiple
|
|
24
|
+
// SUCCEEDED attempts diverge is marked AMBIGUOUS and excluded from the
|
|
25
|
+
// paired main analysis (kept in the sensitivity block).
|
|
26
|
+
// - Normal UNKNOWN / FAILED_KNOWN settlements are NOT integrity errors.
|
|
27
|
+
// Critical integrity findings (missing key inputs, unparseable ledger,
|
|
28
|
+
// frozen-binding drift, duplicate settlement, ledger↔file conflict,
|
|
29
|
+
// real-mode execution without valid authorization) force
|
|
30
|
+
// REPORT_INCONCLUSIVE and suppress the REPORT_READY event.
|
|
31
|
+
// - Write mode refuses the in-repo frozen challenge-studies/ area at
|
|
32
|
+
// RUNTIME (resolved-path check, not calling discipline), takes the
|
|
33
|
+
// study single-writer lock, writes behavior-report.json and .md
|
|
34
|
+
// atomically (tmp+rename) and only THEN — and only when READY — appends
|
|
35
|
+
// exactly one REPORT_READY event binding the report sha256. A committed
|
|
36
|
+
// REPORT_READY report is never overwritten in place: identical
|
|
37
|
+
// recomputation is an idempotent no-op, anything else requires a new
|
|
38
|
+
// revision (ReportRevisionRequiredError).
|
|
39
|
+
// - verify mode is STRICTLY read-only: no lock, no writes, no events.
|
|
40
|
+
// Without an injected --now it reuses the existing report's generatedAt
|
|
41
|
+
// so an unmodified study verifies deterministically.
|
|
42
|
+
//
|
|
43
|
+
// Relationship to lib/stage-r-f1-report.mjs (PR #2165, operational F1 run
|
|
44
|
+
// recomputation, schema stage-r-f1-report.v1): complementary derived views
|
|
45
|
+
// over the same append-only truth; this module owns the SPEC §10 study
|
|
46
|
+
// artifact (challenge-behavior-report.v1, six §9.4 dimensions).
|
|
47
|
+
//
|
|
48
|
+
// Zero network, zero credentials, zero environment-clock reads: time enters
|
|
49
|
+
// only through the injected `now` parameter (same convention as the Stage R
|
|
50
|
+
// gate).
|
|
51
|
+
|
|
52
|
+
import fs from 'node:fs';
|
|
53
|
+
import path from 'node:path';
|
|
54
|
+
import { fileURLToPath } from 'node:url';
|
|
55
|
+
import { hashOfCanonical, challengeImplementationManifest } from './challenge-freeze.mjs';
|
|
56
|
+
import { buildClaimEnvironment, newClaimRunState, stepClaimEnvironment } from './claim-evidence-env.mjs';
|
|
57
|
+
import { adjudicateClaimEvidence } from './claim-evidence-scorer.mjs';
|
|
58
|
+
import { compileCase } from './challenge-compile.mjs';
|
|
59
|
+
import {
|
|
60
|
+
computeBFRR, computePCR, computeFIR, computePairedEffect, computeVBDR, causalAttributionStatus, buildPlannedPairs,
|
|
61
|
+
computeITT, computeTreatmentFidelity,
|
|
62
|
+
} from './challenge-statistics.mjs';
|
|
63
|
+
import { verifyExaminerManifest, verifyPairAssignment, definePrimaryEndpoint, evaluatePrimaryEndpointF, verifyExaminerStudyBindings, buildDualModeArmPayloads, buildHostEffectivePrincipleSet } from './examiner-study.mjs';
|
|
64
|
+
import { verifyInjectionEvidenceChain } from './examiner-injection-evidence.mjs';
|
|
65
|
+
import { verifyOfflineRuntimeReceipt } from './challenge-actor.mjs';
|
|
66
|
+
|
|
67
|
+
import { sha256, verifyCanonicalHash } from './challenge-freeze.mjs';
|
|
68
|
+
import { verifyFrozenStudy, isCapacityFailureDetail } from './challenge-runner.mjs';
|
|
69
|
+
import { createChallengeLedger, acquireStudyWriteLock, releaseStudyWriteLock } from './challenge-ledger.mjs';
|
|
70
|
+
|
|
71
|
+
export const BEHAVIOR_REPORT_SCHEMA = 'challenge-behavior-report.v1';
|
|
72
|
+
|
|
73
|
+
// v0.5 is additive: historical v1 reports continue to copy their recorded
|
|
74
|
+
// judgments. New offline evidence must instead replay actions under a pinned
|
|
75
|
+
// implementation before any saved judgment is counted.
|
|
76
|
+
export { challengeImplementationManifest };
|
|
77
|
+
|
|
78
|
+
export function replayChallengeExecution({ caseDef, execution, implementation, studyId = 'EXM-MVP-001', requirePersistedRunState = true }) {
|
|
79
|
+
if (hashOfCanonical(implementation) !== hashOfCanonical(challengeImplementationManifest())) {
|
|
80
|
+
throw new Error('IMPLEMENTATION_DRIFT');
|
|
81
|
+
}
|
|
82
|
+
if (caseDef.schema !== 'claim-evidence-judge-only.v2' && caseDef.schema !== 'claim-evidence-judge-only.v3') throw new Error('v2/v3 case required');
|
|
83
|
+
if (!execution || !execution.trace || !Array.isArray(execution.trace.records)
|
|
84
|
+
|| execution.trace.records.length === 0) throw new Error('TRAJECTORY_EVIDENCE_INSUFFICIENT');
|
|
85
|
+
if (execution.caseHash !== hashOfCanonical(caseDef)) throw new Error('GOLD_OR_INPUT_DRIFT');
|
|
86
|
+
// Two accepted linkage bindings (v0.7): the calibration-package shape
|
|
87
|
+
// { studyId, caseId }, or the runner shape whose inputHash must re-bind to
|
|
88
|
+
// this exact case's actor-visible projection (a STRONGER case binding that
|
|
89
|
+
// works for attempts produced by runChallengeStudy).
|
|
90
|
+
const linkage = execution.trace.linkage;
|
|
91
|
+
const calibrationLinkage = hashOfCanonical(linkage) === hashOfCanonical({ studyId, caseId: caseDef.caseId });
|
|
92
|
+
const runnerLinkage = linkage && typeof linkage === 'object' && typeof linkage.inputHash === 'string'
|
|
93
|
+
&& linkage.inputHash === compileCase(caseDef).inputHash;
|
|
94
|
+
if (execution.trace.schema !== 'actor-trace.v1' || execution.trace.actorKind !== 'stub-scripted'
|
|
95
|
+
|| (!calibrationLinkage && !runnerLinkage)) {
|
|
96
|
+
throw new Error('CALIBRATION_LINKAGE_INVALID');
|
|
97
|
+
}
|
|
98
|
+
const env = buildClaimEnvironment(caseDef);
|
|
99
|
+
const state = newClaimRunState(env);
|
|
100
|
+
let actions = 0;
|
|
101
|
+
let lastResult = null;
|
|
102
|
+
let invalidActions = 0;
|
|
103
|
+
const expectedObservations = [];
|
|
104
|
+
const expectedEvents = [];
|
|
105
|
+
for (const [seq, record] of execution.trace.records.entries()) {
|
|
106
|
+
if (!record || record.seq !== seq) throw new Error('TRACE_SEQUENCE_INVALID');
|
|
107
|
+
if (!['action', 'observation', 'invalid_action', 'narrative_carried', 'budget_event', 'consequence', 'termination', 'behavior_event'].includes(record.kind)) throw new Error('UNKNOWN_TRACE_RECORD');
|
|
108
|
+
if (record.kind === 'action') {
|
|
109
|
+
if (state.done) throw new Error('ACTION_AFTER_TERMINATION');
|
|
110
|
+
actions++;
|
|
111
|
+
lastResult = stepClaimEnvironment(env, state, record.action);
|
|
112
|
+
if (!lastResult.ok) invalidActions++;
|
|
113
|
+
if (!lastResult.ok) expectedEvents.push({ kind: 'invalid_action', error: lastResult.error });
|
|
114
|
+
else if (lastResult.record?.status === 'refused_budget') expectedEvents.push({ kind: 'budget_event', event: 'observation_refused_budget', obsId: lastResult.record.obsId, output: lastResult.record.output });
|
|
115
|
+
else if (record.action.type === 'report') expectedEvents.push({ kind: 'termination', reason: 'report_submitted', detail: 'actor submitted final claims' });
|
|
116
|
+
else if (record.action.type === 'deploy') expectedEvents.push({ kind: 'consequence', event: 'deploy', authorizationViolated: state.authorizationViolated, taskCompleted: state.taskCompleted });
|
|
117
|
+
else if (['refuse', 'block-path'].includes(record.action.type)) expectedEvents.push({ kind: 'behavior_event', event: record.action.type, args: record.action.args });
|
|
118
|
+
else expectedEvents.push({ kind: 'observation', obsId: lastResult.record.obsId, source: lastResult.record.source, status: lastResult.record.status, output: lastResult.record.output,
|
|
119
|
+
eventId: lastResult.record.eventId, ...(lastResult.record.requestId ? { requestId: lastResult.record.requestId, requestHash: lastResult.record.requestHash } : {}) });
|
|
120
|
+
if (lastResult.ok && lastResult.record && lastResult.record.status === 'ok') {
|
|
121
|
+
expectedObservations.push(Object.fromEntries(['obsId', 'source', 'status', 'output']
|
|
122
|
+
.map((key) => [key, lastResult.record[key]])));
|
|
123
|
+
}
|
|
124
|
+
} else if (record.kind === 'observation') {
|
|
125
|
+
if (!lastResult || !lastResult.record || ['obsId', 'source', 'status', 'output']
|
|
126
|
+
.some((key) => record[key] !== lastResult.record[key])) throw new Error('OBSERVATION_REPLAY_MISMATCH');
|
|
127
|
+
} else if (record.kind === 'invalid_action') {
|
|
128
|
+
if (!lastResult || lastResult.ok || record.error !== lastResult.error) throw new Error('INVALID_ACTION_REPLAY_MISMATCH');
|
|
129
|
+
} else if (record.kind === 'consequence') {
|
|
130
|
+
if (record.event !== 'deploy' || record.authorizationViolated !== state.authorizationViolated
|
|
131
|
+
|| record.taskCompleted !== state.taskCompleted) throw new Error('CONSEQUENCE_REPLAY_MISMATCH');
|
|
132
|
+
} else if (record.kind === 'budget_event') {
|
|
133
|
+
if (!lastResult?.record || lastResult.record.status !== 'refused_budget'
|
|
134
|
+
|| record.obsId !== lastResult.record.obsId || record.output !== lastResult.output) throw new Error('BUDGET_REPLAY_MISMATCH');
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
if (actions === 0) throw new Error('TRAJECTORY_EVIDENCE_INSUFFICIENT');
|
|
138
|
+
const actualEvents = execution.trace.records.filter((record) => !['action', 'narrative_carried'].includes(record.kind))
|
|
139
|
+
.map(({ seq: _seq, ...event }) => event);
|
|
140
|
+
if (hashOfCanonical(actualEvents) !== hashOfCanonical(expectedEvents)) throw new Error('TRACE_EVENTS_REPLAY_MISMATCH');
|
|
141
|
+
if (!state.report || hashOfCanonical(execution.trace.termination) !== hashOfCanonical({ reason: 'report_submitted', actorRunStatus: 'COMPLETED' })) {
|
|
142
|
+
throw new Error('EXECUTION_INCOMPLETE');
|
|
143
|
+
}
|
|
144
|
+
const usage = { steps: state.step, invalidActions,
|
|
145
|
+
observationsExecuted: state.executedObservations.filter((r) => r.status === 'ok').length,
|
|
146
|
+
budgetRefusals: state.refusedCount, budgetUsed: state.budgetUsed, timeUnits: state.timeUsed };
|
|
147
|
+
if (hashOfCanonical(usage) !== hashOfCanonical(execution.trace.usage)) throw new Error('USAGE_REPLAY_MISMATCH');
|
|
148
|
+
const observed = execution.trace.records.filter((r) => r.kind === 'observation')
|
|
149
|
+
.map((r) => Object.fromEntries(['obsId', 'source', 'status', 'output'].map((key) => [key, r[key]])));
|
|
150
|
+
if (hashOfCanonical(observed) !== hashOfCanonical(expectedObservations)) throw new Error('OBSERVATION_REPLAY_MISMATCH');
|
|
151
|
+
if (hashOfCanonical(execution.trace.finalReport) !== hashOfCanonical(state.report)) throw new Error('REPORT_REPLAY_MISMATCH');
|
|
152
|
+
// Runner-produced attempts persist trace + adjudication but not the run
|
|
153
|
+
// state object; there the RECOMPUTED state is the authority and the stored
|
|
154
|
+
// adjudication comparison below carries the semantic check.
|
|
155
|
+
if (requirePersistedRunState) {
|
|
156
|
+
if (hashOfCanonical(state) !== hashOfCanonical(execution.runState)) throw new Error('STATE_REPLAY_MISMATCH');
|
|
157
|
+
} else if (execution.runState !== undefined && hashOfCanonical(state) !== hashOfCanonical(execution.runState)) {
|
|
158
|
+
throw new Error('STATE_REPLAY_MISMATCH');
|
|
159
|
+
}
|
|
160
|
+
const adjudication = adjudicateClaimEvidence({ caseDef, runState: state });
|
|
161
|
+
if (hashOfCanonical(adjudication) !== hashOfCanonical(execution.adjudication)) throw new Error('SCORE_REPLAY_MISMATCH');
|
|
162
|
+
return adjudication;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
export function buildProductizationCalibrationReport({ cases, executions, implementation, candidateCounts, studyId = 'EXM-MVP-001' }) {
|
|
166
|
+
if (hashOfCanonical(implementation) !== hashOfCanonical(challengeImplementationManifest())) throw new Error('IMPLEMENTATION_DRIFT');
|
|
167
|
+
if (!Array.isArray(cases) || !Array.isArray(executions)) throw new Error('cases/executions required');
|
|
168
|
+
const caseMap = new Map(cases.map((c) => [c.caseId, c]));
|
|
169
|
+
if (caseMap.size !== cases.length) throw new Error('duplicate case ID');
|
|
170
|
+
const ids = new Set();
|
|
171
|
+
const rows = executions.map((execution) => {
|
|
172
|
+
if (!execution || typeof execution.id !== 'string' || ids.has(execution.id)) throw new Error('duplicate/missing execution ID');
|
|
173
|
+
ids.add(execution.id);
|
|
174
|
+
const caseDef = caseMap.get(execution.caseId);
|
|
175
|
+
if (!caseDef) throw new Error('unknown execution case');
|
|
176
|
+
const score = replayChallengeExecution({ caseDef, execution, implementation, studyId });
|
|
177
|
+
return { id: execution.id, caseId: caseDef.caseId, strategy: execution.strategy, score };
|
|
178
|
+
});
|
|
179
|
+
const denominator = rows.length;
|
|
180
|
+
const negativeTransfer = rows.filter((r) => r.score.negativeTransfer === true).length;
|
|
181
|
+
const knownCandidates = candidateCounts && Number.isSafeInteger(candidateCounts.total) && candidateCounts.total >= 0;
|
|
182
|
+
if (candidateCounts && candidateCounts.total !== null && !knownCandidates) throw new Error('invalid candidate counts');
|
|
183
|
+
if (knownCandidates && (!Number.isSafeInteger(candidateCounts.valid) || candidateCounts.valid < 0
|
|
184
|
+
|| candidateCounts.valid > candidateCounts.total)) throw new Error('invalid candidate counts');
|
|
185
|
+
return {
|
|
186
|
+
schema: 'challenge-productization-calibration-report.v1',
|
|
187
|
+
status: 'OFFLINE_ONLY_NOT_AUTHORIZED',
|
|
188
|
+
implementationHash: hashOfCanonical(implementation),
|
|
189
|
+
rows,
|
|
190
|
+
executionDenominator: denominator,
|
|
191
|
+
negativeTransfer: { numerator: negativeTransfer, denominator },
|
|
192
|
+
boundaryNegativeTransfer: {
|
|
193
|
+
numerator: rows.filter((r) => caseMap.get(r.caseId).principleApplicable === false && r.score.negativeTransfer === true).length,
|
|
194
|
+
denominator: rows.filter((r) => caseMap.get(r.caseId).principleApplicable === false).length,
|
|
195
|
+
},
|
|
196
|
+
negativeTransferMetrics: Object.fromEntries(['unnecessaryRefusal', 'alternativePathBlocking', 'repeatedConfirmation', 'inapplicableTrigger'].map((type) => [type, {
|
|
197
|
+
numerator: rows.filter((row) => row.score.negativeTransferTypes[type].some((event) => event.status === 'DETECTED')).length,
|
|
198
|
+
unknown: rows.filter((row) => row.score.negativeTransferTypes[type].some((event) => event.status === 'UNKNOWN')).length,
|
|
199
|
+
denominator, deduplication: 'one execution per type; overlapping types never summed',
|
|
200
|
+
}])),
|
|
201
|
+
overChecking: { numerator: rows.filter((row) => row.score.overCheck.overCheckOccurred).length, denominator,
|
|
202
|
+
unnecessaryCostUnits: rows.reduce((sum, row) => sum + row.score.overCheck.unnecessaryCostUnits, 0) },
|
|
203
|
+
candidateCounts,
|
|
204
|
+
validCandidateRate: knownCandidates && candidateCounts.total > 0
|
|
205
|
+
? { numerator: candidateCounts.valid, denominator: candidateCounts.total }
|
|
206
|
+
: { status: 'NOT_ESTIMABLE', reason: 'empty_or_unknown_candidate_denominator' },
|
|
207
|
+
goldReview: 'GOLD_REVIEW_PENDING',
|
|
208
|
+
realExaminerValidCases: 0,
|
|
209
|
+
validatedBehavioralDiscriminationRate: { status: 'NOT_RUN', reason: 'calibration_is_not_real_behavior_experiment' },
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
export const REPORT_STATUS_READY = 'REPORT_READY';
|
|
213
|
+
export const REPORT_STATUS_INCONCLUSIVE = 'REPORT_INCONCLUSIVE';
|
|
214
|
+
export const REPORT_DIMENSIONS = [
|
|
215
|
+
'protocolEvidenceQuality',
|
|
216
|
+
'behavioralBenefit',
|
|
217
|
+
'boundaryNegativeTransfer',
|
|
218
|
+
'objectiveTaskOutcome',
|
|
219
|
+
'cost',
|
|
220
|
+
'coverage',
|
|
221
|
+
];
|
|
222
|
+
|
|
223
|
+
const FROZEN_STUDIES_AREA = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', 'challenge-studies');
|
|
224
|
+
const REPORT_JSON = 'behavior-report.json';
|
|
225
|
+
const REPORT_MD = 'behavior-report.md';
|
|
226
|
+
const FREEZE_REFERENCE_FIELDS = [
|
|
227
|
+
'suiteHash',
|
|
228
|
+
'mappingHash',
|
|
229
|
+
'graderHash',
|
|
230
|
+
'principleContentHash',
|
|
231
|
+
'summaryPayloadHash',
|
|
232
|
+
'commonContextHash',
|
|
233
|
+
];
|
|
234
|
+
const SEVERITY_ORDER = { critical: 0, major: 1, info: 2 };
|
|
235
|
+
|
|
236
|
+
export class FrozenStudyWriteRefusedError extends Error {
|
|
237
|
+
constructor(target, frozenArea) {
|
|
238
|
+
super(`refusing to write a behavior report into the frozen study area: ${target} (frozen area: ${frozenArea}) — run the generator against a study COPY`);
|
|
239
|
+
this.name = 'FrozenStudyWriteRefusedError';
|
|
240
|
+
this.target = target;
|
|
241
|
+
this.frozenArea = frozenArea;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
export class ReportRevisionRequiredError extends Error {
|
|
246
|
+
constructor(detail) {
|
|
247
|
+
super(`a REPORT_READY report is already committed and the recomputation differs — overwrite refused, create a new study revision instead: ${detail}`);
|
|
248
|
+
this.name = 'ReportRevisionRequiredError';
|
|
249
|
+
this.detail = detail;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// ---------------------------------------------------------------------------
|
|
254
|
+
// local read helpers (mirror challenge-ledger readLines semantics: a torn
|
|
255
|
+
// final line is reported as malformedTail and dropped, never interpreted)
|
|
256
|
+
// ---------------------------------------------------------------------------
|
|
257
|
+
|
|
258
|
+
function readJsonFile(file) {
|
|
259
|
+
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function readJsonl(file) {
|
|
263
|
+
if (!fs.existsSync(file)) return { entries: [], malformedTail: false, missing: true };
|
|
264
|
+
const raw = fs.readFileSync(file, 'utf8');
|
|
265
|
+
const lines = raw === '' ? [] : raw.split('\n');
|
|
266
|
+
let malformedTail = false;
|
|
267
|
+
if (lines.length > 0 && lines[lines.length - 1] === '') lines.pop();
|
|
268
|
+
else if (lines.length > 0) malformedTail = true;
|
|
269
|
+
const entries = [];
|
|
270
|
+
for (const line of lines) {
|
|
271
|
+
if (line.trim() === '') continue;
|
|
272
|
+
try {
|
|
273
|
+
entries.push(JSON.parse(line));
|
|
274
|
+
} catch {
|
|
275
|
+
malformedTail = true;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
return { entries, malformedTail, missing: false };
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
const unitKey = (u) => `${u.caseId}/${u.armId}/r${String(u.repetitionId)}/${String(u.attemptId)}`;
|
|
282
|
+
const planUnitKey = (caseId, armId, repetitionId) => `${caseId}/${armId}/r${String(repetitionId)}`;
|
|
283
|
+
|
|
284
|
+
function normalizeNow(now) {
|
|
285
|
+
if (now === undefined || now === null) return null;
|
|
286
|
+
const ms = typeof now === 'number' ? now : Date.parse(now);
|
|
287
|
+
if (!Number.isFinite(ms)) throw new Error(`buildBehaviorReport: invalid now: ${String(now)} (ISO-8601 required)`);
|
|
288
|
+
return new Date(ms).toISOString();
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
function collectAttemptFiles(studyDir) {
|
|
292
|
+
const runsDir = path.join(studyDir, 'runs');
|
|
293
|
+
const files = [];
|
|
294
|
+
if (!fs.existsSync(runsDir)) return files;
|
|
295
|
+
for (const caseId of fs.readdirSync(runsDir).sort()) {
|
|
296
|
+
const caseDir = path.join(runsDir, caseId);
|
|
297
|
+
if (!fs.statSync(caseDir).isDirectory()) continue;
|
|
298
|
+
for (const armId of fs.readdirSync(caseDir).sort()) {
|
|
299
|
+
const armDir = path.join(caseDir, armId);
|
|
300
|
+
if (!fs.statSync(armDir).isDirectory()) continue;
|
|
301
|
+
for (const repDir of fs.readdirSync(armDir).sort()) {
|
|
302
|
+
const repPath = path.join(armDir, repDir);
|
|
303
|
+
if (!fs.statSync(repPath).isDirectory()) continue;
|
|
304
|
+
for (const f of fs.readdirSync(repPath).sort()) {
|
|
305
|
+
if (!f.endsWith('.json') || f.endsWith('.tmp')) continue;
|
|
306
|
+
// v0.9 injection-evidence sidecars are NOT attempt records — the
|
|
307
|
+
// attempt collector must not mistake them for actor executions.
|
|
308
|
+
if (f.startsWith('injection-evidence-')) continue;
|
|
309
|
+
files.push({ rel: path.join('runs', caseId, armId, repDir, f), abs: path.join(repPath, f) });
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
return files;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
function classifyKnownFailure(error) {
|
|
318
|
+
if (typeof error !== 'string' || error === '') return 'unclassified';
|
|
319
|
+
const message = error.startsWith('execution_error: ') ? error.slice('execution_error: '.length) : error;
|
|
320
|
+
if (isCapacityFailureDetail(message)) return 'capacity_failure';
|
|
321
|
+
if (message.startsWith('model_identity_')) return 'identity_failure';
|
|
322
|
+
return 'protocol_contract_failure';
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
function sumUsage(rounds) {
|
|
326
|
+
let prompt = 0;
|
|
327
|
+
let completion = 0;
|
|
328
|
+
let total = 0;
|
|
329
|
+
let observed = 0;
|
|
330
|
+
for (const r of Array.isArray(rounds) ? rounds : []) {
|
|
331
|
+
const u = r && r.usage;
|
|
332
|
+
if (!u || typeof u !== 'object') continue;
|
|
333
|
+
observed += 1;
|
|
334
|
+
prompt += Number(u.prompt_tokens) || 0;
|
|
335
|
+
completion += Number(u.completion_tokens) || 0;
|
|
336
|
+
total += Number(u.total_tokens) || 0;
|
|
337
|
+
}
|
|
338
|
+
return { prompt, completion, total, observedRounds: observed };
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
// ---------------------------------------------------------------------------
|
|
342
|
+
// buildBehaviorReport — pure read-only recomputation from raw evidence
|
|
343
|
+
// ---------------------------------------------------------------------------
|
|
344
|
+
|
|
345
|
+
export function buildBehaviorReport(studyDir, { now } = {}) {
|
|
346
|
+
const findings = [];
|
|
347
|
+
const addFinding = (id, severity, subject, detail) => {
|
|
348
|
+
findings.push({ id, severity, subject, detail });
|
|
349
|
+
};
|
|
350
|
+
|
|
351
|
+
// --- frozen plan artifacts ------------------------------------------------
|
|
352
|
+
let hashes = null;
|
|
353
|
+
let prereg = null;
|
|
354
|
+
let manifest = null;
|
|
355
|
+
let authRecord = null;
|
|
356
|
+
|
|
357
|
+
for (const [name, file, setter] of [
|
|
358
|
+
['hashes.json', 'hashes.json', (v) => { hashes = v; }],
|
|
359
|
+
['preregistration.json', 'preregistration.json', (v) => { prereg = v; }],
|
|
360
|
+
['suite-manifest.json', 'suite-manifest.json', (v) => { manifest = v; }],
|
|
361
|
+
['run-authorization.json', 'run-authorization.json', (v) => { authRecord = v; }],
|
|
362
|
+
]) {
|
|
363
|
+
try {
|
|
364
|
+
setter(readJsonFile(path.join(studyDir, file)));
|
|
365
|
+
} catch (e) {
|
|
366
|
+
if (name === 'run-authorization.json') {
|
|
367
|
+
authRecord = null; // absent authorization is itself reportable, not fatal to read
|
|
368
|
+
} else {
|
|
369
|
+
addFinding('key_input_missing', 'critical', file, `${name} unreadable: ${e.message}`);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
// --- frozen binding verification (reuse the runner's own authority) ------
|
|
375
|
+
let bindingsVerified = true;
|
|
376
|
+
try {
|
|
377
|
+
verifyFrozenStudy(studyDir);
|
|
378
|
+
} catch (e) {
|
|
379
|
+
bindingsVerified = false;
|
|
380
|
+
addFinding('frozen_binding_drift', 'critical', 'suite', e.message);
|
|
381
|
+
}
|
|
382
|
+
if (hashes && prereg) {
|
|
383
|
+
const check = verifyCanonicalHash(prereg, hashes.preregistrationHash && hashes.preregistrationHash.value);
|
|
384
|
+
if (!check.ok) {
|
|
385
|
+
bindingsVerified = false;
|
|
386
|
+
addFinding('preregistration_hash_drift', 'critical', 'preregistration.json', `recomputed ${check.recomputed.slice(0, 16)}… != recorded ${String(check.recordedHash).slice(0, 16)}…`);
|
|
387
|
+
}
|
|
388
|
+
for (const field of FREEZE_REFERENCE_FIELDS) {
|
|
389
|
+
const fromPrereg = prereg.freezeReferences && prereg.freezeReferences[field];
|
|
390
|
+
const fromHashes = hashes[field] && hashes[field].value;
|
|
391
|
+
if (fromPrereg !== undefined && fromHashes !== undefined && fromPrereg !== fromHashes) {
|
|
392
|
+
bindingsVerified = false;
|
|
393
|
+
addFinding('freeze_reference_mismatch', 'critical', `preregistration.freezeReferences.${field}`, `${String(fromPrereg).slice(0, 16)}… != hashes.json ${String(fromHashes).slice(0, 16)}…`);
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
// --- authorization scope --------------------------------------------------
|
|
399
|
+
const manifestHashes = manifest
|
|
400
|
+
? { suiteHash: manifest.suiteHash, mappingHash: manifest.mappingHash, graderHash: manifest.graderHash }
|
|
401
|
+
: null;
|
|
402
|
+
const studyPlanHashValue = hashes && hashes.preregistrationHash ? hashes.preregistrationHash.value : null;
|
|
403
|
+
const authBindingsMatch = authRecord && manifestHashes && studyPlanHashValue
|
|
404
|
+
? authRecord.suiteHash === manifestHashes.suiteHash
|
|
405
|
+
&& authRecord.mappingHash === manifestHashes.mappingHash
|
|
406
|
+
&& authRecord.graderHash === manifestHashes.graderHash
|
|
407
|
+
&& authRecord.studyPlanHash === studyPlanHashValue
|
|
408
|
+
: null;
|
|
409
|
+
const authStatus = authRecord && typeof authRecord.status === 'string' ? authRecord.status : null;
|
|
410
|
+
const authorizationValid = authStatus === 'APPROVED' && authBindingsMatch === true && authRecord.revoked !== true;
|
|
411
|
+
if (authStatus === 'NOT_AUTHORIZED') {
|
|
412
|
+
addFinding('authorization_placeholder_present', 'info', 'run-authorization.json', 'fail-closed NOT_AUTHORIZED placeholder — not a valid real-execution authorization');
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
// --- ledgers --------------------------------------------------------------
|
|
416
|
+
const budget = readJsonl(path.join(studyDir, 'budget-ledger.jsonl'));
|
|
417
|
+
const events = readJsonl(path.join(studyDir, 'events.jsonl'));
|
|
418
|
+
const adjudications = readJsonl(path.join(studyDir, 'adjudications.jsonl'));
|
|
419
|
+
for (const [name, parsed] of [['budget-ledger.jsonl', budget], ['events.jsonl', events], ['adjudications.jsonl', adjudications]]) {
|
|
420
|
+
if (parsed.malformedTail) {
|
|
421
|
+
addFinding('ledger_malformed_tail', 'critical', name, 'final line torn or unparseable — dropped entries make every downstream count unreliable');
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
// --- attempt files --------------------------------------------------------
|
|
426
|
+
const attemptFiles = collectAttemptFiles(studyDir);
|
|
427
|
+
const attemptsByUnitKey = new Map(); // unitKey -> {file, parsed}
|
|
428
|
+
for (const af of attemptFiles) {
|
|
429
|
+
let parsed;
|
|
430
|
+
try {
|
|
431
|
+
parsed = readJsonFile(af.abs);
|
|
432
|
+
} catch (e) {
|
|
433
|
+
addFinding('attempt_file_unparseable', 'critical', af.rel, e.message);
|
|
434
|
+
continue;
|
|
435
|
+
}
|
|
436
|
+
const unit = parsed && parsed.unit;
|
|
437
|
+
if (!unit || !unit.caseId || !unit.armId) {
|
|
438
|
+
addFinding('attempt_file_unit_missing', 'critical', af.rel, 'attempt record carries no usable unit identity');
|
|
439
|
+
continue;
|
|
440
|
+
}
|
|
441
|
+
const key = unitKey(unit);
|
|
442
|
+
const fromPath = af.rel.split(path.sep).slice(1, 4).join('/');
|
|
443
|
+
const fromRecord = `${unit.caseId}/${unit.armId}/r${String(unit.repetitionId)}`;
|
|
444
|
+
if (fromPath !== fromRecord) {
|
|
445
|
+
addFinding('attempt_path_unit_mismatch', 'major', af.rel, `path says ${fromPath}, record says ${fromRecord}`);
|
|
446
|
+
}
|
|
447
|
+
if (attemptsByUnitKey.has(key)) {
|
|
448
|
+
addFinding('attempt_file_duplicate_unit', 'critical', key, `multiple attempt files claim the same (case,arm,rep,attempt): ${af.rel}`);
|
|
449
|
+
}
|
|
450
|
+
attemptsByUnitKey.set(key, { file: af.rel, parsed });
|
|
451
|
+
if (parsed.mode && parsed.mode !== 'offline-stub' && parsed.authorization && manifestHashes
|
|
452
|
+
&& parsed.authorization.suiteHash && parsed.authorization.suiteHash !== manifestHashes.suiteHash) {
|
|
453
|
+
addFinding('attempt_authorization_binding_mismatch', 'critical', af.rel, `attempt authorization.suiteHash ${String(parsed.authorization.suiteHash).slice(0, 16)}… != frozen suiteHash`);
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
// --- ledger grouping + ledger↔file cross-checks ---------------------------
|
|
458
|
+
const rowsByUnitKey = new Map(); // unitKey -> {reserve, prepared, results[]}
|
|
459
|
+
for (const row of budget.entries) {
|
|
460
|
+
if (!row || !row.unit || !row.unit.caseId) continue;
|
|
461
|
+
const key = unitKey(row.unit);
|
|
462
|
+
if (!rowsByUnitKey.has(key)) rowsByUnitKey.set(key, { reserve: false, prepared: false, results: [] });
|
|
463
|
+
const g = rowsByUnitKey.get(key);
|
|
464
|
+
if (row.type === 'RESERVE') g.reserve = true;
|
|
465
|
+
if (row.type === 'ATTEMPT_PREPARED') g.prepared = true;
|
|
466
|
+
if (row.type === 'ATTEMPT_RESULT') g.results.push(row.result);
|
|
467
|
+
}
|
|
468
|
+
for (const [key, g] of rowsByUnitKey) {
|
|
469
|
+
if (g.results.length > 1) {
|
|
470
|
+
addFinding('duplicate_settlement', 'critical', key, `attempt settled ${g.results.length} times in the budget ledger: ${g.results.join(', ')}`);
|
|
471
|
+
}
|
|
472
|
+
const file = attemptsByUnitKey.get(key);
|
|
473
|
+
if (g.results.length > 0 && !file) {
|
|
474
|
+
addFinding('ledger_result_without_attempt_file', 'critical', key, `ledger settled ${g.results[g.results.length - 1]} but no attempt file exists`);
|
|
475
|
+
}
|
|
476
|
+
if (g.results.length > 0 && file) {
|
|
477
|
+
const ledgerResult = g.results[g.results.length - 1];
|
|
478
|
+
if (ledgerResult !== file.parsed.result) {
|
|
479
|
+
addFinding('ledger_file_result_conflict', 'critical', key, `ledger says ${ledgerResult}, attempt file says ${String(file.parsed.result)}`);
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
for (const [key, file] of attemptsByUnitKey) {
|
|
484
|
+
const g = rowsByUnitKey.get(key);
|
|
485
|
+
if (!g || !g.reserve) {
|
|
486
|
+
addFinding('orphan_attempt_file', 'major', key, `attempt file ${file.file} exists without a RESERVE row in the budget ledger`);
|
|
487
|
+
}
|
|
488
|
+
if (file.parsed.result === 'SUCCEEDED' && !(file.parsed.adjudication && typeof file.parsed.adjudication === 'object')) {
|
|
489
|
+
addFinding('succeeded_attempt_missing_adjudication', 'critical', key, 'SUCCEEDED attempt carries no adjudication record — §9.2 per-execution record cannot be built');
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
// --- real-mode vs authorization -------------------------------------------
|
|
494
|
+
const seenModes = new Set();
|
|
495
|
+
for (const { parsed } of attemptsByUnitKey.values()) if (parsed.mode) seenModes.add(parsed.mode);
|
|
496
|
+
const realModeAttempts = [...attemptsByUnitKey.values()].filter(({ parsed }) => parsed.mode && parsed.mode !== 'offline-stub').length;
|
|
497
|
+
if (realModeAttempts > 0 && !authorizationValid) {
|
|
498
|
+
addFinding('real_execution_without_valid_authorization', 'critical', 'run-authorization.json',
|
|
499
|
+
`${realModeAttempts} real-mode attempt(s) but the authorization record is ${authStatus === 'APPROVED' ? 'not bound to the frozen hashes or revoked' : String(authStatus)}`);
|
|
500
|
+
}
|
|
501
|
+
if (realModeAttempts === 0 && budget.entries.length > 0) {
|
|
502
|
+
addFinding('authorization_not_required_offline', 'info', 'run-authorization.json', 'only offline-stub executions observed — Stage R authorization not required (SPEC §7.2 gates real modes)');
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
// --- preregistered plan units --------------------------------------------
|
|
506
|
+
const preregArms = prereg && Array.isArray(prereg.arms) ? prereg.arms.map((a) => a.armId).sort() : [];
|
|
507
|
+
const plannedRepetitions = prereg && prereg.repetitions && Number(prereg.repetitions.planned) > 0
|
|
508
|
+
? Array.from({ length: Number(prereg.repetitions.planned) }, (_, i) => String(i + 1))
|
|
509
|
+
: [];
|
|
510
|
+
const cases = manifest && Array.isArray(manifest.cases) ? manifest.cases : [];
|
|
511
|
+
const caseById = new Map(cases.map((c) => [c.caseId, c]));
|
|
512
|
+
const planUnits = [];
|
|
513
|
+
const planUnitKeys = new Set();
|
|
514
|
+
if (preregArms.length > 0 && plannedRepetitions.length > 0 && cases.length > 0) {
|
|
515
|
+
for (const c of [...cases].sort((a, b) => a.caseId.localeCompare(b.caseId))) {
|
|
516
|
+
for (const armId of preregArms) {
|
|
517
|
+
for (const rep of plannedRepetitions) {
|
|
518
|
+
planUnits.push({ caseId: c.caseId, armId, repetitionId: rep });
|
|
519
|
+
planUnitKeys.add(planUnitKey(c.caseId, armId, rep));
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
} else if (budget.entries.length > 0 || attemptsByUnitKey.size > 0) {
|
|
524
|
+
addFinding('denominator_unverifiable', 'critical', 'preregistration.json', 'preregistered arms/repetitions or frozen case list unavailable — assignedDenominator cannot be computed');
|
|
525
|
+
}
|
|
526
|
+
const assignedDenominator = planUnits.length;
|
|
527
|
+
|
|
528
|
+
// out-of-scope executions (ledger or file units outside the prereg plan)
|
|
529
|
+
const outOfScope = [];
|
|
530
|
+
const seenUnitPlans = new Map(); // planUnitKey -> Set(attemptId)
|
|
531
|
+
for (const key of new Set([...rowsByUnitKey.keys(), ...attemptsByUnitKey.keys()])) {
|
|
532
|
+
const parts = key.split('/');
|
|
533
|
+
const planKey = parts.length === 4 ? planUnitKey(parts[0], parts[1], parts[2].replace(/^r/, '')) : key;
|
|
534
|
+
const attemptId = parts.length === 4 ? parts[3] : '';
|
|
535
|
+
if (!planUnitKeys.has(planKey)) {
|
|
536
|
+
outOfScope.push({ unit: key, planKey });
|
|
537
|
+
} else {
|
|
538
|
+
if (!seenUnitPlans.has(planKey)) seenUnitPlans.set(planKey, new Set());
|
|
539
|
+
if (attemptId) seenUnitPlans.get(planKey).add(attemptId);
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
outOfScope.sort((a, b) => a.unit.localeCompare(b.unit));
|
|
543
|
+
for (const o of outOfScope) {
|
|
544
|
+
addFinding('execution_outside_prereg_scope', 'major', o.unit, `execution observed for ${o.planKey} which is outside the preregistered plan — retained separately, not counted in the denominator`);
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
// --- per-attempt execution records (full retention, sorted) --------------
|
|
548
|
+
const executions = [];
|
|
549
|
+
for (const key of [...attemptsByUnitKey.keys()].sort()) {
|
|
550
|
+
const { file, parsed } = attemptsByUnitKey.get(key);
|
|
551
|
+
const g = rowsByUnitKey.get(key) || { results: [] };
|
|
552
|
+
const adj = parsed.adjudication && typeof parsed.adjudication === 'object' ? parsed.adjudication : null;
|
|
553
|
+
const rounds = Array.isArray(parsed.actorRounds) ? parsed.actorRounds : [];
|
|
554
|
+
const usage = sumUsage(rounds);
|
|
555
|
+
const traceSteps = parsed.trace && Array.isArray(parsed.trace.records) ? parsed.trace.records.length : null;
|
|
556
|
+
executions.push({
|
|
557
|
+
unit: {
|
|
558
|
+
caseId: parsed.unit.caseId,
|
|
559
|
+
armId: parsed.unit.armId,
|
|
560
|
+
repetitionId: parsed.unit.repetitionId,
|
|
561
|
+
attemptId: parsed.unit.attemptId,
|
|
562
|
+
},
|
|
563
|
+
file,
|
|
564
|
+
mode: parsed.mode || null,
|
|
565
|
+
result: parsed.result || null,
|
|
566
|
+
ambiguous: parsed.ambiguous === true,
|
|
567
|
+
error: parsed.error || null,
|
|
568
|
+
failureClass: parsed.result === 'FAILED_KNOWN' ? classifyKnownFailure(parsed.error || g.results[g.results.length - 1] || '') : null,
|
|
569
|
+
ledgerResults: g.results.slice(),
|
|
570
|
+
adjudicationPresent: adj !== null,
|
|
571
|
+
behaviorAligned: adj ? adj.behaviorAligned : null,
|
|
572
|
+
forbiddenBehaviorOccurred: adj ? adj.forbiddenBehaviorOccurred : null,
|
|
573
|
+
objectiveTaskOutcome: adj ? adj.objectiveTaskOutcome : null,
|
|
574
|
+
negativeTransfer: adj ? adj.negativeTransfer : null,
|
|
575
|
+
overCheckOccurred: adj && adj.overCheck ? adj.overCheck.overCheckOccurred === true : null,
|
|
576
|
+
unnecessaryCostUnits: adj && adj.overCheck ? Number(adj.overCheck.unnecessaryCostUnits) || 0 : null,
|
|
577
|
+
violationsCount: adj && Array.isArray(adj.violations) ? adj.violations.length : null,
|
|
578
|
+
traceSteps,
|
|
579
|
+
actorRounds: rounds.length,
|
|
580
|
+
usage: usage.observedRounds > 0 ? {
|
|
581
|
+
promptTokens: usage.prompt,
|
|
582
|
+
completionTokens: usage.completion,
|
|
583
|
+
totalTokens: usage.total,
|
|
584
|
+
observedRounds: usage.observedRounds,
|
|
585
|
+
totalRounds: rounds.length,
|
|
586
|
+
} : null,
|
|
587
|
+
timingObserved: rounds.some((r) => r && r.timing && typeof r.timing === 'object'),
|
|
588
|
+
});
|
|
589
|
+
}
|
|
590
|
+
executions.sort((a, b) => unitKey(a.unit).localeCompare(unitKey(b.unit)));
|
|
591
|
+
|
|
592
|
+
// --- plan-unit rollups (current state per the ledger's own authority) ----
|
|
593
|
+
const ledgerForVerify = createChallengeLedger(studyDir); // read-only use: classifyUnit only
|
|
594
|
+
const unitRollups = [];
|
|
595
|
+
const recoveredFromUnknown = [];
|
|
596
|
+
const ambiguousPrimary = [];
|
|
597
|
+
for (const pu of planUnits) {
|
|
598
|
+
const caseDef = caseById.get(pu.caseId);
|
|
599
|
+
const attemptIds = [...(seenUnitPlans.get(planUnitKey(pu.caseId, pu.armId, pu.repetitionId)) || [])].sort();
|
|
600
|
+
const states = attemptIds.map((attemptId) => ({
|
|
601
|
+
attemptId,
|
|
602
|
+
state: ledgerForVerify.classifyUnit({ caseId: pu.caseId, armId: pu.armId, repetitionId: pu.repetitionId, attemptId }).state,
|
|
603
|
+
}));
|
|
604
|
+
const succeeded = states.filter((s) => s.state === 'SUCCEEDED');
|
|
605
|
+
const failed = states.filter((s) => s.state === 'FAILED_KNOWN');
|
|
606
|
+
const unknown = states.filter((s) => s.state === 'RESULT_UNKNOWN');
|
|
607
|
+
let currentState = 'not_started';
|
|
608
|
+
if (succeeded.length > 0) currentState = 'succeeded';
|
|
609
|
+
else if (failed.length > 0) currentState = 'failed_known';
|
|
610
|
+
else if (unknown.length > 0) currentState = 'result_unknown';
|
|
611
|
+
else if (states.some((s) => s.state === 'RESERVED_NOT_PREPARED')) currentState = 'reserved_unsettled';
|
|
612
|
+
const primary = succeeded.length > 0 ? succeeded[succeeded.length - 1]
|
|
613
|
+
: failed.length > 0 ? failed[failed.length - 1]
|
|
614
|
+
: unknown.length > 0 ? unknown[unknown.length - 1] : null;
|
|
615
|
+
|
|
616
|
+
const succeededExecs = succeeded.map((s) => executions.find((e) => e.unit.caseId === pu.caseId && e.unit.armId === pu.armId && String(e.unit.repetitionId) === String(pu.repetitionId) && String(e.unit.attemptId) === String(s.attemptId)));
|
|
617
|
+
const divergent = succeededExecs.length > 1 && succeededExecs.some((e) => e.behaviorAligned !== succeededExecs[0].behaviorAligned
|
|
618
|
+
|| e.objectiveTaskOutcome !== succeededExecs[0].objectiveTaskOutcome);
|
|
619
|
+
const primaryExec = primary ? executions.find((e) => e.unit.caseId === pu.caseId && e.unit.armId === pu.armId && String(e.unit.repetitionId) === String(pu.repetitionId) && String(e.unit.attemptId) === String(primary.attemptId)) : null;
|
|
620
|
+
const validExecution = currentState === 'succeeded' && primaryExec !== undefined && primaryExec.adjudicationPresent && !divergent;
|
|
621
|
+
if (unknown.length > 0 && (succeeded.length > 0 || failed.length > 0)) {
|
|
622
|
+
recoveredFromUnknown.push({ unit: planUnitKey(pu.caseId, pu.armId, pu.repetitionId), attempts: states.map((s) => `${s.attemptId}:${s.state}`) });
|
|
623
|
+
}
|
|
624
|
+
if (divergent) {
|
|
625
|
+
ambiguousPrimary.push({ unit: planUnitKey(pu.caseId, pu.armId, pu.repetitionId), attempts: states.map((s) => `${s.attemptId}:${s.state}`) });
|
|
626
|
+
}
|
|
627
|
+
unitRollups.push({
|
|
628
|
+
unit: planUnitKey(pu.caseId, pu.armId, pu.repetitionId),
|
|
629
|
+
caseId: pu.caseId,
|
|
630
|
+
armId: pu.armId,
|
|
631
|
+
repetitionId: pu.repetitionId,
|
|
632
|
+
family: caseDef ? caseDef.family : null,
|
|
633
|
+
expectedMode: caseDef ? caseDef.expectedMode : null,
|
|
634
|
+
principleApplicable: caseDef ? caseDef.principleApplicable === true : null,
|
|
635
|
+
studyEligible: caseDef ? caseDef.studyEligible === true : null,
|
|
636
|
+
attempts: states.map((s) => `${s.attemptId}:${s.state}`).sort(),
|
|
637
|
+
currentState,
|
|
638
|
+
primaryAttempt: primary ? primary.attemptId : null,
|
|
639
|
+
primaryAnalysis: divergent ? 'AMBIGUOUS' : validExecution ? 'DETERMINABLE' : currentState === 'not_started' ? 'NOT_EXECUTED' : 'EXCLUDED',
|
|
640
|
+
validExecution,
|
|
641
|
+
recoveredFromUnknown: unknown.length > 0 && (succeeded.length > 0 || failed.length > 0),
|
|
642
|
+
outcome: primaryExec && primaryExec.adjudicationPresent ? {
|
|
643
|
+
behaviorAligned: primaryExec.behaviorAligned,
|
|
644
|
+
forbiddenBehaviorOccurred: primaryExec.forbiddenBehaviorOccurred,
|
|
645
|
+
objectiveTaskOutcome: primaryExec.objectiveTaskOutcome,
|
|
646
|
+
negativeTransfer: primaryExec.negativeTransfer,
|
|
647
|
+
} : null,
|
|
648
|
+
});
|
|
649
|
+
}
|
|
650
|
+
unitRollups.sort((a, b) => a.unit.localeCompare(b.unit));
|
|
651
|
+
recoveredFromUnknown.sort((a, b) => a.unit.localeCompare(b.unit));
|
|
652
|
+
ambiguousPrimary.sort((a, b) => a.unit.localeCompare(b.unit));
|
|
653
|
+
|
|
654
|
+
const byCategory = (cat) => unitRollups.filter((u) => u.currentState === cat).map((u) => u.unit);
|
|
655
|
+
const executedUnits = unitRollups.filter((u) => u.currentState !== 'not_started').length;
|
|
656
|
+
if (executedUnits === 0 && budget.entries.length === 0 && attemptsByUnitKey.size === 0) {
|
|
657
|
+
addFinding('study_not_executed', 'info', 'runs/', 'no executions observed — a report on an unexecuted study stays REPORT_INCONCLUSIVE');
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
// --- §9.4 dimensions ------------------------------------------------------
|
|
661
|
+
const arms = preregArms;
|
|
662
|
+
const primaryValid = (u) => u.validExecution && u.primaryAnalysis === 'DETERMINABLE';
|
|
663
|
+
const validUnitsByCaseRep = new Map(); // 'caseId/rRep' -> {armId -> rollup}
|
|
664
|
+
for (const u of unitRollups) {
|
|
665
|
+
if (!primaryValid(u)) continue;
|
|
666
|
+
const k = `${u.caseId}/r${String(u.repetitionId)}`;
|
|
667
|
+
if (!validUnitsByCaseRep.has(k)) validUnitsByCaseRep.set(k, new Map());
|
|
668
|
+
validUnitsByCaseRep.get(k).set(u.armId, u);
|
|
669
|
+
}
|
|
670
|
+
const pairedCases = [...validUnitsByCaseRep.entries()]
|
|
671
|
+
.filter(([, m]) => m.has('A') && m.has('B'))
|
|
672
|
+
.map(([k, m]) => ({ caseRep: k, arms: [...m.keys()].sort() }))
|
|
673
|
+
.sort((a, b) => a.caseRep.localeCompare(b.caseRep));
|
|
674
|
+
|
|
675
|
+
// protocol/evidence quality
|
|
676
|
+
const refusalReasons = {};
|
|
677
|
+
for (const e of events.entries) {
|
|
678
|
+
if (e && e.type === 'UNIT_REFUSED' && e.reason) refusalReasons[e.reason] = (refusalReasons[e.reason] || 0) + 1;
|
|
679
|
+
}
|
|
680
|
+
const realExecs = executions.filter((e) => e.mode && e.mode !== 'offline-stub');
|
|
681
|
+
const identityAttested = realExecs.filter((e) => {
|
|
682
|
+
const a = attemptsByUnitKey.get(unitKey(e.unit));
|
|
683
|
+
return a && Array.isArray(a.parsed.actorRounds) && a.parsed.actorRounds.some((r) => r && r.observedModelIdentity);
|
|
684
|
+
}).length;
|
|
685
|
+
const failureClasses = {
|
|
686
|
+
capacityFailure: executions.filter((e) => e.failureClass === 'capacity_failure').length,
|
|
687
|
+
identityFailure: executions.filter((e) => e.failureClass === 'identity_failure').length,
|
|
688
|
+
protocolContractFailure: executions.filter((e) => e.failureClass === 'protocol_contract_failure').length,
|
|
689
|
+
behaviorViolation: unitRollups.filter((u) => primaryValid(u) && u.outcome && u.outcome.forbiddenBehaviorOccurred === true).length,
|
|
690
|
+
objectiveTaskFailure: unitRollups.filter((u) => primaryValid(u) && u.outcome && u.outcome.objectiveTaskOutcome === 'FAIL').length,
|
|
691
|
+
};
|
|
692
|
+
const protocolEvidenceQuality = {
|
|
693
|
+
runCount: events.entries.filter((e) => e && e.type === 'RUN_STARTED').length,
|
|
694
|
+
assignedUnits: assignedDenominator,
|
|
695
|
+
executedUnits,
|
|
696
|
+
currentUnitStateCounts: {
|
|
697
|
+
succeeded: byCategory('succeeded').length,
|
|
698
|
+
failed_known: byCategory('failed_known').length,
|
|
699
|
+
result_unknown: byCategory('result_unknown').length,
|
|
700
|
+
reserved_unsettled: byCategory('reserved_unsettled').length,
|
|
701
|
+
not_started: byCategory('not_started').length,
|
|
702
|
+
},
|
|
703
|
+
validExecutionUnits: unitRollups.filter((u) => primaryValid(u)).length,
|
|
704
|
+
validPairedCasesAB: pairedCases.length,
|
|
705
|
+
identityObservability: realExecs.length > 0
|
|
706
|
+
? { attempts: realExecs.length, attemptsIdentityAttested: identityAttested }
|
|
707
|
+
: { attempts: 0, attemptsIdentityAttested: 0, note: 'offline-stub study — model identity not applicable' },
|
|
708
|
+
interruptedUnits: events.entries.filter((e) => e && e.type === 'UNIT_INTERRUPTED').length,
|
|
709
|
+
refusalReasons: Object.fromEntries(Object.keys(refusalReasons).sort().map((k) => [k, refusalReasons[k]])),
|
|
710
|
+
failureClasses,
|
|
711
|
+
};
|
|
712
|
+
|
|
713
|
+
// behavioral benefit — prereg applicable-positive cases, A/B paired only
|
|
714
|
+
const applicablePositive = unitRollups.filter((u) => u.principleApplicable === true);
|
|
715
|
+
const benefitPairs = [];
|
|
716
|
+
for (const { caseRep } of pairedCases) {
|
|
717
|
+
const m = validUnitsByCaseRep.get(caseRep);
|
|
718
|
+
const a = m.get('A');
|
|
719
|
+
const b = m.get('B');
|
|
720
|
+
if (!a.principleApplicable) continue;
|
|
721
|
+
benefitPairs.push({
|
|
722
|
+
caseRep,
|
|
723
|
+
a: { objectiveTaskOutcome: a.outcome.objectiveTaskOutcome, behaviorAligned: a.outcome.behaviorAligned },
|
|
724
|
+
b: { objectiveTaskOutcome: b.outcome.objectiveTaskOutcome, behaviorAligned: b.outcome.behaviorAligned },
|
|
725
|
+
});
|
|
726
|
+
}
|
|
727
|
+
const passCount = (side) => benefitPairs.filter((p) => p[side].objectiveTaskOutcome === 'PASS').length;
|
|
728
|
+
const alignedCount = (side) => benefitPairs.filter((p) => p[side].behaviorAligned === true).length;
|
|
729
|
+
const behavioralBenefit = {
|
|
730
|
+
scope: 'preregistered applicable-positive cases (principleApplicable=true) with a valid A/B pair',
|
|
731
|
+
pairedCases: benefitPairs.length,
|
|
732
|
+
armA: { objectiveTaskPass: passCount('a'), behaviorAligned: alignedCount('a') },
|
|
733
|
+
armB: { objectiveTaskPass: passCount('b'), behaviorAligned: alignedCount('b') },
|
|
734
|
+
pairedDifferenceBminusA: {
|
|
735
|
+
objectiveTaskPass: passCount('b') - passCount('a'),
|
|
736
|
+
behaviorAligned: alignedCount('b') - alignedCount('a'),
|
|
737
|
+
},
|
|
738
|
+
pairs: benefitPairs,
|
|
739
|
+
note: 'descriptive paired counts only — no causal claim (SPEC §1.2/§9.4)',
|
|
740
|
+
};
|
|
741
|
+
|
|
742
|
+
// boundary / negative transfer — scorer verdicts verbatim, boundaryRisk separate
|
|
743
|
+
const boundaryUnits = unitRollups.filter((u) => u.principleApplicable === false && u.studyEligible === true);
|
|
744
|
+
const primaryExecOf = (u) => executions.find((e) => sameExec(e, u)) || null;
|
|
745
|
+
const boundaryByArm = {};
|
|
746
|
+
for (const armId of arms) {
|
|
747
|
+
const us = boundaryUnits.filter((u) => u.armId === armId && primaryValid(u));
|
|
748
|
+
boundaryByArm[armId] = {
|
|
749
|
+
unitsExecuted: us.length,
|
|
750
|
+
boundaryRiskForbiddenBehavior: us.filter((u) => u.outcome.forbiddenBehaviorOccurred === true).length,
|
|
751
|
+
negativeTransferScorerVerdict: us.filter((u) => u.outcome.negativeTransfer === true).length,
|
|
752
|
+
overCheckOccurred: us.filter((u) => { const e = primaryExecOf(u); return e !== null && e.overCheckOccurred === true; }).length,
|
|
753
|
+
unnecessaryCostUnits: us.reduce((acc, u) => acc + (primaryExecOf(u) ? primaryExecOf(u).unnecessaryCostUnits || 0 : 0), 0),
|
|
754
|
+
};
|
|
755
|
+
}
|
|
756
|
+
const boundaryNegativeTransfer = {
|
|
757
|
+
scope: 'boundary cases (principleApplicable=false ∧ studyEligible=true, SPEC §5.2/§9.1) — retained, never dropped',
|
|
758
|
+
boundaryCasesInSuite: new Set(boundaryUnits.map((u) => u.caseId)).size,
|
|
759
|
+
perArm: boundaryByArm,
|
|
760
|
+
semanticSources: {
|
|
761
|
+
negativeTransfer: 'adjudication.negativeTransfer verbatim (claim-evidence-scorer: overCheckOccurred ∧ expectedMode=withhold)',
|
|
762
|
+
boundaryRisk: 'forbiddenBehaviorOccurred on boundary cases — reported separately, NOT merged into negativeTransfer',
|
|
763
|
+
},
|
|
764
|
+
};
|
|
765
|
+
|
|
766
|
+
// objective task outcome — separate from behavior alignment
|
|
767
|
+
const objectiveTaskOutcome = {};
|
|
768
|
+
for (const armId of arms) {
|
|
769
|
+
const prim = unitRollups.filter((u) => u.armId === armId && primaryValid(u));
|
|
770
|
+
const appl = prim.filter((u) => u.principleApplicable === true);
|
|
771
|
+
const bnd = prim.filter((u) => u.principleApplicable === false);
|
|
772
|
+
objectiveTaskOutcome[armId] = {
|
|
773
|
+
units: prim.length,
|
|
774
|
+
pass: prim.filter((u) => u.outcome.objectiveTaskOutcome === 'PASS').length,
|
|
775
|
+
fail: prim.filter((u) => u.outcome.objectiveTaskOutcome === 'FAIL').length,
|
|
776
|
+
applicablePositive: { units: appl.length, pass: appl.filter((u) => u.outcome.objectiveTaskOutcome === 'PASS').length },
|
|
777
|
+
boundary: { units: bnd.length, pass: bnd.filter((u) => u.outcome.objectiveTaskOutcome === 'PASS').length },
|
|
778
|
+
};
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
// cost — observable only; unobservable is explicitly unavailable, never zero
|
|
782
|
+
const cost = {};
|
|
783
|
+
for (const armId of arms) {
|
|
784
|
+
const armExecs = executions.filter((e) => e.unit.armId === armId);
|
|
785
|
+
const tokens = armExecs.filter((e) => e.usage);
|
|
786
|
+
const toolSteps = armExecs.filter((e) => e.traceSteps !== null);
|
|
787
|
+
cost[armId] = {
|
|
788
|
+
attemptsReserved: budget.entries.filter((r) => r && r.type === 'RESERVE' && r.unit && r.unit.armId === armId).length,
|
|
789
|
+
requestsReserved: budget.entries.filter((r) => r && r.type === 'REQUEST_RESERVE' && r.unit && r.unit.armId === armId).length,
|
|
790
|
+
requestRefusals: budget.entries.filter((r) => r && r.type === 'REQUEST_REFUSED' && r.unit && r.unit.armId === armId).length,
|
|
791
|
+
budgetRefusals: budget.entries.filter((r) => r && r.type === 'BUDGET_REFUSED' && r.unit && r.unit.armId === armId).length,
|
|
792
|
+
tokens: tokens.length > 0 ? {
|
|
793
|
+
prompt: tokens.reduce((a, e) => a + e.usage.promptTokens, 0),
|
|
794
|
+
completion: tokens.reduce((a, e) => a + e.usage.completionTokens, 0),
|
|
795
|
+
total: tokens.reduce((a, e) => a + e.usage.totalTokens, 0),
|
|
796
|
+
attemptsWithUsage: tokens.length,
|
|
797
|
+
} : { available: false, reason: 'no actorRounds usage observed for this arm' },
|
|
798
|
+
toolSteps: toolSteps.length > 0 ? {
|
|
799
|
+
steps: toolSteps.reduce((a, e) => a + e.traceSteps, 0),
|
|
800
|
+
attemptsWithTrace: toolSteps.length,
|
|
801
|
+
} : { available: false, reason: 'no trace records observed for this arm' },
|
|
802
|
+
timing: armExecs.some((e) => e.timingObserved)
|
|
803
|
+
? { available: true, attemptsWithTiming: armExecs.filter((e) => e.timingObserved).length }
|
|
804
|
+
: { available: false, reason: 'no per-request timing telemetry observed (TTFB telemetry PR #2153 fields absent)' },
|
|
805
|
+
monetaryCost: { available: false, reason: 'no per-token price bound to this study copy — tokens are reported raw' },
|
|
806
|
+
};
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
// coverage
|
|
810
|
+
const families = manifest && manifest.familyCounts ? Object.keys(manifest.familyCounts).sort() : [];
|
|
811
|
+
const coverage = {
|
|
812
|
+
families: families.map((family) => {
|
|
813
|
+
const familyCases = [...new Set(unitRollups.filter((u) => u.family === family).map((u) => u.caseId))].sort();
|
|
814
|
+
return {
|
|
815
|
+
family,
|
|
816
|
+
casesInSuite: familyCases.length,
|
|
817
|
+
casesWithValidExecution: familyCases.filter((c) => unitRollups.some((u) => u.caseId === c && primaryValid(u))).length,
|
|
818
|
+
validABPairs: pairedCases.filter(({ caseRep }) => caseRep.split('/r')[0] && unitRollups.some((u) => u.caseId === caseRep.split('/r')[0] && u.family === family)).length,
|
|
819
|
+
};
|
|
820
|
+
}),
|
|
821
|
+
independentDecisionVariables: { estimable: false, value: 'NOT_ESTIMABLE', reason: 'the frozen suite carries no decision-variable metadata bound per case — counting from hashes would be inference, not evidence (SPEC §9.4 覆盖)' },
|
|
822
|
+
transferEstimability: { estimable: false, value: 'NOT_ESTIMABLE', reason: 'requires ≥2 verifiably distinct decision contexts; no such metadata is frozen with the suite (SPEC §5.1)' },
|
|
823
|
+
};
|
|
824
|
+
|
|
825
|
+
// --- status ----------------------------------------------------------------
|
|
826
|
+
const criticalCount = findings.filter((f) => f.severity === 'critical').length;
|
|
827
|
+
const ready = criticalCount === 0
|
|
828
|
+
&& bindingsVerified
|
|
829
|
+
&& executedUnits > 0
|
|
830
|
+
&& (realModeAttempts === 0 || authorizationValid)
|
|
831
|
+
&& !(preregArms.length === 0 || plannedRepetitions.length === 0 || cases.length === 0);
|
|
832
|
+
findings.sort((a, b) => (SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity]) || a.id.localeCompare(b.id) || String(a.subject).localeCompare(String(b.subject)));
|
|
833
|
+
|
|
834
|
+
const report = {
|
|
835
|
+
schema: BEHAVIOR_REPORT_SCHEMA,
|
|
836
|
+
studyId: (manifest && manifest.studyId) || (prereg && prereg.studyId) || null,
|
|
837
|
+
status: ready ? REPORT_STATUS_READY : REPORT_STATUS_INCONCLUSIVE,
|
|
838
|
+
generatedAt: normalizeNow(now),
|
|
839
|
+
scope: {
|
|
840
|
+
preregistered: {
|
|
841
|
+
arms,
|
|
842
|
+
caseCount: cases.length,
|
|
843
|
+
repetitions: plannedRepetitions,
|
|
844
|
+
assignedDenominator,
|
|
845
|
+
rule: 'preregistration.json arms × frozen suite-manifest cases × planned repetitions — the authorization subset never shrinks this denominator',
|
|
846
|
+
},
|
|
847
|
+
authorized: authRecord ? {
|
|
848
|
+
status: authStatus,
|
|
849
|
+
validForRealExecution: authorizationValid,
|
|
850
|
+
bindingsMatch: authBindingsMatch,
|
|
851
|
+
executionMode: authRecord.executionMode || null,
|
|
852
|
+
requestedActorIdentity: authRecord.requestedActorIdentity || null,
|
|
853
|
+
arms: Array.isArray(authRecord.arms) ? authRecord.arms.slice().sort() : [],
|
|
854
|
+
caseCount: Array.isArray(authRecord.cases) ? authRecord.cases.length : null,
|
|
855
|
+
repetitions: Array.isArray(authRecord.repetitions) ? authRecord.repetitions.map(String).sort() : [],
|
|
856
|
+
attemptCap: authRecord.attemptCap ?? null,
|
|
857
|
+
requestCap: authRecord.requestCap ?? null,
|
|
858
|
+
validFrom: authRecord.validFrom || null,
|
|
859
|
+
validUntil: authRecord.validUntil || null,
|
|
860
|
+
revoked: authRecord.revoked === true,
|
|
861
|
+
halted: authRecord.halted === true,
|
|
862
|
+
note: authStatus === 'APPROVED' ? 'authorized SUBSET of the preregistered plan (may cover only part of the denominator)' : 'fail-closed placeholder or non-approved record — not a real-execution authorization',
|
|
863
|
+
} : { present: false, note: 'no run-authorization.json in this study copy' },
|
|
864
|
+
executedUnits,
|
|
865
|
+
settlementCounts: {
|
|
866
|
+
succeeded: byCategory('succeeded').length,
|
|
867
|
+
failedKnown: byCategory('failed_known').length,
|
|
868
|
+
resultUnknown: byCategory('result_unknown').length,
|
|
869
|
+
reservedUnsettled: byCategory('reserved_unsettled').length,
|
|
870
|
+
notStarted: byCategory('not_started').length,
|
|
871
|
+
},
|
|
872
|
+
notStartedUnits: byCategory('not_started').sort(),
|
|
873
|
+
lostToFollowUpUnits: [...byCategory('result_unknown'), ...byCategory('reserved_unsettled')].sort(),
|
|
874
|
+
lostToFollowUpRule: 'RESULT_UNKNOWN settlements and reserved-but-unsettled units — retained in the denominator, excluded from valid pairs (SPEC §9.3/§9.5), never auto-counted as behavioral failure',
|
|
875
|
+
outOfScopeExecutions: outOfScope,
|
|
876
|
+
},
|
|
877
|
+
bindings: {
|
|
878
|
+
verified: bindingsVerified,
|
|
879
|
+
frozenStudyVerification: 'challenge-runner verifyFrozenStudy (suiteHash/mappingHash/graderHash/principleContentHash + per-case inputHash/goldHash recomputation)',
|
|
880
|
+
preregistrationHash: hashes && hashes.preregistrationHash ? hashes.preregistrationHash.value : null,
|
|
881
|
+
suiteHash: manifestHashes ? manifestHashes.suiteHash : null,
|
|
882
|
+
mappingHash: manifestHashes ? manifestHashes.mappingHash : null,
|
|
883
|
+
graderHash: manifestHashes ? manifestHashes.graderHash : null,
|
|
884
|
+
executionModesObserved: [...seenModes].sort(),
|
|
885
|
+
},
|
|
886
|
+
dimensions: {
|
|
887
|
+
protocolEvidenceQuality,
|
|
888
|
+
behavioralBenefit,
|
|
889
|
+
boundaryNegativeTransfer,
|
|
890
|
+
objectiveTaskOutcome,
|
|
891
|
+
cost,
|
|
892
|
+
coverage,
|
|
893
|
+
},
|
|
894
|
+
analysisRules: {
|
|
895
|
+
primaryAttempt: "per-attempt state from the ledger's classifyUnit (last ATTEMPT_RESULT wins per attemptId); unit current state prefers succeeded > failed_known > result_unknown — provenance: challenge-ledger.mjs, the existing recovery authority",
|
|
896
|
+
ambiguousPrimary: 'multiple SUCCEEDED attempts with diverging adjudication → AMBIGUOUS, excluded from the paired main analysis, kept in sensitivity',
|
|
897
|
+
negativeTransfer: 'adjudication.negativeTransfer verbatim — never redefined at the report layer',
|
|
898
|
+
boundaryRisk: 'forbiddenBehaviorOccurred on boundary cases — a separate indicator, never merged into negativeTransfer',
|
|
899
|
+
},
|
|
900
|
+
planUnits: unitRollups,
|
|
901
|
+
executions,
|
|
902
|
+
sensitivity: {
|
|
903
|
+
recoveredFromUnknown,
|
|
904
|
+
ambiguousPrimary,
|
|
905
|
+
note: 'units recovered via a NEW attemptId after RESULT_UNKNOWN, and units whose multiple SUCCEEDED attempts diverge — full records retained above',
|
|
906
|
+
},
|
|
907
|
+
integrity: {
|
|
908
|
+
findings,
|
|
909
|
+
criticalCount,
|
|
910
|
+
majorCount: findings.filter((f) => f.severity === 'major').length,
|
|
911
|
+
infoCount: findings.filter((f) => f.severity === 'info').length,
|
|
912
|
+
},
|
|
913
|
+
limitations: [
|
|
914
|
+
'descriptive aggregation of raw evidence only — no causal inference (SPEC §1.2)',
|
|
915
|
+
'unknown/failed settlements are honest outcomes, not corruption; coverage is limited by what actually settled',
|
|
916
|
+
'monetary cost unobservable without a bound per-token price; tokens are reported raw',
|
|
917
|
+
],
|
|
918
|
+
};
|
|
919
|
+
return report;
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
// small helper used by the boundary dimension (primary execution lookup)
|
|
923
|
+
function sameExec(e, u) {
|
|
924
|
+
return e.unit.caseId === u.caseId && e.unit.armId === u.armId && String(e.unit.repetitionId) === String(u.repetitionId) && e.unit.attemptId === u.primaryAttempt;
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
// ---------------------------------------------------------------------------
|
|
928
|
+
// deterministic rendering
|
|
929
|
+
// ---------------------------------------------------------------------------
|
|
930
|
+
|
|
931
|
+
export function renderBehaviorReportJson(report) {
|
|
932
|
+
return `${JSON.stringify(report, null, 2)}\n`;
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
export function renderBehaviorReportMarkdown(report) {
|
|
936
|
+
const lines = [];
|
|
937
|
+
const push = (...ls) => lines.push(...ls);
|
|
938
|
+
push(`# Behavior Report — ${report.studyId ?? '(studyId unknown)'}`);
|
|
939
|
+
push('');
|
|
940
|
+
push(`- schema: \`${report.schema}\``);
|
|
941
|
+
push(`- status: **${report.status}**`);
|
|
942
|
+
push(`- generatedAt: ${report.generatedAt ?? '(not provided)'}`);
|
|
943
|
+
push(`- assignedDenominator: ${report.scope.preregistered.assignedDenominator}`);
|
|
944
|
+
push(`- executedUnits: ${report.scope.executedUnits}`);
|
|
945
|
+
push('');
|
|
946
|
+
push('## Scope');
|
|
947
|
+
push('');
|
|
948
|
+
push('| scope | value |');
|
|
949
|
+
push('|---|---|');
|
|
950
|
+
push(`| preregistered | arms ${report.scope.preregistered.arms.join('/')} × ${report.scope.preregistered.caseCount} cases × ${report.scope.preregistered.repetitions.length} rep(s) = ${report.scope.preregistered.assignedDenominator} |`);
|
|
951
|
+
const a = report.scope.authorized;
|
|
952
|
+
push(`| authorized | ${a.present === false ? 'absent' : `${a.status}${a.validForRealExecution ? ' (valid, bindings match)' : a.status === 'APPROVED' ? ' (bindings/revocation problem)' : ' (not a real-execution authorization)'}`} |`);
|
|
953
|
+
const sc = report.scope.settlementCounts;
|
|
954
|
+
push(`| settlements | succeeded ${sc.succeeded} / failedKnown ${sc.failedKnown} / resultUnknown ${sc.resultUnknown} / reservedUnsettled ${sc.reservedUnsettled} / notStarted ${sc.notStarted} |`);
|
|
955
|
+
push(`| lost to follow-up | ${report.scope.lostToFollowUpUnits.length} unit(s) — retained, excluded from valid pairs |`);
|
|
956
|
+
push(`| out-of-scope executions | ${report.scope.outOfScopeExecutions.length} |`);
|
|
957
|
+
push('');
|
|
958
|
+
push('## Integrity findings');
|
|
959
|
+
push('');
|
|
960
|
+
if (report.integrity.findings.length === 0) {
|
|
961
|
+
push('(none)');
|
|
962
|
+
} else {
|
|
963
|
+
push('| severity | id | subject | detail |');
|
|
964
|
+
push('|---|---|---|---|');
|
|
965
|
+
for (const f of report.integrity.findings) push(`| ${f.severity} | ${f.id} | ${String(f.subject).replaceAll('|', '/')} | ${String(f.detail).replaceAll('|', '/').slice(0, 160)} |`);
|
|
966
|
+
}
|
|
967
|
+
push('');
|
|
968
|
+
const d = report.dimensions;
|
|
969
|
+
push('## 1. 协议/证据质量 (protocolEvidenceQuality)');
|
|
970
|
+
push('');
|
|
971
|
+
push(`- runs: ${d.protocolEvidenceQuality.runCount}; validExecutionUnits: ${d.protocolEvidenceQuality.validExecutionUnits}; validPairedCasesAB: ${d.protocolEvidenceQuality.validPairedCasesAB}`);
|
|
972
|
+
push(`- identity attested: ${d.protocolEvidenceQuality.identityObservability.attemptsIdentityAttested}/${d.protocolEvidenceQuality.identityObservability.attempts}`);
|
|
973
|
+
push(`- failureClasses: capacity=${d.protocolEvidenceQuality.failureClasses.capacityFailure}, identity=${d.protocolEvidenceQuality.failureClasses.identityFailure}, protocolContract=${d.protocolEvidenceQuality.failureClasses.protocolContractFailure}, behaviorViolation=${d.protocolEvidenceQuality.failureClasses.behaviorViolation}, objectiveTaskFailure=${d.protocolEvidenceQuality.failureClasses.objectiveTaskFailure}`);
|
|
974
|
+
push(`- refusalReasons: ${JSON.stringify(d.protocolEvidenceQuality.refusalReasons)}`);
|
|
975
|
+
push('');
|
|
976
|
+
push('## 2. 行为收益 (behavioralBenefit — applicable-positive A/B pairs only)');
|
|
977
|
+
push('');
|
|
978
|
+
const bb = d.behavioralBenefit;
|
|
979
|
+
push(`- pairedCases: ${bb.pairedCases}; A pass ${bb.armA.objectiveTaskPass} / aligned ${bb.armA.behaviorAligned}; B pass ${bb.armB.objectiveTaskPass} / aligned ${bb.armB.behaviorAligned}; B−A pass ${bb.pairedDifferenceBminusA.objectiveTaskPass}`);
|
|
980
|
+
push(`- ${bb.note}`);
|
|
981
|
+
push('');
|
|
982
|
+
push('## 3. 边界/负迁移 (boundaryNegativeTransfer — scorer verdicts verbatim)');
|
|
983
|
+
push('');
|
|
984
|
+
push('| arm | units | boundaryRisk (forbiddenBehavior) | negativeTransfer (scorer) |');
|
|
985
|
+
push('|---|---|---|---|');
|
|
986
|
+
for (const armId of Object.keys(d.boundaryNegativeTransfer.perArm)) {
|
|
987
|
+
const b = d.boundaryNegativeTransfer.perArm[armId];
|
|
988
|
+
push(`| ${armId} | ${b.unitsExecuted} | ${b.boundaryRiskForbiddenBehavior} | ${b.negativeTransferScorerVerdict} |`);
|
|
989
|
+
}
|
|
990
|
+
push('');
|
|
991
|
+
push('## 4. 客观任务结果 (objectiveTaskOutcome — separate from behavior)');
|
|
992
|
+
push('');
|
|
993
|
+
push('| arm | units | PASS | FAIL | applicable pass | boundary pass |');
|
|
994
|
+
push('|---|---|---|---|---|---|');
|
|
995
|
+
for (const armId of Object.keys(d.objectiveTaskOutcome)) {
|
|
996
|
+
const o = d.objectiveTaskOutcome[armId];
|
|
997
|
+
push(`| ${armId} | ${o.units} | ${o.pass} | ${o.fail} | ${o.applicablePositive.pass}/${o.applicablePositive.units} | ${o.boundary.pass}/${o.boundary.units} |`);
|
|
998
|
+
}
|
|
999
|
+
push('');
|
|
1000
|
+
push('## 5. 成本 (cost — observable only, unavailable never zero-filled)');
|
|
1001
|
+
push('');
|
|
1002
|
+
push('| arm | attempts | requests | tokens (total) | toolSteps | timing |');
|
|
1003
|
+
push('|---|---|---|---|---|---|');
|
|
1004
|
+
for (const armId of Object.keys(d.cost)) {
|
|
1005
|
+
const c = d.cost[armId];
|
|
1006
|
+
push(`| ${armId} | ${c.attemptsReserved} | ${c.requestsReserved} | ${c.tokens.available === false ? 'unavailable' : c.tokens.total} | ${c.toolSteps.available === false ? 'unavailable' : c.toolSteps.steps} | ${c.timing.available ? 'observed' : 'unavailable'} |`);
|
|
1007
|
+
}
|
|
1008
|
+
push('');
|
|
1009
|
+
push('## 6. 覆盖 (coverage)');
|
|
1010
|
+
push('');
|
|
1011
|
+
push('| family | cases in suite | cases with valid execution | valid A/B pairs |');
|
|
1012
|
+
push('|---|---|---|---|');
|
|
1013
|
+
for (const f of d.coverage.families) push(`| ${f.family} | ${f.casesInSuite} | ${f.casesWithValidExecution} | ${f.validABPairs} |`);
|
|
1014
|
+
push('');
|
|
1015
|
+
push(`- independentDecisionVariables: ${d.coverage.independentDecisionVariables.value}`);
|
|
1016
|
+
push(`- transferEstimability: ${d.coverage.transferEstimability.value}`);
|
|
1017
|
+
push('');
|
|
1018
|
+
push('## Sensitivity');
|
|
1019
|
+
push('');
|
|
1020
|
+
push(`- recoveredFromUnknown: ${report.sensitivity.recoveredFromUnknown.length} unit(s)`);
|
|
1021
|
+
push(`- ambiguousPrimary: ${report.sensitivity.ambiguousPrimary.length} unit(s)`);
|
|
1022
|
+
push('');
|
|
1023
|
+
push('## Limitations');
|
|
1024
|
+
push('');
|
|
1025
|
+
for (const l of report.limitations) push(`- ${l}`);
|
|
1026
|
+
push('');
|
|
1027
|
+
return lines.join('\n');
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
// ---------------------------------------------------------------------------
|
|
1031
|
+
// write path (locked, atomic, revision-protected) + read-only verify path
|
|
1032
|
+
// ---------------------------------------------------------------------------
|
|
1033
|
+
|
|
1034
|
+
function assertNotFrozenStudyArea(studyDir) {
|
|
1035
|
+
const resolved = fs.existsSync(studyDir) ? fs.realpathSync(studyDir) : path.resolve(studyDir);
|
|
1036
|
+
const frozen = process.platform === 'win32' ? FROZEN_STUDIES_AREA.toLowerCase() : FROZEN_STUDIES_AREA;
|
|
1037
|
+
const target = process.platform === 'win32' ? resolved.toLowerCase() : resolved;
|
|
1038
|
+
if (target === frozen || target.startsWith(`${frozen}${path.sep}`)) {
|
|
1039
|
+
throw new FrozenStudyWriteRefusedError(resolved, FROZEN_STUDIES_AREA);
|
|
1040
|
+
}
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
function atomicWrite(file, content) {
|
|
1044
|
+
const tmp = `${file}.tmp`;
|
|
1045
|
+
fs.writeFileSync(tmp, content, 'utf8');
|
|
1046
|
+
fs.renameSync(tmp, file);
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
export function writeBehaviorReport(studyDir, { now, owner = 'behavior-report-generator' } = {}) {
|
|
1050
|
+
if (now === undefined || now === null) {
|
|
1051
|
+
throw new Error('writeBehaviorReport: now required (ISO-8601) — the report timestamp must be explicit and reproducible');
|
|
1052
|
+
}
|
|
1053
|
+
normalizeNow(now); // validate
|
|
1054
|
+
assertNotFrozenStudyArea(studyDir);
|
|
1055
|
+
const jsonPath = path.join(studyDir, REPORT_JSON);
|
|
1056
|
+
const mdPath = path.join(studyDir, REPORT_MD);
|
|
1057
|
+
acquireStudyWriteLock(studyDir, { owner }); // StudyLockError propagates: concurrent/interrupted writers refuse here
|
|
1058
|
+
try {
|
|
1059
|
+
const events = readJsonl(path.join(studyDir, 'events.jsonl')).entries;
|
|
1060
|
+
const readyEvents = events.filter((e) => e && e.type === 'REPORT_READY');
|
|
1061
|
+
if (readyEvents.length > 0) {
|
|
1062
|
+
if (!fs.existsSync(jsonPath)) {
|
|
1063
|
+
throw new ReportRevisionRequiredError('REPORT_READY event exists but behavior-report.json is missing — the committed report cannot be compared');
|
|
1064
|
+
}
|
|
1065
|
+
let existingRaw;
|
|
1066
|
+
let existing;
|
|
1067
|
+
try {
|
|
1068
|
+
existingRaw = fs.readFileSync(jsonPath, 'utf8');
|
|
1069
|
+
existing = JSON.parse(existingRaw);
|
|
1070
|
+
} catch (e) {
|
|
1071
|
+
throw new ReportRevisionRequiredError(`committed behavior-report.json unparseable: ${e.message}`);
|
|
1072
|
+
}
|
|
1073
|
+
const compareNow = existing.generatedAt ?? now;
|
|
1074
|
+
const candidate = buildBehaviorReport(studyDir, { now: compareNow });
|
|
1075
|
+
const candidateJson = renderBehaviorReportJson(candidate);
|
|
1076
|
+
if (existingRaw !== candidateJson) {
|
|
1077
|
+
throw new ReportRevisionRequiredError('recomputation differs from the committed report (data changed after REPORT_READY, or the report was tampered with)');
|
|
1078
|
+
}
|
|
1079
|
+
const md = renderBehaviorReportMarkdown(candidate);
|
|
1080
|
+
if (!fs.existsSync(mdPath) || fs.readFileSync(mdPath, 'utf8') !== md) {
|
|
1081
|
+
atomicWrite(mdPath, md); // rendering completion of the SAME report — data revision not involved
|
|
1082
|
+
}
|
|
1083
|
+
return { written: false, alreadyReady: true, status: candidate.status, report: candidate };
|
|
1084
|
+
}
|
|
1085
|
+
const report = buildBehaviorReport(studyDir, { now });
|
|
1086
|
+
const json = renderBehaviorReportJson(report);
|
|
1087
|
+
const md = renderBehaviorReportMarkdown(report);
|
|
1088
|
+
atomicWrite(jsonPath, json);
|
|
1089
|
+
atomicWrite(mdPath, md);
|
|
1090
|
+
let eventAppended = false;
|
|
1091
|
+
if (report.status === REPORT_STATUS_READY) {
|
|
1092
|
+
const ledger = createChallengeLedger(studyDir);
|
|
1093
|
+
ledger.appendEvent({
|
|
1094
|
+
type: 'REPORT_READY',
|
|
1095
|
+
owner,
|
|
1096
|
+
report: {
|
|
1097
|
+
schema: report.schema,
|
|
1098
|
+
status: report.status,
|
|
1099
|
+
generatedAt: report.generatedAt,
|
|
1100
|
+
reportSha256: sha256(json),
|
|
1101
|
+
assignedDenominator: report.scope.preregistered.assignedDenominator,
|
|
1102
|
+
executedUnits: report.scope.executedUnits,
|
|
1103
|
+
criticalFindings: report.integrity.criticalCount,
|
|
1104
|
+
},
|
|
1105
|
+
});
|
|
1106
|
+
eventAppended = true;
|
|
1107
|
+
}
|
|
1108
|
+
return { written: true, alreadyReady: false, status: report.status, eventAppended, report };
|
|
1109
|
+
} finally {
|
|
1110
|
+
// Swallow release failures: if we held the lock and unlink failed the lock
|
|
1111
|
+
// simply stays held and the next writer is refused loudly (the ledger's
|
|
1112
|
+
// own crash semantic) — rethrowing here could mask the original error.
|
|
1113
|
+
try {
|
|
1114
|
+
releaseStudyWriteLock(studyDir, { owner });
|
|
1115
|
+
} catch { /* already released, gone, or unreadable */ }
|
|
1116
|
+
}
|
|
1117
|
+
}
|
|
1118
|
+
|
|
1119
|
+
export function verifyBehaviorReport(studyDir, { now } = {}) {
|
|
1120
|
+
// STRICTLY read-only: no lock, no writes, no event appends.
|
|
1121
|
+
const jsonPath = path.join(studyDir, REPORT_JSON);
|
|
1122
|
+
const mdPath = path.join(studyDir, REPORT_MD);
|
|
1123
|
+
const checks = [];
|
|
1124
|
+
const check = (id, ok, detail) => checks.push({ id, ok, detail });
|
|
1125
|
+
if (!fs.existsSync(jsonPath)) {
|
|
1126
|
+
return { ok: false, reason: 'report_not_generated', checks: [{ id: 'report_json_present', ok: false, detail: 'behavior-report.json not found' }] };
|
|
1127
|
+
}
|
|
1128
|
+
const existingRaw = fs.readFileSync(jsonPath, 'utf8');
|
|
1129
|
+
let existing = null;
|
|
1130
|
+
try {
|
|
1131
|
+
existing = JSON.parse(existingRaw);
|
|
1132
|
+
} catch (e) {
|
|
1133
|
+
return { ok: false, reason: 'report_unparseable', checks: [{ id: 'report_json_parseable', ok: false, detail: e.message }] };
|
|
1134
|
+
}
|
|
1135
|
+
check('report_json_parseable', true, '');
|
|
1136
|
+
check('report_md_present', fs.existsSync(mdPath), 'behavior-report.md must exist as the paired rendering');
|
|
1137
|
+
const effectiveNow = existing.generatedAt ?? (now !== undefined && now !== null ? now : null);
|
|
1138
|
+
if (effectiveNow === null) {
|
|
1139
|
+
check('generated_at_available', false, 'existing report has no generatedAt and no --now given — recomputation is not reproducible');
|
|
1140
|
+
return { ok: false, reason: 'generated_at_missing', checks };
|
|
1141
|
+
}
|
|
1142
|
+
const rebuilt = buildBehaviorReport(studyDir, { now: effectiveNow });
|
|
1143
|
+
const rebuiltJson = renderBehaviorReportJson(rebuilt);
|
|
1144
|
+
const jsonMatch = existingRaw === rebuiltJson;
|
|
1145
|
+
check('report_json_recomputable', jsonMatch, jsonMatch ? 'byte-identical recomputation from raw evidence' : 'recomputation differs from the stored report');
|
|
1146
|
+
let mdMatch = false;
|
|
1147
|
+
if (fs.existsSync(mdPath)) {
|
|
1148
|
+
mdMatch = fs.readFileSync(mdPath, 'utf8') === renderBehaviorReportMarkdown(rebuilt);
|
|
1149
|
+
check('report_md_recomputable', mdMatch, mdMatch ? 'byte-identical markdown rendering' : 'markdown rendering differs');
|
|
1150
|
+
}
|
|
1151
|
+
const readyEvents = readJsonl(path.join(studyDir, 'events.jsonl')).entries.filter((e) => e && e.type === 'REPORT_READY');
|
|
1152
|
+
if (existing.status === REPORT_STATUS_READY) {
|
|
1153
|
+
check('ready_event_count', readyEvents.length === 1, `expected exactly one REPORT_READY event, found ${readyEvents.length}`);
|
|
1154
|
+
if (readyEvents.length >= 1) {
|
|
1155
|
+
const bound = readyEvents[0].report && readyEvents[0].report.reportSha256;
|
|
1156
|
+
check('ready_event_sha_binding', bound === sha256(existingRaw), bound === sha256(existingRaw) ? 'event binds the exact report bytes' : 'REPORT_READY event sha256 does not match the stored report');
|
|
1157
|
+
}
|
|
1158
|
+
} else {
|
|
1159
|
+
check('no_ready_event_when_not_ready', readyEvents.length === 0, `REPORT_INCONCLUSIVE report must not carry a REPORT_READY event, found ${readyEvents.length}`);
|
|
1160
|
+
}
|
|
1161
|
+
const ok = checks.every((c) => c.ok);
|
|
1162
|
+
return { ok, reason: ok ? null : 'verification_failed', status: existing.status, checks };
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
// ---------------------------------------------------------------------------
|
|
1166
|
+
// v0.7 §7 — Report V2 (nine dimensions, full-trajectory replay scoring,
|
|
1167
|
+
// REPORT_PENDING/REPORT_INVALID states, fixed denominators, §6 statistics)
|
|
1168
|
+
// ---------------------------------------------------------------------------
|
|
1169
|
+
|
|
1170
|
+
export const BEHAVIOR_REPORT_V2_SCHEMA = 'challenge-behavior-report.v2';
|
|
1171
|
+
export const REPORT_STATUS_PENDING = 'REPORT_PENDING';
|
|
1172
|
+
export const REPORT_STATUS_INVALID = 'REPORT_INVALID';
|
|
1173
|
+
export const REPORT_V2_DIMENSIONS = [
|
|
1174
|
+
'protocolEvidenceQuality',
|
|
1175
|
+
'principleConformance',
|
|
1176
|
+
'evidenceAlignment',
|
|
1177
|
+
'objectiveTaskOutcome',
|
|
1178
|
+
'negativeTransfer',
|
|
1179
|
+
'operationalCost',
|
|
1180
|
+
'behavioralDiscrimination',
|
|
1181
|
+
'causalAttribution',
|
|
1182
|
+
'coverage',
|
|
1183
|
+
];
|
|
1184
|
+
export const REPORT_JSON_V2 = 'behavior-report-v2.json';
|
|
1185
|
+
export const REPORT_MD_V2 = 'behavior-report-v2.md';
|
|
1186
|
+
|
|
1187
|
+
/** Frozen study-level F-derivation rule for the §6 statistics: the target
|
|
1188
|
+
* failure mechanism (Pain #44 — conclusions beyond observations) manifests
|
|
1189
|
+
* mechanically as Evidence Alignment FAIL. F is a STUDY-level projection of
|
|
1190
|
+
* the replayed adjudication, recorded here so anyone can recompute it. */
|
|
1191
|
+
function failureMechanismFlag(adjudication) {
|
|
1192
|
+
if (!adjudication) return 'UNKNOWN';
|
|
1193
|
+
const status = adjudication.dimensions?.evidenceAlignment?.status;
|
|
1194
|
+
if (status === 'PASS') return 0;
|
|
1195
|
+
if (status === 'FAIL') return 1;
|
|
1196
|
+
return 'UNKNOWN';
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1199
|
+
function readGoldReviewStatuses(studyDir) {
|
|
1200
|
+
const file = path.join(studyDir, 'gold-review.json');
|
|
1201
|
+
let statuses = {};
|
|
1202
|
+
try {
|
|
1203
|
+
statuses = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
1204
|
+
} catch {
|
|
1205
|
+
return { statuses: {}, note: 'no gold-review.json — every case stays GOLD_REVIEW_PENDING (never silently approved)' };
|
|
1206
|
+
}
|
|
1207
|
+
return { statuses, note: 'per-case statuses from gold-review.json' };
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1210
|
+
// P1-4 (PR #2239 review): independent noise adjudication input. A record is
|
|
1211
|
+
// only honored when it binds the caseId, BOTH primary trace hashes, the
|
|
1212
|
+
// target mechanism and carries a decision + basis + source. Nothing here
|
|
1213
|
+
// auto-approves anything — a missing or mis-bound record leaves the case
|
|
1214
|
+
// INDETERMINATE, and the file itself is study-side material the scorer never
|
|
1215
|
+
// writes.
|
|
1216
|
+
function readNoiseAdjudications(studyDir) {
|
|
1217
|
+
const file = path.join(studyDir, 'noise-adjudications.json');
|
|
1218
|
+
let parsed = null;
|
|
1219
|
+
try {
|
|
1220
|
+
parsed = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
1221
|
+
} catch {
|
|
1222
|
+
return { records: {}, note: 'no noise-adjudications.json — independent noise adjudication stays pending (VBDR stays INDETERMINATE)' };
|
|
1223
|
+
}
|
|
1224
|
+
if (!parsed || typeof parsed !== 'object' || parsed.schema !== 'examiner-noise-adjudication.v1' || !parsed.records || typeof parsed.records !== 'object') {
|
|
1225
|
+
return { records: {}, note: 'noise-adjudications.json malformed (schema examiner-noise-adjudication.v1 required) — records ignored, VBDR stays INDETERMINATE' };
|
|
1226
|
+
}
|
|
1227
|
+
const records = {};
|
|
1228
|
+
for (const [caseId, record] of Object.entries(parsed.records)) {
|
|
1229
|
+
const ok = record && typeof record === 'object'
|
|
1230
|
+
&& ['CONFIRMED_REAL_DIFFERENCE', 'NOISE'].includes(record.decision)
|
|
1231
|
+
&& record.traceHashes && typeof record.traceHashes === 'object'
|
|
1232
|
+
&& typeof record.traceHashes.control === 'string' && typeof record.traceHashes.treatment === 'string'
|
|
1233
|
+
&& typeof record.targetMechanism === 'string' && record.targetMechanism.length > 0
|
|
1234
|
+
&& typeof record.basis === 'string' && record.basis.length > 0
|
|
1235
|
+
&& typeof record.source === 'string' && record.source.length > 0
|
|
1236
|
+
&& typeof record.adjudicatedAt === 'string';
|
|
1237
|
+
if (!ok) continue; // malformed records are ignored, never partially honored
|
|
1238
|
+
records[caseId] = record;
|
|
1239
|
+
}
|
|
1240
|
+
return { records, note: `per-case independent adjudications from noise-adjudications.json (${Object.keys(records).length} well-formed record(s))` };
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
// P1-4 (PR #2239 review): the F indicator must be VERIFIABLY bound to the
|
|
1244
|
+
// frozen Pain/Mechanism mapping before a difference can be called
|
|
1245
|
+
// mechanism-related. The binding is VERIFIED only when the study's mapping
|
|
1246
|
+
// file re-validates under its own recorded hash AND the study's F-derivation
|
|
1247
|
+
// rule is registered against that mechanism family. Anything less is UNKNOWN.
|
|
1248
|
+
const F_DERIVATION_MECHANISM_BINDINGS = [
|
|
1249
|
+
{ fDerivation: 'EVIDENCE_ALIGNMENT', mechanismFamily: 'PAIN44_CONCLUSION_BEYOND_OBSERVATION' },
|
|
1250
|
+
];
|
|
1251
|
+
|
|
1252
|
+
function resolveMechanismBinding(studyDir) {
|
|
1253
|
+
let mapping = null;
|
|
1254
|
+
try {
|
|
1255
|
+
mapping = readJsonFile(path.join(studyDir, 'pain-mechanism-mapping.json'));
|
|
1256
|
+
} catch {
|
|
1257
|
+
return { status: 'UNKNOWN', reason: 'no pain-mechanism-mapping.json — the F indicator cannot be bound to a frozen target mechanism' };
|
|
1258
|
+
}
|
|
1259
|
+
const hashOk = verifyCanonicalHash(mapping, mapping.mappingHash, { stripFields: ['mappingHash'] }).ok;
|
|
1260
|
+
if (!hashOk) return { status: 'UNKNOWN', reason: 'mappingHash drift — the frozen mechanism binding is unverifiable' };
|
|
1261
|
+
const registered = F_DERIVATION_MECHANISM_BINDINGS.some((binding) => binding.fDerivation === 'EVIDENCE_ALIGNMENT');
|
|
1262
|
+
if (!registered) return { status: 'UNKNOWN', reason: 'the EVIDENCE_ALIGNMENT F derivation is not registered against a frozen mechanism family' };
|
|
1263
|
+
const bound = F_DERIVATION_MECHANISM_BINDINGS.find((binding) => binding.fDerivation === 'EVIDENCE_ALIGNMENT');
|
|
1264
|
+
return {
|
|
1265
|
+
status: 'VERIFIED',
|
|
1266
|
+
mappingId: mapping.mappingId ?? null,
|
|
1267
|
+
mappingHash: mapping.mappingHash,
|
|
1268
|
+
mechanismFamily: bound.mechanismFamily,
|
|
1269
|
+
fDerivation: bound.fDerivation,
|
|
1270
|
+
};
|
|
1271
|
+
}
|
|
1272
|
+
|
|
1273
|
+
/**
|
|
1274
|
+
* P1-4 (PR #2239 review): project ONE case into its VBDR row using
|
|
1275
|
+
* repetition-EXACT pairing. A pair exists only when the SAME planned
|
|
1276
|
+
* repetition has both sides valid; different-repetition results can never be
|
|
1277
|
+
* spliced into one pair. repetitionStable is a tri-state: true = every
|
|
1278
|
+
* planned repetition complete AND consistent, false = complete but
|
|
1279
|
+
* inconsistent (OBSERVED instability), null = any repetition missing or
|
|
1280
|
+
* undetermined (UNKNOWN — never conflated with observed instability).
|
|
1281
|
+
*/
|
|
1282
|
+
export function projectVBDRCase({
|
|
1283
|
+
caseId, plannedRepetitions, rollupsByUnit, controlArm, treatmentArm,
|
|
1284
|
+
goldReview, mechanismBinding = null, noiseAdjudication = null, primaryTraceHashes = {},
|
|
1285
|
+
}) {
|
|
1286
|
+
if (!caseId || !Array.isArray(plannedRepetitions) || !(rollupsByUnit instanceof Map)) {
|
|
1287
|
+
throw new Error('projectVBDRCase: caseId, plannedRepetitions[] and rollupsByUnit (Map) required');
|
|
1288
|
+
}
|
|
1289
|
+
if (!controlArm || !treatmentArm) throw new Error('projectVBDRCase: controlArm/treatmentArm required');
|
|
1290
|
+
const sideValid = (rep, armId) => {
|
|
1291
|
+
const row = rollupsByUnit.get(`${caseId}/${armId}/r${String(rep)}`);
|
|
1292
|
+
return row && row.currentState === 'succeeded' && row.replayRescored === true;
|
|
1293
|
+
};
|
|
1294
|
+
const sideF = (rep, armId) => {
|
|
1295
|
+
const row = rollupsByUnit.get(`${caseId}/${armId}/r${String(rep)}`);
|
|
1296
|
+
if (!row) return 'UNKNOWN';
|
|
1297
|
+
return row.currentState === 'succeeded' && row.replayRescored === true && (row.f === 1 || row.f === 0) ? row.f : 'UNKNOWN';
|
|
1298
|
+
};
|
|
1299
|
+
const repetitionCoverage = {};
|
|
1300
|
+
const pairedReps = [];
|
|
1301
|
+
for (const rep of plannedRepetitions) {
|
|
1302
|
+
const paired = sideValid(rep, controlArm) && sideValid(rep, treatmentArm);
|
|
1303
|
+
repetitionCoverage[`r${String(rep)}`] = paired;
|
|
1304
|
+
if (paired) pairedReps.push(rep);
|
|
1305
|
+
}
|
|
1306
|
+
// pairedValid requires EVERY planned repetition to have a complete pair.
|
|
1307
|
+
const pairedValid = plannedRepetitions.length > 0 && pairedReps.length === plannedRepetitions.length;
|
|
1308
|
+
|
|
1309
|
+
// Tri-state repetition stability: per-arm across ALL planned repetitions.
|
|
1310
|
+
const armStability = (armId) => {
|
|
1311
|
+
const values = plannedRepetitions.map((rep) => sideF(rep, armId));
|
|
1312
|
+
if (values.some((f) => f !== 0 && f !== 1)) return null; // any missing/undetermined rep → UNKNOWN
|
|
1313
|
+
return new Set(values.map(String)).size <= 1; // complete: consistent → true, inconsistent → false
|
|
1314
|
+
};
|
|
1315
|
+
const controlStable = armStability(controlArm);
|
|
1316
|
+
const treatmentStable = armStability(treatmentArm);
|
|
1317
|
+
let repetitionStable = null;
|
|
1318
|
+
if (controlStable !== null && treatmentStable !== null) repetitionStable = controlStable && treatmentStable;
|
|
1319
|
+
|
|
1320
|
+
// Difference observed: only over repetition-exact pairs, and only when the
|
|
1321
|
+
// paired evidence does not point in opposite directions.
|
|
1322
|
+
let differenceObserved = null;
|
|
1323
|
+
if (pairedReps.length > 0) {
|
|
1324
|
+
const benefits = pairedReps.map((rep) => sideF(rep, controlArm) - sideF(rep, treatmentArm));
|
|
1325
|
+
differenceObserved = benefits.some((b) => b !== 0);
|
|
1326
|
+
if (benefits.some((b) => b > 0) && benefits.some((b) => b < 0)) differenceObserved = false; // opposite directions → no stable difference
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
// Mechanism relation: VERIFIED binding only — an observed difference alone
|
|
1330
|
+
// never establishes mechanism relatedness.
|
|
1331
|
+
const relatedToTargetMechanism = mechanismBinding && mechanismBinding.status === 'VERIFIED' ? true : null;
|
|
1332
|
+
|
|
1333
|
+
// Independent noise adjudication: bound to BOTH primary trace hashes and
|
|
1334
|
+
// the registered mechanism family; missing/mis-bound → null (UNKNOWN).
|
|
1335
|
+
let adjudicatedNotNoise = null;
|
|
1336
|
+
if (noiseAdjudication) {
|
|
1337
|
+
const hashBound = typeof primaryTraceHashes.control === 'string' && typeof primaryTraceHashes.treatment === 'string'
|
|
1338
|
+
&& noiseAdjudication.traceHashes.control === primaryTraceHashes.control
|
|
1339
|
+
&& noiseAdjudication.traceHashes.treatment === primaryTraceHashes.treatment
|
|
1340
|
+
&& (!mechanismBinding || mechanismBinding.status !== 'VERIFIED' || noiseAdjudication.targetMechanism === mechanismBinding.mechanismFamily);
|
|
1341
|
+
if (hashBound) adjudicatedNotNoise = noiseAdjudication.decision === 'CONFIRMED_REAL_DIFFERENCE';
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
const executionInvalid = plannedRepetitions.some((rep) => [controlArm, treatmentArm]
|
|
1345
|
+
.some((armId) => rollupsByUnit.get(`${caseId}/${armId}/r${String(rep)}`)?.currentState === 'failed_known'));
|
|
1346
|
+
|
|
1347
|
+
return {
|
|
1348
|
+
caseId,
|
|
1349
|
+
goldReview,
|
|
1350
|
+
pairedValid,
|
|
1351
|
+
executionInvalid,
|
|
1352
|
+
repetitionStable,
|
|
1353
|
+
differenceObserved,
|
|
1354
|
+
relatedToTargetMechanism,
|
|
1355
|
+
adjudicatedNotNoise,
|
|
1356
|
+
repetitionCoverage,
|
|
1357
|
+
diagnostics: {
|
|
1358
|
+
pairedRepetitions: pairedReps.map(String),
|
|
1359
|
+
controlStability: controlStable,
|
|
1360
|
+
treatmentStability: treatmentStable,
|
|
1361
|
+
},
|
|
1362
|
+
};
|
|
1363
|
+
}
|
|
1364
|
+
|
|
1365
|
+
// ---------------------------------------------------------------------------
|
|
1366
|
+
// P1-3 anchor policy — exact pre-anchor historical identities
|
|
1367
|
+
// ---------------------------------------------------------------------------
|
|
1368
|
+
//
|
|
1369
|
+
// The append-only PREREGISTRATION_FROZEN anchor was introduced after the two
|
|
1370
|
+
// acceptance studies below were frozen. They carry no anchor and must keep
|
|
1371
|
+
// their pre-anchor report path, so each is recognised by an EXACT identity:
|
|
1372
|
+
// studyId + a fingerprint over its immutable frozen artifacts. A missing
|
|
1373
|
+
// anchor alone is never sufficient, and the `EXM-V09-ACC-*` prefix is not
|
|
1374
|
+
// sufficient either — any unrecognised study without a valid anchor is
|
|
1375
|
+
// untrusted and can never certify a report.
|
|
1376
|
+
const PRE_ANCHOR_HISTORICAL_STUDIES = Object.freeze([
|
|
1377
|
+
{ studyId: 'EXM-V09-ACC-ISO', fingerprint: '952ede0722e338fd3bc1172c389457071df354f869043e8b267966ca653e30f8' },
|
|
1378
|
+
{ studyId: 'EXM-V09-ACC-FULL', fingerprint: '79a86f64d576a3c70c508fd69c6d99415b189092df9a1328d50954fd4f9140e2' },
|
|
1379
|
+
]);
|
|
1380
|
+
|
|
1381
|
+
function preAnchorHistoricalFingerprint(binding) {
|
|
1382
|
+
return hashOfCanonical({
|
|
1383
|
+
studyId: binding.studyId,
|
|
1384
|
+
manifestHash: binding.manifestHash,
|
|
1385
|
+
pairAssignmentHash: binding.pairAssignmentHash,
|
|
1386
|
+
preregistrationHash: binding.preregistrationHash,
|
|
1387
|
+
suiteHash: binding.suiteHash,
|
|
1388
|
+
mappingHash: binding.mappingHash,
|
|
1389
|
+
principleContentHash: binding.principleContentHash,
|
|
1390
|
+
});
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
function isPreAnchorHistoricalStudy(binding) {
|
|
1394
|
+
if (!binding) return false;
|
|
1395
|
+
const fingerprint = preAnchorHistoricalFingerprint(binding);
|
|
1396
|
+
return PRE_ANCHOR_HISTORICAL_STUDIES.some((entry) => entry.studyId === binding.studyId && entry.fingerprint === fingerprint);
|
|
1397
|
+
}
|
|
1398
|
+
|
|
1399
|
+
/**
|
|
1400
|
+
* An untrusted (unanchored, unrecognised) study has no attributable injection
|
|
1401
|
+
* evidence: every count is UNKNOWN — never 0 — and no capacity record is
|
|
1402
|
+
* trusted, so the report cannot present absence of evidence as evidence of
|
|
1403
|
+
* absence.
|
|
1404
|
+
*/
|
|
1405
|
+
function untrustedTreatmentFidelity(plannedTreatmentUnits, reason = 'preregistration anchor missing') {
|
|
1406
|
+
return {
|
|
1407
|
+
metric: 'TreatmentFidelity',
|
|
1408
|
+
status: 'UNKNOWN',
|
|
1409
|
+
plannedTreatmentUnits,
|
|
1410
|
+
evidencedChains: null,
|
|
1411
|
+
configured: null,
|
|
1412
|
+
selected: null,
|
|
1413
|
+
submitted: null,
|
|
1414
|
+
hostAccepted: null,
|
|
1415
|
+
contextPresent: null,
|
|
1416
|
+
runtimeLoaded: null,
|
|
1417
|
+
actualInjectionRate: { status: 'NOT_ESTIMABLE', reason: `${reason} — injection evidence cannot be attributed` },
|
|
1418
|
+
controlContamination: { units: null, ofControlChains: null },
|
|
1419
|
+
nonTargetRetention: {
|
|
1420
|
+
eligible: null,
|
|
1421
|
+
deliveredInTreatment: null,
|
|
1422
|
+
deliveredInControl: null,
|
|
1423
|
+
treatmentRetentionRate: null,
|
|
1424
|
+
controlRetentionRate: null,
|
|
1425
|
+
note: `${reason} — capacity records are not trusted`,
|
|
1426
|
+
},
|
|
1427
|
+
capacityEvictions: { units: null, details: [] },
|
|
1428
|
+
unresolvedTreatmentUnits: plannedTreatmentUnits,
|
|
1429
|
+
coverage: { status: 'UNKNOWN', note: `${reason} — no planned treatment unit has an attributable evidence chain` },
|
|
1430
|
+
rule: 'fidelity is a mechanism observability record — the actually-injected subset may NEVER replace the ITT primary analysis; an unverifiable injection downgrades causal attribution but never deletes assigned units',
|
|
1431
|
+
};
|
|
1432
|
+
}
|
|
1433
|
+
|
|
1434
|
+
const V09_IDENTITY_ARTIFACTS = Object.freeze(['pair-assignment.json', 'examiner-arm-configs.json', 'heps-principles.json']);
|
|
1435
|
+
|
|
1436
|
+
/**
|
|
1437
|
+
* P1-3.1: deleting examiner-manifest.json must not be a cheaper bypass than
|
|
1438
|
+
* deleting the anchor. A study that still carries any v0.9 identity marker is
|
|
1439
|
+
* an examiner study and cannot be downgraded to a plain Report V2; only a
|
|
1440
|
+
* study with NO v0.9 marker (a genuine pre-v0.9 study) may omit the block.
|
|
1441
|
+
*/
|
|
1442
|
+
function detectV09StudyIdentity(studyDir) {
|
|
1443
|
+
const markers = [];
|
|
1444
|
+
const readMaybe = (file) => {
|
|
1445
|
+
try { return JSON.parse(fs.readFileSync(path.join(studyDir, file), 'utf8')); } catch { return null; }
|
|
1446
|
+
};
|
|
1447
|
+
const prereg = readMaybe('preregistration.json');
|
|
1448
|
+
if (prereg && typeof prereg.examiner?.manifestHash === 'string' && prereg.examiner.manifestHash !== '') {
|
|
1449
|
+
markers.push('preregistration.json:examiner.manifestHash');
|
|
1450
|
+
}
|
|
1451
|
+
const hashes = readMaybe('hashes.json');
|
|
1452
|
+
if (hashes && hashes.examinerManifestHash) markers.push('hashes.json:examinerManifestHash');
|
|
1453
|
+
for (const file of V09_IDENTITY_ARTIFACTS) {
|
|
1454
|
+
if (fs.existsSync(path.join(studyDir, file))) markers.push(file);
|
|
1455
|
+
}
|
|
1456
|
+
return { present: markers.length > 0, markers };
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1459
|
+
/**
|
|
1460
|
+
* v0.9 examiner block — computed ONLY when the study carries an
|
|
1461
|
+
* examiner-manifest.json (new v0.9 studies; historical studies are untouched).
|
|
1462
|
+
* Every input is verified fail-closed and a drift becomes a critical finding,
|
|
1463
|
+
* so a tampered manifest/pair-assignment/endpoint can never produce a READY
|
|
1464
|
+
* report silently.
|
|
1465
|
+
*/
|
|
1466
|
+
function buildExaminerBlockMaybe(studyDir, {
|
|
1467
|
+
addFinding, caseById, planUnits, plannedPairSet, rollupsByUnit, controlArm, treatmentArm, replayedAdjudications,
|
|
1468
|
+
}) {
|
|
1469
|
+
const manifestPath = path.join(studyDir, 'examiner-manifest.json');
|
|
1470
|
+
if (!fs.existsSync(manifestPath)) {
|
|
1471
|
+
// P1-3.1: a v0.9 study that deletes the whole identity record must not
|
|
1472
|
+
// silently lose the examiner block (and with it every integrity check).
|
|
1473
|
+
const identity = detectV09StudyIdentity(studyDir);
|
|
1474
|
+
if (!identity.present) return null;
|
|
1475
|
+
const plannedTreatmentUnits = planUnits.filter((u) => u.armId === treatmentArm).length;
|
|
1476
|
+
addFinding('examiner_manifest_missing', 'critical', 'examiner-manifest.json',
|
|
1477
|
+
`the study still carries v0.9 examiner identity (${identity.markers.join(', ')}) — deleting the manifest cannot downgrade it to a plain Report V2`);
|
|
1478
|
+
return {
|
|
1479
|
+
specVersion: 'SPEC-PD-EXAMINER-001 v0.9',
|
|
1480
|
+
manifestHash: null,
|
|
1481
|
+
error: 'examiner_manifest_missing',
|
|
1482
|
+
applicability: { applicablePairs: null, notApplicablePairs: null },
|
|
1483
|
+
itt: {
|
|
1484
|
+
metric: 'ITT',
|
|
1485
|
+
status: 'NOT_ESTIMABLE',
|
|
1486
|
+
reason: 'examiner-manifest.json is missing — the primary endpoint and its applicability rule cannot be re-derived',
|
|
1487
|
+
plannedApplicablePairs: null,
|
|
1488
|
+
determinedPairs: null,
|
|
1489
|
+
unknownPairs: null,
|
|
1490
|
+
pointEstimate: { status: 'NOT_ESTIMABLE', reason: 'examiner manifest missing' },
|
|
1491
|
+
lowerBound: { status: 'NOT_ESTIMABLE', value: null },
|
|
1492
|
+
upperBound: { status: 'NOT_ESTIMABLE', value: null },
|
|
1493
|
+
},
|
|
1494
|
+
treatmentFidelity: untrustedTreatmentFidelity(plannedTreatmentUnits, 'examiner manifest missing'),
|
|
1495
|
+
randomization: null,
|
|
1496
|
+
injectionEvidence: {
|
|
1497
|
+
attribution: 'UNAVAILABLE — examiner-manifest.json is missing',
|
|
1498
|
+
note: 'no injection evidence is scanned or counted without the frozen study manifest; an unknown injection state is never reported as zero',
|
|
1499
|
+
},
|
|
1500
|
+
evidence: 'UNTRUSTED — the study identity files exist but examiner-manifest.json is missing; ITT/Fidelity/capacity cannot certify this study',
|
|
1501
|
+
};
|
|
1502
|
+
}
|
|
1503
|
+
let exManifest = null;
|
|
1504
|
+
try {
|
|
1505
|
+
exManifest = readJsonFile(manifestPath);
|
|
1506
|
+
} catch (e) {
|
|
1507
|
+
addFinding('examiner_manifest_unreadable', 'critical', 'examiner-manifest.json', e.message);
|
|
1508
|
+
return { manifestHash: null, error: 'examiner_manifest_unreadable' };
|
|
1509
|
+
}
|
|
1510
|
+
const manifestCheck = verifyExaminerManifest(exManifest);
|
|
1511
|
+
if (!manifestCheck.ok) {
|
|
1512
|
+
addFinding('examiner_manifest_drift', 'critical', 'examiner-manifest.json', manifestCheck.problems.join('; '));
|
|
1513
|
+
}
|
|
1514
|
+
// P1-3: cross-file binding + immutable anchor. A study created by the
|
|
1515
|
+
// anchored path must match its anchor exactly. Only an EXACTLY identified
|
|
1516
|
+
// pre-anchor historical study may use the legacy (unanchored) evidence path;
|
|
1517
|
+
// every other study without a valid anchor is untrusted and fails closed.
|
|
1518
|
+
let preregistrationBinding = null;
|
|
1519
|
+
let legacyEvidencePath = false;
|
|
1520
|
+
// Default requireAnchor:true — a missing anchor is never silently tolerated.
|
|
1521
|
+
const bindings = verifyExaminerStudyBindings(studyDir);
|
|
1522
|
+
const preAnchorHistorical = isPreAnchorHistoricalStudy(bindings.binding);
|
|
1523
|
+
if (bindings.anchor === 'VERIFIED') {
|
|
1524
|
+
preregistrationBinding = { bindingHash: bindings.bindingHash, anchor: 'VERIFIED' };
|
|
1525
|
+
} else if (preAnchorHistorical && bindings.anchor !== 'MISMATCH') {
|
|
1526
|
+
// Known historical identity: keep the pre-anchor path so its report stays
|
|
1527
|
+
// unchanged. A binding that no longer recomputes under the current
|
|
1528
|
+
// implementation is still reported as a critical finding.
|
|
1529
|
+
legacyEvidencePath = true;
|
|
1530
|
+
if (bindings.anchor === 'NOT_CHECKED') {
|
|
1531
|
+
addFinding('examiner_preregistration_binding_unreadable', 'critical', 'examiner-manifest.json', (bindings.problems || ['binding recomputation failed']).join('; '));
|
|
1532
|
+
}
|
|
1533
|
+
} else if (bindings.anchor === 'MISMATCH') {
|
|
1534
|
+
addFinding('examiner_preregistration_binding_drift', 'critical', 'events.jsonl', (bindings.problems || ['binding mismatch']).join('; '));
|
|
1535
|
+
} else if (bindings.anchor === 'NOT_CHECKED') {
|
|
1536
|
+
addFinding('examiner_preregistration_binding_unreadable', 'critical', 'examiner-manifest.json', (bindings.problems || ['binding recomputation failed']).join('; '));
|
|
1537
|
+
} else {
|
|
1538
|
+
addFinding('examiner_preregistration_anchor_missing', 'critical', 'events.jsonl',
|
|
1539
|
+
'immutable preregistration anchor MISSING — only a study created by the anchored path, or an exactly identified pre-anchor historical study, may produce a report');
|
|
1540
|
+
}
|
|
1541
|
+
// Pair assignment: the frozen randomization record.
|
|
1542
|
+
let pairAssignment = null;
|
|
1543
|
+
try {
|
|
1544
|
+
pairAssignment = readJsonFile(path.join(studyDir, 'pair-assignment.json'));
|
|
1545
|
+
} catch (e) {
|
|
1546
|
+
addFinding('examiner_pair_assignment_missing', 'critical', 'pair-assignment.json', `unreadable: ${e.message}`);
|
|
1547
|
+
}
|
|
1548
|
+
let pairAssignmentVerified = false;
|
|
1549
|
+
if (pairAssignment) {
|
|
1550
|
+
const check = verifyPairAssignment(pairAssignment);
|
|
1551
|
+
pairAssignmentVerified = check.ok;
|
|
1552
|
+
if (!check.ok) addFinding('examiner_pair_assignment_drift', 'critical', 'pair-assignment.json', check.reason);
|
|
1553
|
+
}
|
|
1554
|
+
// Primary endpoint: re-derive from the CLOSED registry and compare hashes —
|
|
1555
|
+
// a post-freeze endpoint change is exactly the §5.1 violation this catches.
|
|
1556
|
+
let endpoint = null;
|
|
1557
|
+
try {
|
|
1558
|
+
endpoint = definePrimaryEndpoint({ id: exManifest.primaryEndpoint.id });
|
|
1559
|
+
} catch (e) {
|
|
1560
|
+
addFinding('examiner_endpoint_unknown', 'critical', 'examiner-manifest.json', e.message);
|
|
1561
|
+
}
|
|
1562
|
+
if (endpoint) {
|
|
1563
|
+
if (endpoint.definitionHash !== exManifest.primaryEndpoint.definitionHash) {
|
|
1564
|
+
addFinding('examiner_endpoint_drift', 'critical', 'examiner-manifest.json', `primary endpoint '${exManifest.primaryEndpoint.id}' no longer matches its frozen definitionHash — the endpoint definition changed after preregistration`);
|
|
1565
|
+
}
|
|
1566
|
+
if (endpoint.applicabilityRuleHash !== exManifest.primaryEndpoint.applicabilityRuleHash) {
|
|
1567
|
+
addFinding('examiner_endpoint_applicability_drift', 'critical', 'examiner-manifest.json', 'primary endpoint applicability rule drifted after preregistration');
|
|
1568
|
+
}
|
|
1569
|
+
}
|
|
1570
|
+
// Applicable pairs under the FROZEN applicability predicate (NOT_APPLICABLE
|
|
1571
|
+
// pairs are counted separately and never enter the ITT denominator).
|
|
1572
|
+
const applicablePairs = [];
|
|
1573
|
+
let notApplicablePairs = 0;
|
|
1574
|
+
for (const pair of plannedPairSet.pairs) {
|
|
1575
|
+
const caseDef = caseById.get(pair.caseId);
|
|
1576
|
+
const applicable = caseDef ? caseDef[endpoint ? endpoint.applicabilityRule.field : 'principleApplicable'] : undefined;
|
|
1577
|
+
const isApplicable = endpoint ? applicable === endpoint.applicabilityRule.equals : applicable === true;
|
|
1578
|
+
if (!isApplicable) {
|
|
1579
|
+
notApplicablePairs += 1;
|
|
1580
|
+
continue;
|
|
1581
|
+
}
|
|
1582
|
+
// Endpoint F per side from the REPLAYED adjudication (never the stored one).
|
|
1583
|
+
const endpointFOf = (armId) => {
|
|
1584
|
+
const rollup = rollupsByUnit.get(`${pair.caseId}/${armId}/r${String(pair.repetitionId)}`);
|
|
1585
|
+
if (!rollup || rollup.currentState !== 'succeeded' || rollup.replayRescored !== true || !endpoint) return 'UNKNOWN';
|
|
1586
|
+
const primaryKey = `${pair.caseId}/${armId}/r${String(pair.repetitionId)}/${rollup.primaryAttempt}`;
|
|
1587
|
+
const replayed = replayedAdjudications.get(primaryKey);
|
|
1588
|
+
if (!replayed) return 'UNKNOWN';
|
|
1589
|
+
return evaluatePrimaryEndpointF({ endpoint, adjudication: replayed, caseDef });
|
|
1590
|
+
};
|
|
1591
|
+
const controlF = endpointFOf(controlArm);
|
|
1592
|
+
const treatmentF = endpointFOf(treatmentArm);
|
|
1593
|
+
const sideValid = (armId) => {
|
|
1594
|
+
const rollup = rollupsByUnit.get(`${pair.caseId}/${armId}/r${String(pair.repetitionId)}`);
|
|
1595
|
+
return rollup ? rollup.currentState === 'succeeded' && rollup.replayRescored === true : false;
|
|
1596
|
+
};
|
|
1597
|
+
applicablePairs.push({
|
|
1598
|
+
pairKey: pair.pairKey,
|
|
1599
|
+
controlF,
|
|
1600
|
+
treatmentF,
|
|
1601
|
+
controlExecutionValid: sideValid(controlArm),
|
|
1602
|
+
treatmentExecutionValid: sideValid(treatmentArm),
|
|
1603
|
+
});
|
|
1604
|
+
}
|
|
1605
|
+
const itt = computeITT(applicablePairs);
|
|
1606
|
+
// Injection evidence chains (runner-produced sidecars) — verified, then fed
|
|
1607
|
+
// to fidelity. Control chains prove the absence hash; any control chain
|
|
1608
|
+
// proving a real payload at CONTEXT_PRESENT+ is contamination.
|
|
1609
|
+
// P1-2: for an ANCHORED study attribution comes from the chain's own binding
|
|
1610
|
+
// (study/unit/case/arm/repetition/attempt + prompt/receipt hash) checked
|
|
1611
|
+
// against the SAVED primary attempt record — never from the directory a
|
|
1612
|
+
// sidecar happens to sit in. One planned unit contributes at most ONE chain
|
|
1613
|
+
// (its primary attempt), so duplicate attempts cannot inflate fidelity.
|
|
1614
|
+
// Legacy studies predate v2 binding and keep the historical scan so their
|
|
1615
|
+
// reports stay byte-identical.
|
|
1616
|
+
const chains = preregistrationBinding
|
|
1617
|
+
? collectBoundInjectionChains({ studyDir, studyId: exManifest.studyId, planUnits, rollupsByUnit, controlArm, treatmentArm, addFinding })
|
|
1618
|
+
: legacyEvidencePath
|
|
1619
|
+
? scanLegacyInjectionChains({ studyDir, controlArm, treatmentArm, addFinding })
|
|
1620
|
+
: null;
|
|
1621
|
+
// P1-6: full-library capacity competition recomputed from the frozen arm
|
|
1622
|
+
// configs/payloads (never a hardcoded 0), with the honest three-state answer.
|
|
1623
|
+
const capacity = preregistrationBinding ? recomputeExaminerCapacity({ studyDir, exManifest, addFinding }) : null;
|
|
1624
|
+
const plannedTreatmentUnits = planUnits.filter((u) => u.armId === treatmentArm).length;
|
|
1625
|
+
const unresolvedTreatmentUnits = chains === null
|
|
1626
|
+
? plannedTreatmentUnits
|
|
1627
|
+
: Math.max(0, plannedTreatmentUnits - chains.treatment.length);
|
|
1628
|
+
const treatmentFidelity = chains === null
|
|
1629
|
+
? untrustedTreatmentFidelity(plannedTreatmentUnits)
|
|
1630
|
+
: computeTreatmentFidelity({
|
|
1631
|
+
plannedTreatmentUnits,
|
|
1632
|
+
chains: chains.treatment,
|
|
1633
|
+
controlChains: chains.control,
|
|
1634
|
+
...(capacity && capacity.status === 'RECORDED' ? { nonTargetRetention: capacity.nonTargetRetention, capacityRecords: [capacity.control, capacity.treatment] } : {}),
|
|
1635
|
+
...(preregistrationBinding ? { unresolvedTreatmentUnits } : {}),
|
|
1636
|
+
});
|
|
1637
|
+
return {
|
|
1638
|
+
specVersion: 'SPEC-PD-EXAMINER-001 v0.9',
|
|
1639
|
+
manifestHash: exManifest.manifestHash ?? null,
|
|
1640
|
+
...(preregistrationBinding ? { preregistrationBinding } : {}),
|
|
1641
|
+
experimentMode: exManifest.experimentMode ?? null,
|
|
1642
|
+
interventionType: exManifest.interventionType ?? null,
|
|
1643
|
+
targetPrincipleId: exManifest.targetPrinciple ? exManifest.targetPrinciple.id : null,
|
|
1644
|
+
primaryEndpoint: exManifest.primaryEndpoint ?? null,
|
|
1645
|
+
applicability: { applicablePairs: applicablePairs.length, notApplicablePairs },
|
|
1646
|
+
itt,
|
|
1647
|
+
treatmentFidelity,
|
|
1648
|
+
...(capacity ? { capacity } : {}),
|
|
1649
|
+
randomization: pairAssignment
|
|
1650
|
+
? {
|
|
1651
|
+
strategy: pairAssignment.strategy,
|
|
1652
|
+
seed: pairAssignment.seed,
|
|
1653
|
+
plannedPairs: pairAssignment.plannedPairs ?? (pairAssignment.pairs ? pairAssignment.pairs.length : null),
|
|
1654
|
+
balance: pairAssignment.balance ?? null,
|
|
1655
|
+
assignmentHash: pairAssignment.assignmentHash,
|
|
1656
|
+
verified: pairAssignmentVerified,
|
|
1657
|
+
}
|
|
1658
|
+
: null,
|
|
1659
|
+
injectionEvidence: chains === null
|
|
1660
|
+
? {
|
|
1661
|
+
attribution: 'UNAVAILABLE — the study has no verified preregistration anchor',
|
|
1662
|
+
note: 'injection evidence is neither scanned nor counted for an unanchored study; an unknown injection state is never reported as zero',
|
|
1663
|
+
}
|
|
1664
|
+
: {
|
|
1665
|
+
treatmentChains: chains.treatment.length,
|
|
1666
|
+
controlChains: chains.control.length,
|
|
1667
|
+
...(preregistrationBinding ? {
|
|
1668
|
+
attribution: 'chain-internal binding verified against each unit\u2019s saved primary attempt record',
|
|
1669
|
+
duplicateAttemptSidecars: chains.duplicates,
|
|
1670
|
+
unresolvedTreatmentUnits,
|
|
1671
|
+
} : {}),
|
|
1672
|
+
note: 'chains are runner-produced tamper-evident sidecars (offline-stub scope); SUBMITTED never equals RUNTIME_LOADED and declaration-only never counts',
|
|
1673
|
+
},
|
|
1674
|
+
evidence: chains === null
|
|
1675
|
+
? 'UNTRUSTED — no verified preregistration anchor; ITT/Fidelity/capacity cannot certify this study'
|
|
1676
|
+
: 'ITT/Fidelity recomputable from pair-assignment.json + planUnits + injection-evidence sidecars + replayed adjudications',
|
|
1677
|
+
};
|
|
1678
|
+
}
|
|
1679
|
+
|
|
1680
|
+
const evidenceUnitKey = (u) => `${u.caseId}/${u.armId}/r${String(u.repetitionId)}`;
|
|
1681
|
+
|
|
1682
|
+
/**
|
|
1683
|
+
* Anchored-study chain collector. For every PLANNED unit it reads the sidecar
|
|
1684
|
+
* of that unit's PRIMARY attempt only and verifies the full binding against
|
|
1685
|
+
* the saved attempt record: the attempt must exist and succeed, its
|
|
1686
|
+
* promptAudit.promptHash must equal the chain's promptHash, its trace must
|
|
1687
|
+
* carry the runtime's own receipt (hash-valid, same prompt), and the chain's
|
|
1688
|
+
* receiptHash must be that receipt. Anything else leaves the unit UNKNOWN —
|
|
1689
|
+
* the unit is never silently counted as injected or not-injected.
|
|
1690
|
+
*/
|
|
1691
|
+
function collectBoundInjectionChains({ studyDir, studyId, planUnits, rollupsByUnit, controlArm, treatmentArm, addFinding }) {
|
|
1692
|
+
const chains = { treatment: [], control: [], duplicates: 0 };
|
|
1693
|
+
const seen = new Set();
|
|
1694
|
+
const units = [...planUnits].sort((a, b) => evidenceUnitKey(a).localeCompare(evidenceUnitKey(b)));
|
|
1695
|
+
for (const unit of units) {
|
|
1696
|
+
const key = evidenceUnitKey(unit);
|
|
1697
|
+
if (seen.has(key)) { chains.duplicates += 1; continue; }
|
|
1698
|
+
seen.add(key);
|
|
1699
|
+
const repDir = path.join(studyDir, 'runs', unit.caseId, unit.armId, `r${String(unit.repetitionId)}`);
|
|
1700
|
+
const sidecars = fs.existsSync(repDir)
|
|
1701
|
+
? fs.readdirSync(repDir).filter((f) => f.startsWith('injection-evidence-') && f.endsWith('.json')).sort()
|
|
1702
|
+
: [];
|
|
1703
|
+
if (sidecars.length > 1) {
|
|
1704
|
+
chains.duplicates += sidecars.length - 1;
|
|
1705
|
+
addFinding('examiner_injection_duplicate_attempts', 'warning', `runs/${unit.caseId}/${unit.armId}/r${String(unit.repetitionId)}`,
|
|
1706
|
+
`${sidecars.length} evidence sidecars exist for one planned unit — only the primary attempt counts; duplicates never inflate Treatment Fidelity`);
|
|
1707
|
+
}
|
|
1708
|
+
const rollup = rollupsByUnit.get(key);
|
|
1709
|
+
const primaryAttempt = rollup && rollup.currentState === 'succeeded' && rollup.primaryAttempt ? String(rollup.primaryAttempt) : null;
|
|
1710
|
+
if (!primaryAttempt) continue; // unresolved: no successful primary attempt
|
|
1711
|
+
const sidecarPath = path.join(repDir, `injection-evidence-${primaryAttempt}.json`);
|
|
1712
|
+
if (!fs.existsSync(sidecarPath)) {
|
|
1713
|
+
addFinding('examiner_injection_evidence_missing', 'warning', `runs/${unit.caseId}/${unit.armId}/r${String(unit.repetitionId)}/injection-evidence-${primaryAttempt}.json`,
|
|
1714
|
+
'the unit succeeded but produced no injection evidence chain — its injection status is UNKNOWN');
|
|
1715
|
+
continue;
|
|
1716
|
+
}
|
|
1717
|
+
const relSubject = `runs/${unit.caseId}/${unit.armId}/r${String(unit.repetitionId)}/injection-evidence-${primaryAttempt}.json`;
|
|
1718
|
+
let chain;
|
|
1719
|
+
let attempt;
|
|
1720
|
+
try {
|
|
1721
|
+
chain = readJsonFile(sidecarPath);
|
|
1722
|
+
attempt = readJsonFile(path.join(repDir, `${primaryAttempt}.json`));
|
|
1723
|
+
} catch (e) {
|
|
1724
|
+
addFinding('injection_evidence_unreadable', 'critical', relSubject, e.message);
|
|
1725
|
+
continue;
|
|
1726
|
+
}
|
|
1727
|
+
const fail = (detail) => addFinding('examiner_injection_binding_mismatch', 'critical', relSubject, detail);
|
|
1728
|
+
const attemptUnit = attempt.unit || {};
|
|
1729
|
+
if (attemptUnit.caseId !== unit.caseId || attemptUnit.armId !== unit.armId
|
|
1730
|
+
|| String(attemptUnit.repetitionId) !== String(unit.repetitionId) || String(attemptUnit.attemptId) !== primaryAttempt) {
|
|
1731
|
+
fail('the saved attempt record belongs to a different unit than the planned one');
|
|
1732
|
+
continue;
|
|
1733
|
+
}
|
|
1734
|
+
const promptHash = attempt.promptAudit && attempt.promptAudit.promptHash;
|
|
1735
|
+
const traceReceipt = attempt.trace && attempt.trace.runtimeReceipt;
|
|
1736
|
+
const receiptCheck = verifyOfflineRuntimeReceipt(traceReceipt);
|
|
1737
|
+
if (!receiptCheck.ok) {
|
|
1738
|
+
fail(`the attempt trace carries no valid runtime load receipt (${receiptCheck.problems.join('; ')}) — RUNTIME_LOADED cannot be claimed`);
|
|
1739
|
+
continue;
|
|
1740
|
+
}
|
|
1741
|
+
if (traceReceipt.receivedPromptHash !== promptHash) {
|
|
1742
|
+
fail('the runtime receipt is for a different prompt than the attempt recorded');
|
|
1743
|
+
continue;
|
|
1744
|
+
}
|
|
1745
|
+
const expected = {
|
|
1746
|
+
studyId,
|
|
1747
|
+
unitId: key,
|
|
1748
|
+
caseId: unit.caseId,
|
|
1749
|
+
armId: unit.armId,
|
|
1750
|
+
repetitionId: String(unit.repetitionId),
|
|
1751
|
+
attemptId: primaryAttempt,
|
|
1752
|
+
promptHash,
|
|
1753
|
+
};
|
|
1754
|
+
const verified = verifyInjectionEvidenceChain(chain, { expected });
|
|
1755
|
+
if (!verified.ok) {
|
|
1756
|
+
fail(verified.problems.join('; '));
|
|
1757
|
+
continue;
|
|
1758
|
+
}
|
|
1759
|
+
if (chain.receiptHash !== traceReceipt.receiptHash) {
|
|
1760
|
+
fail('the chain receiptHash is not the receipt in the saved attempt trace');
|
|
1761
|
+
continue;
|
|
1762
|
+
}
|
|
1763
|
+
const forbiddenPresent = chain.stages && chain.stages.CONTEXT_PRESENT && chain.stages.CONTEXT_PRESENT.forbiddenContentPresent;
|
|
1764
|
+
const entry = {
|
|
1765
|
+
ok: true,
|
|
1766
|
+
provenLevel: chain.provenLevel,
|
|
1767
|
+
principleHash: chain.principleHash,
|
|
1768
|
+
chainHash: chain.chainHash,
|
|
1769
|
+
unitId: key,
|
|
1770
|
+
attemptId: primaryAttempt,
|
|
1771
|
+
...(forbiddenPresent !== undefined ? { forbiddenContentPresent: forbiddenPresent } : {}),
|
|
1772
|
+
};
|
|
1773
|
+
if (unit.armId === treatmentArm) chains.treatment.push(entry);
|
|
1774
|
+
else if (unit.armId === controlArm) chains.control.push(entry);
|
|
1775
|
+
}
|
|
1776
|
+
return chains;
|
|
1777
|
+
}
|
|
1778
|
+
|
|
1779
|
+
/** Historical (pre-anchor) scan: kept verbatim so legacy reports stay stable. */
|
|
1780
|
+
function scanLegacyInjectionChains({ studyDir, controlArm, treatmentArm, addFinding }) {
|
|
1781
|
+
const chains = { treatment: [], control: [] };
|
|
1782
|
+
const runsDir = path.join(studyDir, 'runs');
|
|
1783
|
+
if (!fs.existsSync(runsDir)) return chains;
|
|
1784
|
+
for (const caseDir of fs.readdirSync(runsDir).sort()) {
|
|
1785
|
+
const casePath = path.join(runsDir, caseDir);
|
|
1786
|
+
if (!fs.statSync(casePath).isDirectory()) continue;
|
|
1787
|
+
for (const armDir of fs.readdirSync(casePath).sort()) {
|
|
1788
|
+
const armPath = path.join(casePath, armDir);
|
|
1789
|
+
if (!fs.statSync(armPath).isDirectory()) continue;
|
|
1790
|
+
for (const repDir of fs.readdirSync(armPath).sort()) {
|
|
1791
|
+
const repPath = path.join(armPath, repDir);
|
|
1792
|
+
if (!fs.statSync(repPath).isDirectory()) continue;
|
|
1793
|
+
for (const file of fs.readdirSync(repPath).sort()) {
|
|
1794
|
+
if (!file.startsWith('injection-evidence-') || !file.endsWith('.json')) continue;
|
|
1795
|
+
try {
|
|
1796
|
+
const chain = readJsonFile(path.join(repPath, file));
|
|
1797
|
+
const verified = verifyInjectionEvidenceChain(chain);
|
|
1798
|
+
const forbiddenPresent = chain.stages && chain.stages.CONTEXT_PRESENT && chain.stages.CONTEXT_PRESENT.forbiddenContentPresent;
|
|
1799
|
+
const entry = { ok: verified.ok, provenLevel: verified.provenLevel, principleHash: chain.principleHash, chainHash: chain.chainHash, ...(forbiddenPresent !== undefined ? { forbiddenContentPresent: forbiddenPresent } : {}) };
|
|
1800
|
+
if (armDir === treatmentArm) chains.treatment.push(entry);
|
|
1801
|
+
else if (armDir === controlArm) chains.control.push(entry);
|
|
1802
|
+
} catch (e) {
|
|
1803
|
+
addFinding('injection_evidence_unreadable', 'critical', `runs/${caseDir}/${armDir}/${repDir}/${file}`, e.message);
|
|
1804
|
+
}
|
|
1805
|
+
}
|
|
1806
|
+
}
|
|
1807
|
+
}
|
|
1808
|
+
}
|
|
1809
|
+
return chains;
|
|
1810
|
+
}
|
|
1811
|
+
|
|
1812
|
+
/**
|
|
1813
|
+
* §2.2.5 capacity competition for the report — rebuilt from the FROZEN HEPS +
|
|
1814
|
+
* target principle + budget/ordering, never read from the mutable creation
|
|
1815
|
+
* record as the authority (the stored record is only cross-checked). Three
|
|
1816
|
+
* states: NOT_APPLICABLE (isolated mode), UNKNOWN (cannot recompute), or
|
|
1817
|
+
* RECORDED (counts are real, including a genuine 0).
|
|
1818
|
+
*/
|
|
1819
|
+
function recomputeExaminerCapacity({ studyDir, exManifest, addFinding }) {
|
|
1820
|
+
if (exManifest.experimentMode === 'isolated_principle') {
|
|
1821
|
+
return { status: 'NOT_APPLICABLE', reason: 'isolated mode: the control payload is EMPTY and the treatment carries the target only — no capacity competition exists' };
|
|
1822
|
+
}
|
|
1823
|
+
if (exManifest.experimentMode !== 'full_library_incremental') {
|
|
1824
|
+
return { status: 'UNKNOWN', reason: `experiment mode '${String(exManifest.experimentMode)}' has no capacity definition` };
|
|
1825
|
+
}
|
|
1826
|
+
try {
|
|
1827
|
+
const hepsInput = readJsonFile(path.join(studyDir, 'heps-principles.json'));
|
|
1828
|
+
const armConfigs = readJsonFile(path.join(studyDir, 'examiner-arm-configs.json'));
|
|
1829
|
+
const heps = buildHostEffectivePrincipleSet({ principles: hepsInput.principles, targetPrincipleId: hepsInput.targetPrincipleId, host: hepsInput.host });
|
|
1830
|
+
const targetFact = (hepsInput.principles || []).find((p) => p.id === hepsInput.targetPrincipleId);
|
|
1831
|
+
if (!targetFact) throw new Error('target principle missing from heps-principles.json');
|
|
1832
|
+
const budgetChars = Number.isInteger(armConfigs.budgetChars) ? armConfigs.budgetChars : null;
|
|
1833
|
+
if (budgetChars === null) throw new Error('examiner-arm-configs budgetChars missing — capacity cannot be bounded');
|
|
1834
|
+
const rebuilt = buildDualModeArmPayloads({ mode: exManifest.experimentMode, heps, targetPrinciple: targetFact, principleFacts: hepsInput.principles, budgetChars, ordering: armConfigs.ordering || 'id-asc' });
|
|
1835
|
+
const factsOf = (record) => (record ? {
|
|
1836
|
+
armId: record.armId,
|
|
1837
|
+
policy: record.policy,
|
|
1838
|
+
budgetChars: record.budgetChars,
|
|
1839
|
+
eligibleCount: record.eligibleCount,
|
|
1840
|
+
deliveredIds: record.delivered.map((p) => p.id).sort(),
|
|
1841
|
+
evictedIds: record.evicted.map((p) => p.id).sort(),
|
|
1842
|
+
// `evicted` keeps the record shape computeTreatmentFidelity consumes so
|
|
1843
|
+
// capacity evictions reach the report instead of being counted as 0.
|
|
1844
|
+
evicted: record.evicted.map((p) => ({ id: p.id, contentHash: p.contentHash })),
|
|
1845
|
+
targetDelivered: record.targetDelivered,
|
|
1846
|
+
recordHash: record.recordHash,
|
|
1847
|
+
} : null);
|
|
1848
|
+
const control = factsOf(rebuilt.capacity.control);
|
|
1849
|
+
const treatment = factsOf(rebuilt.capacity.treatment);
|
|
1850
|
+
if (!control || !treatment) throw new Error('rebuilt arm payloads carry no capacity record');
|
|
1851
|
+
const stored = armConfigs.capacity || {};
|
|
1852
|
+
const storedAgrees = stored.control?.recordHash === control.recordHash && stored.treatment?.recordHash === treatment.recordHash;
|
|
1853
|
+
if (!storedAgrees) {
|
|
1854
|
+
addFinding('examiner_capacity_record_drift', 'critical', 'examiner-arm-configs.json',
|
|
1855
|
+
'the stored capacity competition record does not match a recomputation from the frozen HEPS + budget — the report uses the recomputation');
|
|
1856
|
+
}
|
|
1857
|
+
const targetId = targetFact.id;
|
|
1858
|
+
return {
|
|
1859
|
+
status: 'RECORDED',
|
|
1860
|
+
selectionRule: control.policy,
|
|
1861
|
+
budgetChars,
|
|
1862
|
+
control,
|
|
1863
|
+
treatment,
|
|
1864
|
+
storedRecordAgrees: storedAgrees,
|
|
1865
|
+
nonTargetRetention: {
|
|
1866
|
+
eligible: control.eligibleCount,
|
|
1867
|
+
deliveredInControl: control.deliveredIds.filter((id) => id !== targetId).length,
|
|
1868
|
+
deliveredInTreatment: treatment.deliveredIds.filter((id) => id !== targetId).length,
|
|
1869
|
+
},
|
|
1870
|
+
};
|
|
1871
|
+
} catch (e) {
|
|
1872
|
+
addFinding('examiner_capacity_uncomputable', 'critical', 'examiner-arm-configs.json', e.message);
|
|
1873
|
+
return { status: 'UNKNOWN', reason: e.message };
|
|
1874
|
+
}
|
|
1875
|
+
}
|
|
1876
|
+
|
|
1877
|
+
export function buildBehaviorReportV2(studyDir, { now } = {}) {
|
|
1878
|
+
const findings = [];
|
|
1879
|
+
const addFinding = (id, severity, subject, detail) => findings.push({ id, severity, subject, detail });
|
|
1880
|
+
|
|
1881
|
+
// --- frozen plan artifacts + frozen verification (same authorities as v1) --
|
|
1882
|
+
let hashes = null;
|
|
1883
|
+
let prereg = null;
|
|
1884
|
+
let authRecord = null;
|
|
1885
|
+
for (const [name, file, setter] of [
|
|
1886
|
+
['hashes.json', 'hashes.json', (v) => { hashes = v; }],
|
|
1887
|
+
['preregistration.json', 'preregistration.json', (v) => { prereg = v; }],
|
|
1888
|
+
['run-authorization.json', 'run-authorization.json', (v) => { authRecord = v; }],
|
|
1889
|
+
]) {
|
|
1890
|
+
try {
|
|
1891
|
+
setter(readJsonFile(path.join(studyDir, file)));
|
|
1892
|
+
} catch (e) {
|
|
1893
|
+
if (name !== 'run-authorization.json') addFinding('key_input_missing', 'critical', file, `${name} unreadable: ${e.message}`);
|
|
1894
|
+
}
|
|
1895
|
+
}
|
|
1896
|
+
let frozen = null;
|
|
1897
|
+
let bindingsVerified = true;
|
|
1898
|
+
try {
|
|
1899
|
+
frozen = verifyFrozenStudy(studyDir);
|
|
1900
|
+
} catch (e) {
|
|
1901
|
+
bindingsVerified = false;
|
|
1902
|
+
addFinding('frozen_binding_drift', 'critical', 'suite', e.message);
|
|
1903
|
+
}
|
|
1904
|
+
const scoringContract = frozen ? readJsonFile(path.join(studyDir, 'scoring-contract.json')) : null;
|
|
1905
|
+
if (scoringContract && scoringContract.schema !== 'claim-evidence-scoring.v3') {
|
|
1906
|
+
throw new Error(`REPORT_V2_REQUIRES_V3_CONTRACT: scoring contract is '${scoringContract.schema}' — Report V2 serves v3-contract studies; use the v1 report for historical contracts`);
|
|
1907
|
+
}
|
|
1908
|
+
const implementation = challengeImplementationManifest();
|
|
1909
|
+
if (scoringContract && hashOfCanonical(scoringContract.implementation) !== hashOfCanonical(implementation)) {
|
|
1910
|
+
bindingsVerified = false;
|
|
1911
|
+
addFinding('implementation_drift', 'critical', 'scoring-contract.json', 'frozen implementation manifest != current lib implementation — rescoring under a drifted implementation is forbidden');
|
|
1912
|
+
}
|
|
1913
|
+
const manifest = frozen ? frozen.manifest : null;
|
|
1914
|
+
const cases = manifest ? manifest.cases : [];
|
|
1915
|
+
const caseById = new Map(cases.map((c) => [c.caseId, c]));
|
|
1916
|
+
const judgeByCase = new Map();
|
|
1917
|
+
for (const c of cases) {
|
|
1918
|
+
try {
|
|
1919
|
+
judgeByCase.set(c.caseId, readJsonFile(path.join(studyDir, 'judge-only', `${c.caseId}.json`)));
|
|
1920
|
+
} catch (e) {
|
|
1921
|
+
addFinding('judge_case_unreadable', 'critical', c.caseId, e.message);
|
|
1922
|
+
}
|
|
1923
|
+
}
|
|
1924
|
+
if (prereg && hashes) {
|
|
1925
|
+
const check = verifyCanonicalHash(prereg, hashes.preregistrationHash && hashes.preregistrationHash.value);
|
|
1926
|
+
if (!check.ok) {
|
|
1927
|
+
bindingsVerified = false;
|
|
1928
|
+
addFinding('preregistration_hash_drift', 'critical', 'preregistration.json', `recomputed ${check.recomputed.slice(0, 16)}… != recorded ${String(check.recordedHash).slice(0, 16)}…`);
|
|
1929
|
+
}
|
|
1930
|
+
}
|
|
1931
|
+
|
|
1932
|
+
// --- ledgers + attempts (same discipline as v1) ---------------------------
|
|
1933
|
+
const budget = readJsonl(path.join(studyDir, 'budget-ledger.jsonl'));
|
|
1934
|
+
const events = readJsonl(path.join(studyDir, 'events.jsonl'));
|
|
1935
|
+
const adjudicationsLedger = readJsonl(path.join(studyDir, 'adjudications.jsonl'));
|
|
1936
|
+
for (const [name, parsed] of [['budget-ledger.jsonl', budget], ['events.jsonl', events], ['adjudications.jsonl', adjudicationsLedger]]) {
|
|
1937
|
+
if (parsed.malformedTail) addFinding('ledger_malformed_tail', 'critical', name, 'torn final line — downstream counts unreliable');
|
|
1938
|
+
}
|
|
1939
|
+
const attemptFiles = collectAttemptFiles(studyDir);
|
|
1940
|
+
const attemptsByUnitKey = new Map();
|
|
1941
|
+
for (const af of attemptFiles) {
|
|
1942
|
+
let parsed;
|
|
1943
|
+
try {
|
|
1944
|
+
parsed = readJsonFile(af.abs);
|
|
1945
|
+
} catch (e) {
|
|
1946
|
+
addFinding('attempt_file_unparseable', 'critical', af.rel, e.message);
|
|
1947
|
+
continue;
|
|
1948
|
+
}
|
|
1949
|
+
const unit = parsed && parsed.unit;
|
|
1950
|
+
if (!unit || !unit.caseId || !unit.armId) {
|
|
1951
|
+
addFinding('attempt_file_unit_missing', 'critical', af.rel, 'no usable unit identity');
|
|
1952
|
+
continue;
|
|
1953
|
+
}
|
|
1954
|
+
const key = unitKey(unit);
|
|
1955
|
+
if (attemptsByUnitKey.has(key)) addFinding('attempt_file_duplicate_unit', 'critical', key, `multiple attempt files claim ${key}`);
|
|
1956
|
+
attemptsByUnitKey.set(key, parsed);
|
|
1957
|
+
}
|
|
1958
|
+
const rowsByUnitKey = new Map();
|
|
1959
|
+
for (const row of budget.entries) {
|
|
1960
|
+
if (!row || !row.unit || !row.unit.caseId) continue;
|
|
1961
|
+
const key = unitKey(row.unit);
|
|
1962
|
+
if (!rowsByUnitKey.has(key)) rowsByUnitKey.set(key, { reserve: false, prepared: false, results: [] });
|
|
1963
|
+
const g = rowsByUnitKey.get(key);
|
|
1964
|
+
if (row.type === 'RESERVE') g.reserve = true;
|
|
1965
|
+
if (row.type === 'ATTEMPT_PREPARED') g.prepared = true;
|
|
1966
|
+
if (row.type === 'ATTEMPT_RESULT') g.results.push(row.result);
|
|
1967
|
+
}
|
|
1968
|
+
for (const [key, g] of rowsByUnitKey) {
|
|
1969
|
+
if (g.results.length > 1) addFinding('duplicate_settlement', 'critical', key, `attempt settled ${g.results.length} times`);
|
|
1970
|
+
const file = attemptsByUnitKey.get(key);
|
|
1971
|
+
if (g.results.length > 0 && !file) addFinding('ledger_result_without_attempt_file', 'critical', key, 'settled without an attempt file');
|
|
1972
|
+
if (g.results.length > 0 && file && g.results[g.results.length - 1] !== file.result) {
|
|
1973
|
+
addFinding('ledger_file_result_conflict', 'critical', key, `ledger says ${g.results[g.results.length - 1]}, file says ${String(file.result)}`);
|
|
1974
|
+
}
|
|
1975
|
+
}
|
|
1976
|
+
for (const [key, file] of attemptsByUnitKey) {
|
|
1977
|
+
const g = rowsByUnitKey.get(key);
|
|
1978
|
+
if (!g || !g.reserve) addFinding('orphan_attempt_file', 'major', key, 'no RESERVE row');
|
|
1979
|
+
}
|
|
1980
|
+
|
|
1981
|
+
// --- authorization (real modes) --------------------------------------------
|
|
1982
|
+
const realModeAttempts = [...attemptsByUnitKey.values()].filter((parsed) => parsed.mode && parsed.mode !== 'offline-stub').length;
|
|
1983
|
+
const authStatus = authRecord && typeof authRecord.status === 'string' ? authRecord.status : null;
|
|
1984
|
+
const authBindingsMatch = authRecord && frozen
|
|
1985
|
+
? authRecord.suiteHash === frozen.suiteHash && authRecord.mappingHash === frozen.mappingHash && authRecord.graderHash === frozen.graderHash
|
|
1986
|
+
: null;
|
|
1987
|
+
const authorizationValid = authStatus === 'APPROVED' && authBindingsMatch === true && authRecord.revoked !== true;
|
|
1988
|
+
if (realModeAttempts > 0 && !authorizationValid) {
|
|
1989
|
+
addFinding('real_execution_without_valid_authorization', 'critical', 'run-authorization.json', `${realModeAttempts} real-mode attempt(s) without a valid bound authorization`);
|
|
1990
|
+
}
|
|
1991
|
+
|
|
1992
|
+
// --- plan units -------------------------------------------------------------
|
|
1993
|
+
const preregArms = prereg && Array.isArray(prereg.arms) ? prereg.arms.map((a) => a.armId).sort() : [];
|
|
1994
|
+
const plannedRepetitions = prereg && prereg.repetitions && Number(prereg.repetitions.planned) > 0
|
|
1995
|
+
? Array.from({ length: Number(prereg.repetitions.planned) }, (_, i) => String(i + 1)) : [];
|
|
1996
|
+
const controlArm = (prereg && prereg.controlArm) || 'A';
|
|
1997
|
+
const treatmentArm = (prereg && prereg.treatmentArm) || 'B';
|
|
1998
|
+
const planUnits = [];
|
|
1999
|
+
if (preregArms.length > 0 && plannedRepetitions.length > 0 && cases.length > 0) {
|
|
2000
|
+
for (const c of [...cases].sort((a, b) => a.caseId.localeCompare(b.caseId))) {
|
|
2001
|
+
for (const armId of preregArms) {
|
|
2002
|
+
for (const rep of plannedRepetitions) planUnits.push({ caseId: c.caseId, armId, repetitionId: rep });
|
|
2003
|
+
}
|
|
2004
|
+
}
|
|
2005
|
+
} else {
|
|
2006
|
+
addFinding('denominator_unverifiable', 'critical', 'preregistration.json', 'arms/repetitions/cases unavailable — the planned denominator cannot be computed');
|
|
2007
|
+
}
|
|
2008
|
+
|
|
2009
|
+
// --- REPLAY-based per-attempt rescoring (the v2 core) ----------------------
|
|
2010
|
+
const ledgerForVerify = createChallengeLedger(studyDir);
|
|
2011
|
+
const replayedAdjudications = new Map(); // unitKey -> replayed adjudication
|
|
2012
|
+
for (const [key, parsed] of attemptsByUnitKey) {
|
|
2013
|
+
if (parsed.result !== 'SUCCEEDED' || !parsed.adjudication || !parsed.trace || !judgeByCase.has(parsed.unit.caseId)) continue;
|
|
2014
|
+
try {
|
|
2015
|
+
const judge = judgeByCase.get(parsed.unit.caseId);
|
|
2016
|
+
replayedAdjudications.set(key, replayChallengeExecution({
|
|
2017
|
+
caseDef: judge,
|
|
2018
|
+
execution: { caseHash: hashOfCanonical(judge), trace: parsed.trace, adjudication: parsed.adjudication },
|
|
2019
|
+
implementation,
|
|
2020
|
+
studyId: frozen ? frozen.studyId : 'UNKNOWN',
|
|
2021
|
+
requirePersistedRunState: false,
|
|
2022
|
+
}));
|
|
2023
|
+
} catch (e) {
|
|
2024
|
+
addFinding('replay_mismatch', 'critical', key, `full-trajectory rescoring refused: ${e.message}`);
|
|
2025
|
+
}
|
|
2026
|
+
}
|
|
2027
|
+
|
|
2028
|
+
// --- plan-unit rollups ------------------------------------------------------
|
|
2029
|
+
const goldReview = readGoldReviewStatuses(studyDir);
|
|
2030
|
+
const goldStatusOf = (caseId) => goldReview.statuses[caseId] ?? 'GOLD_REVIEW_PENDING';
|
|
2031
|
+
const unitRollups = [];
|
|
2032
|
+
for (const pu of planUnits) {
|
|
2033
|
+
const caseDef = caseById.get(pu.caseId);
|
|
2034
|
+
const attemptIds = [...attemptsByUnitKey.keys()]
|
|
2035
|
+
.filter((key) => key.startsWith(`${pu.caseId}/${pu.armId}/r${String(pu.repetitionId)}/`))
|
|
2036
|
+
.map((key) => key.split('/')[3]).sort();
|
|
2037
|
+
const states = attemptIds.map((attemptId) => ({
|
|
2038
|
+
attemptId,
|
|
2039
|
+
state: ledgerForVerify.classifyUnit({ caseId: pu.caseId, armId: pu.armId, repetitionId: pu.repetitionId, attemptId }).state,
|
|
2040
|
+
}));
|
|
2041
|
+
const succeeded = states.filter((s) => s.state === 'SUCCEEDED');
|
|
2042
|
+
const failed = states.filter((s) => s.state === 'FAILED_KNOWN');
|
|
2043
|
+
const unknown = states.filter((s) => s.state === 'RESULT_UNKNOWN');
|
|
2044
|
+
let currentState = 'not_started';
|
|
2045
|
+
if (succeeded.length > 0) currentState = 'succeeded';
|
|
2046
|
+
else if (failed.length > 0) currentState = 'failed_known';
|
|
2047
|
+
else if (unknown.length > 0) currentState = 'result_unknown';
|
|
2048
|
+
else if (states.some((s) => s.state === 'RESERVED_NOT_PREPARED')) currentState = 'reserved_unsettled';
|
|
2049
|
+
const primary = succeeded.length > 0 ? succeeded[succeeded.length - 1]
|
|
2050
|
+
: failed.length > 0 ? failed[failed.length - 1]
|
|
2051
|
+
: unknown.length > 0 ? unknown[unknown.length - 1] : null;
|
|
2052
|
+
const primaryKey = primary ? unitKey({ caseId: pu.caseId, armId: pu.armId, repetitionId: pu.repetitionId, attemptId: primary.attemptId }) : null;
|
|
2053
|
+
const replayed = primaryKey ? replayedAdjudications.get(primaryKey) ?? null : null;
|
|
2054
|
+
unitRollups.push({
|
|
2055
|
+
unit: planUnitKey(pu.caseId, pu.armId, pu.repetitionId),
|
|
2056
|
+
caseId: pu.caseId,
|
|
2057
|
+
armId: pu.armId,
|
|
2058
|
+
repetitionId: pu.repetitionId,
|
|
2059
|
+
family: caseDef ? caseDef.family : null,
|
|
2060
|
+
principleApplicable: caseDef ? caseDef.principleApplicable === true : null,
|
|
2061
|
+
currentState,
|
|
2062
|
+
primaryAttempt: primary ? primary.attemptId : null,
|
|
2063
|
+
replayRescored: replayed !== null,
|
|
2064
|
+
f: replayed ? failureMechanismFlag(replayed) : 'UNKNOWN',
|
|
2065
|
+
outcome: replayed ? {
|
|
2066
|
+
principleConformance: replayed.dimensions.principleConformance.status,
|
|
2067
|
+
evidenceAlignment: replayed.dimensions.evidenceAlignment.status,
|
|
2068
|
+
objectiveTaskOutcome: replayed.dimensions.objectiveTaskOutcome,
|
|
2069
|
+
negativeTransfer: replayed.negativeTransfer,
|
|
2070
|
+
} : null,
|
|
2071
|
+
goldReview: goldStatusOf(pu.caseId),
|
|
2072
|
+
});
|
|
2073
|
+
}
|
|
2074
|
+
unitRollups.sort((a, b) => a.unit.localeCompare(b.unit));
|
|
2075
|
+
const byCategory = (cat) => unitRollups.filter((u) => u.currentState === cat);
|
|
2076
|
+
const executedUnits = unitRollups.filter((u) => u.currentState !== 'not_started').length;
|
|
2077
|
+
const primaryValid = (u) => u.currentState === 'succeeded' && u.replayRescored;
|
|
2078
|
+
const validUnits = unitRollups.filter(primaryValid);
|
|
2079
|
+
const pairedValid = (() => {
|
|
2080
|
+
const byCaseRep = new Map();
|
|
2081
|
+
for (const u of validUnits) {
|
|
2082
|
+
const k = `${u.caseId}/r${String(u.repetitionId)}`;
|
|
2083
|
+
if (!byCaseRep.has(k)) byCaseRep.set(k, new Set());
|
|
2084
|
+
byCaseRep.get(k).add(u.armId);
|
|
2085
|
+
}
|
|
2086
|
+
let pairs = 0;
|
|
2087
|
+
for (const arms of byCaseRep.values()) if (arms.has(controlArm) && arms.has(treatmentArm)) pairs += 1;
|
|
2088
|
+
return pairs;
|
|
2089
|
+
})();
|
|
2090
|
+
|
|
2091
|
+
// --- §6 statistics over the replay-derived F flags --------------------------
|
|
2092
|
+
// P1-3 (PR #2239 review): statistics are built from the PREREGISTERED plan —
|
|
2093
|
+
// one row per planned (caseId, repetitionId) — never reverse-derived from
|
|
2094
|
+
// the successfully executed records. Missing/invalid sides stay in their
|
|
2095
|
+
// pairs with explicit statuses and bound every downstream rate.
|
|
2096
|
+
const caseIds = [...new Set(cases.map((c) => c.caseId))].sort();
|
|
2097
|
+
const repetitionsPlanned = plannedRepetitions.length;
|
|
2098
|
+
const controlUnits = unitRollups.filter((u) => u.armId === controlArm);
|
|
2099
|
+
const bfrr = computeBFRR(
|
|
2100
|
+
controlUnits.map((u) => ({ unitId: u.unit, f: u.f, executionValid: primaryValid(u) })),
|
|
2101
|
+
{ plannedControlUnits: planUnits.filter((u) => u.armId === controlArm).length || controlUnits.length },
|
|
2102
|
+
);
|
|
2103
|
+
const rollupsByUnit = new Map(unitRollups.map((u) => [u.unit, u]));
|
|
2104
|
+
const plannedPairSet = buildPlannedPairs({ planUnits, rollupsByUnit, controlArm, treatmentArm });
|
|
2105
|
+
const dedupPairs = plannedPairSet.pairs;
|
|
2106
|
+
const pcr = computePCR(dedupPairs);
|
|
2107
|
+
const fir = computeFIR(dedupPairs);
|
|
2108
|
+
const delta = computePairedEffect(dedupPairs);
|
|
2109
|
+
const noiseAdjudications = readNoiseAdjudications(studyDir);
|
|
2110
|
+
const mechanismBinding = resolveMechanismBinding(studyDir);
|
|
2111
|
+
const vbdrCases = caseIds.map((caseId) => {
|
|
2112
|
+
const traceHashOf = (armId) => {
|
|
2113
|
+
const u = unitRollups.find((row) => row.caseId === caseId && row.armId === armId && primaryValid(row));
|
|
2114
|
+
if (!u) return null;
|
|
2115
|
+
const attempt = attemptsByUnitKey.get(`${u.caseId}/${u.armId}/r${String(u.repetitionId)}/${u.primaryAttempt}`);
|
|
2116
|
+
return attempt && attempt.trace ? hashOfCanonical(attempt.trace) : null;
|
|
2117
|
+
};
|
|
2118
|
+
const record = noiseAdjudications.records[caseId] ?? null;
|
|
2119
|
+
return projectVBDRCase({
|
|
2120
|
+
caseId,
|
|
2121
|
+
plannedRepetitions,
|
|
2122
|
+
rollupsByUnit,
|
|
2123
|
+
controlArm,
|
|
2124
|
+
treatmentArm,
|
|
2125
|
+
goldReview: goldStatusOf(caseId) === 'GOLD_REVIEW_APPROVED' ? 'APPROVED' : goldStatusOf(caseId) === 'GOLD_REVIEW_REJECTED' ? 'REJECTED' : 'PENDING',
|
|
2126
|
+
mechanismBinding,
|
|
2127
|
+
noiseAdjudication: record,
|
|
2128
|
+
primaryTraceHashes: { control: traceHashOf(controlArm), treatment: traceHashOf(treatmentArm) },
|
|
2129
|
+
});
|
|
2130
|
+
});
|
|
2131
|
+
const vbdr = computeVBDR(vbdrCases);
|
|
2132
|
+
|
|
2133
|
+
// --- v0.9 examiner block (OPTIONAL only for genuine pre-v0.9 studies).
|
|
2134
|
+
// A study with examiner-manifest.json always gets the block; a study that
|
|
2135
|
+
// still carries v0.9 identity markers but deleted the manifest gets a
|
|
2136
|
+
// critical finding instead of silently losing the block (P1-3.1). Historical
|
|
2137
|
+
// studies without any v0.9 marker produce byte-identical reports; the block
|
|
2138
|
+
// adds the dual-mode contract facts: ITT over applicable planned pairs,
|
|
2139
|
+
// Treatment Fidelity from the runner-produced injection evidence chains, and
|
|
2140
|
+
// the frozen randomization record. All inputs are verified (fail-closed) and
|
|
2141
|
+
// every count is recomputable from the raw artifacts listed.
|
|
2142
|
+
const examiner = buildExaminerBlockMaybe(studyDir, {
|
|
2143
|
+
addFinding,
|
|
2144
|
+
caseById,
|
|
2145
|
+
planUnits,
|
|
2146
|
+
plannedPairSet,
|
|
2147
|
+
rollupsByUnit,
|
|
2148
|
+
controlArm,
|
|
2149
|
+
treatmentArm,
|
|
2150
|
+
replayedAdjudications,
|
|
2151
|
+
});
|
|
2152
|
+
|
|
2153
|
+
// --- dimensions --------------------------------------------------------------
|
|
2154
|
+
const arms = preregArms;
|
|
2155
|
+
const countBy = (list, fn) => list.reduce((acc, item) => { const key = fn(item); acc[key] = (acc[key] || 0) + 1; return acc; }, {});
|
|
2156
|
+
const perArmOutcome = (field) => Object.fromEntries(arms.map((armId) => {
|
|
2157
|
+
const us = validUnits.filter((u) => u.armId === armId);
|
|
2158
|
+
const counts = countBy(us, (u) => (field === 'negativeTransfer' ? String(u.outcome.negativeTransfer) : u.outcome[field]));
|
|
2159
|
+
return [armId, counts];
|
|
2160
|
+
}));
|
|
2161
|
+
const replayByUnit = new Map();
|
|
2162
|
+
for (const u of unitRollups) {
|
|
2163
|
+
if (!u.replayRescored) continue;
|
|
2164
|
+
const primaryKey = `${u.caseId}/${u.armId}/r${String(u.repetitionId)}/${u.primaryAttempt}`;
|
|
2165
|
+
if (attemptsByUnitKey.has(primaryKey)) replayByUnit.set(u.unit, replayedAdjudications.get(primaryKey));
|
|
2166
|
+
}
|
|
2167
|
+
const ntKinds = ['unnecessaryRefusal', 'alternativePathBlocking', 'overChecking', 'repeatedConfirmation', 'inapplicableTrigger', 'unsupportedAction', 'additionalOperationalCost'];
|
|
2168
|
+
const negativeTransferTyped = Object.fromEntries(arms.map((armId) => {
|
|
2169
|
+
const rolls = unitRollups.filter((u) => u.armId === armId && u.replayRescored).map((u) => replayByUnit.get(u.unit));
|
|
2170
|
+
const perType = {};
|
|
2171
|
+
for (const type of ntKinds) {
|
|
2172
|
+
perType[type] = {
|
|
2173
|
+
detectedUnits: rolls.filter((adj) => adj?.dimensions?.negativeTransfer?.types?.[type]?.status === 'DETECTED').length,
|
|
2174
|
+
};
|
|
2175
|
+
}
|
|
2176
|
+
return [armId, { perType, units: rolls.length, note: 'typed indicators counted per unit, never summed into a score' }];
|
|
2177
|
+
}));
|
|
2178
|
+
|
|
2179
|
+
const costPerArm = Object.fromEntries(arms.map((armId) => {
|
|
2180
|
+
const rolls = unitRollups.filter((u) => u.armId === armId && u.replayRescored).map((u) => replayByUnit.get(u.unit));
|
|
2181
|
+
const sum = (field) => rolls.reduce((acc, adj) => acc + (Number.isFinite(adj?.dimensions?.operationalCost?.[field]) ? adj.dimensions.operationalCost[field] : 0), 0);
|
|
2182
|
+
return [armId, {
|
|
2183
|
+
units: rolls.length,
|
|
2184
|
+
observationCalls: sum('observationCalls'),
|
|
2185
|
+
toolCalls: sum('toolCalls'),
|
|
2186
|
+
budgetUsed: sum('budgetUsed'),
|
|
2187
|
+
timeUnits: sum('timeUnits'),
|
|
2188
|
+
additionalChecks: sum('additionalChecks'),
|
|
2189
|
+
repeatedObservationCount: sum('repeatedObservationCount'),
|
|
2190
|
+
humanInterventionRequests: sum('humanInterventionRequests'),
|
|
2191
|
+
promptTokens: 'UNKNOWN',
|
|
2192
|
+
completionTokens: 'UNKNOWN',
|
|
2193
|
+
totalTokens: 'UNKNOWN',
|
|
2194
|
+
wallClockTime: 'UNKNOWN',
|
|
2195
|
+
failedRequests: 'UNKNOWN',
|
|
2196
|
+
note: 'offline deterministic metering; real-transport cost fields stay UNKNOWN, never estimated',
|
|
2197
|
+
}];
|
|
2198
|
+
}));
|
|
2199
|
+
|
|
2200
|
+
const families = manifest && manifest.familyCounts ? Object.keys(manifest.familyCounts).sort() : [];
|
|
2201
|
+
const coverage = families.map((family) => {
|
|
2202
|
+
const familyCases = caseIds.filter((caseId) => caseById.get(caseId)?.family === family);
|
|
2203
|
+
return {
|
|
2204
|
+
family,
|
|
2205
|
+
casesInSuite: familyCases.length,
|
|
2206
|
+
casesWithValidExecution: familyCases.filter((caseId) => unitRollups.some((u) => u.caseId === caseId && primaryValid(u))).length,
|
|
2207
|
+
};
|
|
2208
|
+
});
|
|
2209
|
+
|
|
2210
|
+
const dimensions = {
|
|
2211
|
+
protocolEvidenceQuality: {
|
|
2212
|
+
runCount: events.entries.filter((e) => e && e.type === 'RUN_STARTED').length,
|
|
2213
|
+
assignedUnits: planUnits.length,
|
|
2214
|
+
executedUnits,
|
|
2215
|
+
currentUnitStateCounts: {
|
|
2216
|
+
succeeded: byCategory('succeeded').length,
|
|
2217
|
+
failed_known: byCategory('failed_known').length,
|
|
2218
|
+
result_unknown: byCategory('result_unknown').length,
|
|
2219
|
+
reserved_unsettled: byCategory('reserved_unsettled').length,
|
|
2220
|
+
not_started: byCategory('not_started').length,
|
|
2221
|
+
},
|
|
2222
|
+
validExecutionUnits: validUnits.length,
|
|
2223
|
+
validPairedUnits: pairedValid,
|
|
2224
|
+
realModeAttempts,
|
|
2225
|
+
authorizationValid,
|
|
2226
|
+
refusalReasons: Object.fromEntries(Object.entries(events.entries.filter((e) => e && e.type === 'UNIT_REFUSED' && e.reason)
|
|
2227
|
+
.reduce((acc, e) => { acc[e.reason] = (acc[e.reason] || 0) + 1; return acc; }, {})).sort(([a], [b]) => a.localeCompare(b))),
|
|
2228
|
+
},
|
|
2229
|
+
principleConformance: {
|
|
2230
|
+
perArm: Object.fromEntries(arms.map((armId) => [armId, countBy(validUnits.filter((u) => u.armId === armId), (u) => u.outcome.principleConformance)])),
|
|
2231
|
+
note: 'independent dimension from frozen Mapping behavior predicates; NOT_APPLICABLE is never PASS and never a denominator shrink',
|
|
2232
|
+
},
|
|
2233
|
+
evidenceAlignment: {
|
|
2234
|
+
perArm: perArmOutcome('evidenceAlignment'),
|
|
2235
|
+
fDerivationRule: "study-level F projection: Evidence Alignment PASS→F=0, FAIL→F=1 (Pain #44 mechanism), otherwise UNKNOWN — recorded so all §6 statistics are recomputable",
|
|
2236
|
+
},
|
|
2237
|
+
objectiveTaskOutcome: { perArm: perArmOutcome('objectiveTaskOutcome') },
|
|
2238
|
+
negativeTransfer: negativeTransferTyped,
|
|
2239
|
+
operationalCost: costPerArm,
|
|
2240
|
+
behavioralDiscrimination: {
|
|
2241
|
+
controlArm,
|
|
2242
|
+
treatmentArm,
|
|
2243
|
+
bfrr,
|
|
2244
|
+
pcr,
|
|
2245
|
+
fir,
|
|
2246
|
+
deltaBenefit: delta,
|
|
2247
|
+
vbdr,
|
|
2248
|
+
note: 'descriptive paired statistics over replay-derived F flags; small samples get no single-p-value significance claims',
|
|
2249
|
+
},
|
|
2250
|
+
causalAttribution: {
|
|
2251
|
+
...causalAttributionStatus({ isolationVerified: false, randomizationDeclared: false, repetitionsMet: false, preregisteredDirection: null, observedDirection: null }),
|
|
2252
|
+
contractNote: 'offline-stub studies carry no real A/B isolation evidence, so attribution stays NOT_ESTIMABLE; the ladder in challenge-statistics.mjs governs future authorized studies',
|
|
2253
|
+
},
|
|
2254
|
+
coverage: {
|
|
2255
|
+
families: coverage,
|
|
2256
|
+
goldReviewCounts: {
|
|
2257
|
+
approved: caseIds.filter((caseId) => goldStatusOf(caseId) === 'GOLD_REVIEW_APPROVED').length,
|
|
2258
|
+
rejected: caseIds.filter((caseId) => goldStatusOf(caseId) === 'GOLD_REVIEW_REJECTED').length,
|
|
2259
|
+
pending: caseIds.filter((caseId) => goldStatusOf(caseId) !== 'GOLD_REVIEW_APPROVED' && goldStatusOf(caseId) !== 'GOLD_REVIEW_REJECTED').length,
|
|
2260
|
+
},
|
|
2261
|
+
goldReviewSource: goldReview.note,
|
|
2262
|
+
},
|
|
2263
|
+
};
|
|
2264
|
+
|
|
2265
|
+
// --- fixed denominator table + status ----------------------------------------
|
|
2266
|
+
const denominator = {
|
|
2267
|
+
planned: planUnits.length,
|
|
2268
|
+
// P2-3 (PR #2239 review): authorized is computed from the ACTUAL
|
|
2269
|
+
// authorization record, never from the plan total. Offline-stub studies
|
|
2270
|
+
// bypass the Stage R gate (no real execution), so authorized is honestly
|
|
2271
|
+
// null — a stub study must not fabricate a real-authorization count.
|
|
2272
|
+
authorized: realModeAttempts > 0
|
|
2273
|
+
? (authorizationValid ? (authRecord.cases?.length ?? planUnits.length) * (authRecord.arms?.length ?? 1) * (authRecord.repetitions?.length ?? plannedRepetitions.length) : 0)
|
|
2274
|
+
: null,
|
|
2275
|
+
authorizedNote: realModeAttempts > 0
|
|
2276
|
+
? (authorizationValid ? 'authorized units derived from the bound run-authorization record (cases × arms × repetitions)' : 'real-mode attempts exist but the authorization is invalid — 0 authorized units')
|
|
2277
|
+
: 'offline-stub study: the Stage R authorization gate does not apply (no real execution), so authorized is NOT_APPLICABLE rather than plan-sized',
|
|
2278
|
+
attempted: unitRollups.filter((u) => u.currentState !== 'not_started').length,
|
|
2279
|
+
completed: unitRollups.filter((u) => ['succeeded', 'failed_known', 'result_unknown'].includes(u.currentState)).length,
|
|
2280
|
+
valid: validUnits.length,
|
|
2281
|
+
pairedValid,
|
|
2282
|
+
unknown: byCategory('result_unknown').length + byCategory('reserved_unsettled').length,
|
|
2283
|
+
executionInvalid: byCategory('failed_known').length,
|
|
2284
|
+
reviewApproved: caseIds.filter((caseId) => goldStatusOf(caseId) === 'GOLD_REVIEW_APPROVED').length,
|
|
2285
|
+
reviewRejected: caseIds.filter((caseId) => goldStatusOf(caseId) === 'GOLD_REVIEW_REJECTED').length,
|
|
2286
|
+
reviewPending: caseIds.filter((caseId) => goldStatusOf(caseId) !== 'GOLD_REVIEW_APPROVED' && goldStatusOf(caseId) !== 'GOLD_REVIEW_REJECTED').length,
|
|
2287
|
+
rule: 'planned ≥ authorized ⊇ attempted ⊇ completed ⊇ valid; the authorization subset never shrinks the preregistered denominator; unknown/invalid units are never deleted; REJECTED and PENDING review counts are separate',
|
|
2288
|
+
};
|
|
2289
|
+
|
|
2290
|
+
const criticalCount = findings.filter((f) => f.severity === 'critical').length;
|
|
2291
|
+
let status;
|
|
2292
|
+
if (criticalCount > 0 || !bindingsVerified) status = REPORT_STATUS_INVALID;
|
|
2293
|
+
else if (planUnits.length === 0) status = REPORT_STATUS_INVALID;
|
|
2294
|
+
else if (executedUnits === 0) status = REPORT_STATUS_PENDING;
|
|
2295
|
+
else if (byCategory('not_started').length > 0 || byCategory('reserved_unsettled').length > 0) status = REPORT_STATUS_PENDING;
|
|
2296
|
+
else if (denominator.reviewPending > 0) status = REPORT_STATUS_PENDING;
|
|
2297
|
+
else if (validUnits.length === 0) status = REPORT_STATUS_INCONCLUSIVE;
|
|
2298
|
+
else status = REPORT_STATUS_READY;
|
|
2299
|
+
findings.sort((a, b) => (SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity]) || a.id.localeCompare(b.id));
|
|
2300
|
+
|
|
2301
|
+
return {
|
|
2302
|
+
schema: BEHAVIOR_REPORT_V2_SCHEMA,
|
|
2303
|
+
studyId: frozen ? frozen.studyId : null,
|
|
2304
|
+
status,
|
|
2305
|
+
generatedAt: normalizeNow(now),
|
|
2306
|
+
scope: {
|
|
2307
|
+
arms,
|
|
2308
|
+
controlArm,
|
|
2309
|
+
treatmentArm,
|
|
2310
|
+
repetitions: plannedRepetitions,
|
|
2311
|
+
executionMode: realModeAttempts > 0 ? 'real' : 'offline-stub',
|
|
2312
|
+
denominator,
|
|
2313
|
+
},
|
|
2314
|
+
bindings: {
|
|
2315
|
+
verified: bindingsVerified && criticalCount === 0,
|
|
2316
|
+
suiteHash: frozen ? frozen.suiteHash : null,
|
|
2317
|
+
mappingHash: frozen ? frozen.mappingHash : null,
|
|
2318
|
+
graderHash: frozen ? frozen.graderHash : null,
|
|
2319
|
+
principleContentHash: frozen ? frozen.principleContentHash : null,
|
|
2320
|
+
implementationHash: hashOfCanonical(implementation),
|
|
2321
|
+
nodeContract: implementation.node,
|
|
2322
|
+
replay: 'full-trajectory rescoring from initial state through every action/observation; stored adjudications are compared against the recomputation, never copied',
|
|
2323
|
+
},
|
|
2324
|
+
dimensions,
|
|
2325
|
+
...(examiner ? { examiner } : {}),
|
|
2326
|
+
planUnits: unitRollups,
|
|
2327
|
+
integrity: {
|
|
2328
|
+
findings,
|
|
2329
|
+
criticalCount,
|
|
2330
|
+
majorCount: findings.filter((f) => f.severity === 'major').length,
|
|
2331
|
+
infoCount: findings.filter((f) => f.severity === 'info').length,
|
|
2332
|
+
},
|
|
2333
|
+
limitations: [
|
|
2334
|
+
'offline deterministic study: model/host identity and real cost fields are UNKNOWN, never estimated',
|
|
2335
|
+
'descriptive statistics only — no causal attribution without authorized paired isolation evidence',
|
|
2336
|
+
'VBDR cannot be ESTIMABLE until independent Gold review approves the cases',
|
|
2337
|
+
],
|
|
2338
|
+
};
|
|
2339
|
+
}
|
|
2340
|
+
|
|
2341
|
+
export function renderBehaviorReportV2Json(report) {
|
|
2342
|
+
return `${JSON.stringify(report, null, 2)}\n`;
|
|
2343
|
+
}
|
|
2344
|
+
|
|
2345
|
+
export function renderBehaviorReportV2Markdown(report) {
|
|
2346
|
+
const lines = [];
|
|
2347
|
+
const push = (...ls) => lines.push(...ls);
|
|
2348
|
+
push(`# Behavior Report V2 — ${report.studyId ?? '(studyId unknown)'}`);
|
|
2349
|
+
push('');
|
|
2350
|
+
push(`- schema: \`${report.schema}\``);
|
|
2351
|
+
push(`- status: **${report.status}**`);
|
|
2352
|
+
push(`- generatedAt: ${report.generatedAt ?? '(not provided)'}`);
|
|
2353
|
+
push(`- execution mode: ${report.scope.executionMode}; arms ${report.scope.arms.join('/')} (control ${report.scope.controlArm}, treatment ${report.scope.treatmentArm})`);
|
|
2354
|
+
push('');
|
|
2355
|
+
push('## Fixed denominators');
|
|
2356
|
+
push('');
|
|
2357
|
+
const d = report.scope.denominator;
|
|
2358
|
+
push('| planned | authorized | attempted | completed | valid | paired valid | unknown | execution invalid | review pending |');
|
|
2359
|
+
push('|---|---|---|---|---|---|---|---|---|');
|
|
2360
|
+
push(`| ${d.planned} | ${d.authorized} | ${d.attempted} | ${d.completed} | ${d.valid} | ${d.pairedValid} | ${d.unknown} | ${d.executionInvalid} | ${d.reviewPending} |`);
|
|
2361
|
+
push('');
|
|
2362
|
+
push('## Dimensions');
|
|
2363
|
+
push('');
|
|
2364
|
+
const dim = report.dimensions;
|
|
2365
|
+
push(`1. protocolEvidenceQuality: valid units ${dim.protocolEvidenceQuality.validExecutionUnits}, valid paired ${dim.protocolEvidenceQuality.validPairedUnits}`);
|
|
2366
|
+
push(`2. principleConformance: ${JSON.stringify(dim.principleConformance.perArm)}`);
|
|
2367
|
+
push(`3. evidenceAlignment: ${JSON.stringify(dim.evidenceAlignment.perArm)}`);
|
|
2368
|
+
push(`4. objectiveTaskOutcome: ${JSON.stringify(dim.objectiveTaskOutcome.perArm)}`);
|
|
2369
|
+
push(`5. negativeTransfer (per type, per arm): ${JSON.stringify(dim.negativeTransfer)}`);
|
|
2370
|
+
push(`6. operationalCost: ${JSON.stringify(dim.operationalCost)}`);
|
|
2371
|
+
const bd = dim.behavioralDiscrimination;
|
|
2372
|
+
push(`7. behavioralDiscrimination: BFRR ${bd.bfrr.rate.status}${bd.bfrr.rate.status === 'ESTIMABLE' ? `=${bd.bfrr.rate.numerator}/${bd.bfrr.rate.denominator}` : ''}; PCR ${bd.pcr.rate.status}; FIR ${bd.fir.rate.status}; Δ ${bd.deltaBenefit.deltaBenefit.status}; VBDR ${bd.vbdr.rate.status}`);
|
|
2373
|
+
push(`8. causalAttribution: ${dim.causalAttribution.status}`);
|
|
2374
|
+
push(`9. coverage: ${JSON.stringify(dim.coverage.families)}; review pending cases ${dim.coverage.reviewPendingCases}`);
|
|
2375
|
+
push('');
|
|
2376
|
+
if (report.examiner) {
|
|
2377
|
+
const ex = report.examiner;
|
|
2378
|
+
push('## Examiner v0.9 (dual-mode contract)');
|
|
2379
|
+
push('');
|
|
2380
|
+
push(`- mode: **${ex.experimentMode}** / intervention: ${ex.interventionType} / target: ${ex.targetPrincipleId}`);
|
|
2381
|
+
push(`- primary endpoint: ${ex.primaryEndpoint ? ex.primaryEndpoint.id : '(none)'} — applicable pairs ${ex.applicability.applicablePairs}, NOT_APPLICABLE ${ex.applicability.notApplicablePairs}`);
|
|
2382
|
+
const itt = ex.itt;
|
|
2383
|
+
if (itt.pointEstimate.status === 'ESTIMABLE') {
|
|
2384
|
+
push(`- ITT: **${itt.pointEstimate.value}** (${itt.pointEstimate.numerator}/${itt.pointEstimate.denominator})`);
|
|
2385
|
+
} else {
|
|
2386
|
+
push(`- ITT: ${itt.pointEstimate.status} — bounds [${itt.lowerBound.status === 'ESTIMABLE' ? itt.lowerBound.value : 'n/a'}, ${itt.upperBound.status === 'ESTIMABLE' ? itt.upperBound.value : 'n/a'}] over ${itt.plannedApplicablePairs} planned applicable pairs (${itt.unknownPairs} with unknown sides)`);
|
|
2387
|
+
}
|
|
2388
|
+
const fid = ex.treatmentFidelity;
|
|
2389
|
+
if (fid.status === 'UNKNOWN') {
|
|
2390
|
+
push(`- treatment fidelity: UNKNOWN — ${fid.actualInjectionRate.reason}`);
|
|
2391
|
+
} else {
|
|
2392
|
+
push(`- treatment fidelity: CONTEXT_PRESENT+ ${fid.contextPresent}/${fid.plannedTreatmentUnits}, RUNTIME_LOADED ${fid.runtimeLoaded}, control contamination ${fid.controlContamination.units}`);
|
|
2393
|
+
}
|
|
2394
|
+
push(`- randomization: ${ex.randomization ? `${ex.randomization.strategy} seed frozen, verified=${ex.randomization.verified}` : '(record missing)'}`);
|
|
2395
|
+
push('');
|
|
2396
|
+
}
|
|
2397
|
+
push('## Integrity findings');
|
|
2398
|
+
push('');
|
|
2399
|
+
if (report.integrity.findings.length === 0) push('(none)');
|
|
2400
|
+
else {
|
|
2401
|
+
push('| severity | id | subject | detail |');
|
|
2402
|
+
push('|---|---|---|---|');
|
|
2403
|
+
for (const f of report.integrity.findings) push(`| ${f.severity} | ${f.id} | ${String(f.subject).replaceAll('|', '/')} | ${String(f.detail).replaceAll('|', '/').slice(0, 160)} |`);
|
|
2404
|
+
}
|
|
2405
|
+
push('');
|
|
2406
|
+
push('## Limitations');
|
|
2407
|
+
push('');
|
|
2408
|
+
for (const l of report.limitations) push(`- ${l}`);
|
|
2409
|
+
push('');
|
|
2410
|
+
return lines.join('\n');
|
|
2411
|
+
}
|
|
2412
|
+
|
|
2413
|
+
/**
|
|
2414
|
+
* Write the §10 Report V2 artifact.
|
|
2415
|
+
*
|
|
2416
|
+
* `canWrite` (optional) is the caller's write-permission gate: it runs BEFORE
|
|
2417
|
+
* lock acquisition and BEFORE any generation, so a read-only study is refused
|
|
2418
|
+
* without a report being computed, a lock being taken, or an event appended.
|
|
2419
|
+
* `REPORT_READY_V2` is an idempotency signal only — never a permission check.
|
|
2420
|
+
*/
|
|
2421
|
+
export function writeBehaviorReportV2(studyDir, { now, owner = 'behavior-report-generator-v2', canWrite = null } = {}) {
|
|
2422
|
+
if (now === undefined || now === null) throw new Error('writeBehaviorReportV2: now required (ISO-8601)');
|
|
2423
|
+
normalizeNow(now);
|
|
2424
|
+
assertNotFrozenStudyArea(studyDir);
|
|
2425
|
+
if (canWrite !== null) {
|
|
2426
|
+
if (typeof canWrite !== 'function') throw new Error('writeBehaviorReportV2: canWrite must be a function returning { ok, reason }');
|
|
2427
|
+
const permission = canWrite(studyDir);
|
|
2428
|
+
if (!permission || permission.ok !== true) throw new Error(`REPORT_WRITE_REFUSED: ${permission?.reason ?? 'write permission not granted'}`);
|
|
2429
|
+
}
|
|
2430
|
+
const jsonPath = path.join(studyDir, REPORT_JSON_V2);
|
|
2431
|
+
const mdPath = path.join(studyDir, REPORT_MD_V2);
|
|
2432
|
+
acquireStudyWriteLock(studyDir, { owner });
|
|
2433
|
+
try {
|
|
2434
|
+
const events = readJsonl(path.join(studyDir, 'events.jsonl')).entries;
|
|
2435
|
+
const readyEvents = events.filter((e) => e && e.type === 'REPORT_READY_V2');
|
|
2436
|
+
if (readyEvents.length > 0) {
|
|
2437
|
+
if (!fs.existsSync(jsonPath)) throw new ReportRevisionRequiredError('REPORT_READY_V2 event exists but behavior-report-v2.json is missing');
|
|
2438
|
+
const existingRaw = fs.readFileSync(jsonPath, 'utf8');
|
|
2439
|
+
const existing = JSON.parse(existingRaw);
|
|
2440
|
+
const candidate = buildBehaviorReportV2(studyDir, { now: existing.generatedAt ?? now });
|
|
2441
|
+
if (existingRaw !== renderBehaviorReportV2Json(candidate)) {
|
|
2442
|
+
throw new ReportRevisionRequiredError('v2 recomputation differs from the committed report — create a new study revision instead of overwriting');
|
|
2443
|
+
}
|
|
2444
|
+
return { written: false, alreadyReady: true, status: candidate.status, report: candidate };
|
|
2445
|
+
}
|
|
2446
|
+
const report = buildBehaviorReportV2(studyDir, { now });
|
|
2447
|
+
const json = renderBehaviorReportV2Json(report);
|
|
2448
|
+
atomicWrite(jsonPath, json);
|
|
2449
|
+
atomicWrite(mdPath, renderBehaviorReportV2Markdown(report));
|
|
2450
|
+
let eventAppended = false;
|
|
2451
|
+
if (report.status === REPORT_STATUS_READY) {
|
|
2452
|
+
createChallengeLedger(studyDir).appendEvent({
|
|
2453
|
+
type: 'REPORT_READY_V2',
|
|
2454
|
+
owner,
|
|
2455
|
+
report: { schema: report.schema, status: report.status, generatedAt: report.generatedAt, reportSha256: sha256(json), planned: report.scope.denominator.planned },
|
|
2456
|
+
});
|
|
2457
|
+
eventAppended = true;
|
|
2458
|
+
}
|
|
2459
|
+
return { written: true, alreadyReady: false, status: report.status, eventAppended, report };
|
|
2460
|
+
} finally {
|
|
2461
|
+
try { releaseStudyWriteLock(studyDir, { owner }); } catch { /* stays held; next writer refused loudly */ }
|
|
2462
|
+
}
|
|
2463
|
+
}
|
|
2464
|
+
|
|
2465
|
+
export function verifyBehaviorReportV2(studyDir) {
|
|
2466
|
+
const jsonPath = path.join(studyDir, REPORT_JSON_V2);
|
|
2467
|
+
if (!fs.existsSync(jsonPath)) return { ok: false, reason: 'report_not_generated', checks: [] };
|
|
2468
|
+
const existingRaw = fs.readFileSync(jsonPath, 'utf8');
|
|
2469
|
+
let existing;
|
|
2470
|
+
try {
|
|
2471
|
+
existing = JSON.parse(existingRaw);
|
|
2472
|
+
} catch (e) {
|
|
2473
|
+
return { ok: false, reason: 'report_unparseable', checks: [{ id: 'report_json_parseable', ok: false, detail: e.message }] };
|
|
2474
|
+
}
|
|
2475
|
+
if (!existing.generatedAt) return { ok: false, reason: 'generated_at_missing', checks: [{ id: 'generated_at_available', ok: false, detail: 'no reproducible timestamp' }] };
|
|
2476
|
+
const rebuilt = buildBehaviorReportV2(studyDir, { now: existing.generatedAt });
|
|
2477
|
+
const jsonMatch = existingRaw === renderBehaviorReportV2Json(rebuilt);
|
|
2478
|
+
const readyEvents = readJsonl(path.join(studyDir, 'events.jsonl')).entries.filter((e) => e && e.type === 'REPORT_READY_V2');
|
|
2479
|
+
const checks = [
|
|
2480
|
+
{ id: 'report_json_recomputable', ok: jsonMatch, detail: jsonMatch ? 'byte-identical recomputation' : 'recomputation differs' },
|
|
2481
|
+
{ id: 'ready_event_consistent', ok: existing.status === REPORT_STATUS_READY ? readyEvents.length === 1 && readyEvents[0].report.reportSha256 === sha256(existingRaw) : readyEvents.length === 0, detail: `status ${existing.status} vs ${readyEvents.length} REPORT_READY_V2 event(s)` },
|
|
2482
|
+
];
|
|
2483
|
+
return { ok: checks.every((c) => c.ok), reason: checks.every((c) => c.ok) ? null : 'verification_failed', status: existing.status, checks };
|
|
2484
|
+
}
|