create-principles-disciple 1.145.42 → 1.146.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/_release/product-identity.json +2 -2
- package/console/dist/server/index.js +8 -0
- package/console/dist/server/routes/examiner-executor.d.ts +260 -0
- package/console/dist/server/routes/examiner-executor.js +628 -0
- package/console/dist/server/routes/examiner.d.ts +52 -0
- package/console/dist/server/routes/examiner.js +299 -0
- package/console/dist/ui/App.js +6 -3
- package/console/dist/ui/api.d.ts +138 -0
- package/console/dist/ui/api.js +56 -0
- package/console/dist/ui/components/layout/app-sidebar.d.ts +10 -0
- package/console/dist/ui/components/layout/app-sidebar.js +10 -2
- package/console/dist/ui/i18n/en.json +63 -0
- package/console/dist/ui/i18n/zh-CN.json +63 -0
- package/console/dist/ui/pages/examiner/ExaminerPage.d.ts +15 -0
- package/console/dist/ui/pages/examiner/ExaminerPage.js +158 -0
- package/console/dist/ui/pages/principles/PrincipleDetailPage.js +2 -1
- package/console/dist/ui/utils/runtime-mode.d.ts +11 -0
- package/console/dist/ui/utils/runtime-mode.js +11 -0
- package/console/dist/web/assets/app.css +44 -0
- package/console/dist/web/assets/app.js +33 -31
- package/console/lab/lib/challenge-actor.mjs +394 -0
- package/console/lab/lib/challenge-arms.mjs +102 -0
- package/console/lab/lib/challenge-behavior-report.mjs +2484 -0
- package/console/lab/lib/challenge-compile.mjs +211 -0
- package/console/lab/lib/challenge-freeze.mjs +215 -0
- package/console/lab/lib/challenge-isolation-probe.mjs +550 -0
- package/console/lab/lib/challenge-ledger.mjs +206 -0
- package/console/lab/lib/challenge-openclaw-host.mjs +630 -0
- package/console/lab/lib/challenge-runner.mjs +868 -0
- package/console/lab/lib/challenge-stage-r-gate.mjs +458 -0
- package/console/lab/lib/challenge-statistics.mjs +591 -0
- package/console/lab/lib/claim-evidence-env.mjs +652 -0
- package/console/lab/lib/claim-evidence-scorer.mjs +784 -0
- package/console/lab/lib/examiner-injection-evidence.mjs +367 -0
- package/console/lab/lib/examiner-study.mjs +1203 -0
- package/console/lab/lib/examiner-v07-cases.mjs +187 -0
- package/console/lab/lib/examiner-v2-cases.mjs +61 -0
- package/console/lab/lib/fixture.mjs +382 -0
- package/console/lab/lib/g2.mjs +146 -0
- package/console/lab/lib/host-actor.mjs +987 -0
- package/console/lab/lib/sidecar.mjs +273 -0
- package/package.json +1 -1
- package/pd-cli/package.json +1 -1
- package/release-manager/package.json +1 -1
|
@@ -0,0 +1,868 @@
|
|
|
1
|
+
// SPEC-PD-EXAMINER-001 v0.3/v0.4 — Phase 1B serial research Runner for
|
|
2
|
+
// challenge studies (offline stub execution has no transport/network/DB/clock).
|
|
3
|
+
//
|
|
4
|
+
// Golden path per SPEC §2.3: frozen suite → actor input → legal actions →
|
|
5
|
+
// oracle observations → actor report → ActorTraceV1 → deterministic scoring →
|
|
6
|
+
// research records. Fail-closed before every unit: all frozen hashes are
|
|
7
|
+
// recomputed (suite/mapping/grader/per-case/principle/arms); drift aborts the
|
|
8
|
+
// unit without touching frozen assets.
|
|
9
|
+
//
|
|
10
|
+
// Crash/recovery semantics (SPEC §7.4/§8, offline analog):
|
|
11
|
+
// RESERVE (never returned) → ATTEMPT_PREPARED → execute → ATTEMPT_RESULT.
|
|
12
|
+
// PREPARED without RESULT ⇒ RESULT_UNKNOWN: NO auto-retry of the same
|
|
13
|
+
// attemptId; recovery of that attempt is refused; only a NEW attemptId may
|
|
14
|
+
// be reserved. A simulated crash also leaves the study writer lock held —
|
|
15
|
+
// recovery requires explicit release by the same owner.
|
|
16
|
+
//
|
|
17
|
+
// Stage R boundary (v0.4 §7.2, Stage R readiness): the execution mode is
|
|
18
|
+
// RUNNER-CONTROLLED — declared once via `execution.mode` (one of
|
|
19
|
+
// 'offline-stub' | 'F1-direct-model' | 'F2-real-host'), never derived from
|
|
20
|
+
// actor self-report. An actor's own `offlineStub`/`kind` declaration is only
|
|
21
|
+
// a SECONDARY consistency check: under a stub declaration a non-stub actor
|
|
22
|
+
// refuses, and under a real declaration the actor must carry the matching
|
|
23
|
+
// executionMode + the authorization-bound model identity. Real modes read
|
|
24
|
+
// the AUTHORITATIVE authorization from the study's own
|
|
25
|
+
// run-authorization.json (readStageRAuthorization — no caller-passed
|
|
26
|
+
// in-memory authorization object exists on this path), and the gate runs
|
|
27
|
+
// BEFORE the actor factory is ever invoked: a NOT_AUTHORIZED / expired /
|
|
28
|
+
// revoked / probe-missing / mode-mismatched refusal constructs nothing,
|
|
29
|
+
// reserves nothing and invokes no actor step. Real execution additionally
|
|
30
|
+
// reserves ONE ledger slot per model REQUEST (auth.requestCap — an attempt
|
|
31
|
+
// may span many requests; the attempt cap is not a request budget). Stub
|
|
32
|
+
// execution keeps the historical Phase 1B contract unchanged.
|
|
33
|
+
|
|
34
|
+
import fs from 'node:fs';
|
|
35
|
+
import path from 'node:path';
|
|
36
|
+
import { hashOfCanonical, sha256, challengeImplementationManifest } from './challenge-freeze.mjs';
|
|
37
|
+
import { verifyCanonicalHash } from './challenge-freeze.mjs';
|
|
38
|
+
import { buildClaimEnvironment, newClaimRunState } from './claim-evidence-env.mjs';
|
|
39
|
+
import { adjudicateClaimEvidence } from './claim-evidence-scorer.mjs';
|
|
40
|
+
import { runControlledActorLoop, SimulatedCrashError, perfectPlayActor, createGreedyObserverStub, createNaiveStub } from './challenge-actor.mjs';
|
|
41
|
+
import { acquireStudyWriteLock, releaseStudyWriteLock, createChallengeLedger } from './challenge-ledger.mjs';
|
|
42
|
+
import { buildStudyArms, auditArmIsolation, composeArmPrompt } from './challenge-arms.mjs';
|
|
43
|
+
import { verifyStageRAuthorization, isOfflineStubActor, readStudyPlanHash, readObservedHostEvidence, readStageRAuthorization, readAuthorizationRevocations } from './challenge-stage-r-gate.mjs';
|
|
44
|
+
import { verifyF2ProbeEvidence } from './challenge-isolation-probe.mjs';
|
|
45
|
+
import { collectOfflineStubInjectionEvidence } from './examiner-injection-evidence.mjs';
|
|
46
|
+
import { verifyExaminerManifest, buildHostEffectivePrincipleSet, buildDualModeArmPayloads, verifyExaminerStudyBindings, buildRuleCodePolicy, verifyRuleCodePolicy } from './examiner-study.mjs';
|
|
47
|
+
|
|
48
|
+
export class FrozenStudyDriftError extends Error {
|
|
49
|
+
constructor(details) {
|
|
50
|
+
super(`frozen study drift detected — refusing to run: ${details}`);
|
|
51
|
+
this.name = 'FrozenStudyDriftError';
|
|
52
|
+
this.details = details;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function readJson(file) {
|
|
57
|
+
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Fail-closed recomputation of every frozen binding. Throws
|
|
62
|
+
* FrozenStudyDriftError on any mismatch; returns the verified refs on success.
|
|
63
|
+
*/
|
|
64
|
+
export function verifyFrozenStudy(studyDir) {
|
|
65
|
+
const problems = [];
|
|
66
|
+
const inputManifest = readJson(path.join(studyDir, 'input-manifest.json'));
|
|
67
|
+
const principle = readJson(path.join(studyDir, 'principle-frozen.json'));
|
|
68
|
+
const mapping = readJson(path.join(studyDir, 'pain-mechanism-mapping.json'));
|
|
69
|
+
const scoringContract = readJson(path.join(studyDir, 'scoring-contract.json'));
|
|
70
|
+
const manifest = readJson(path.join(studyDir, 'suite-manifest.json'));
|
|
71
|
+
|
|
72
|
+
if (!['claim-evidence-scoring.v1', 'claim-evidence-scoring.v2', 'claim-evidence-scoring.v3'].includes(scoringContract.schema)) {
|
|
73
|
+
problems.push('unsupported scoring contract version');
|
|
74
|
+
}
|
|
75
|
+
if (['claim-evidence-scoring.v2', 'claim-evidence-scoring.v3'].includes(scoringContract.schema)
|
|
76
|
+
&& hashOfCanonical(scoringContract.implementation) !== hashOfCanonical(challengeImplementationManifest())) {
|
|
77
|
+
problems.push('v2/v3 scoring implementation drift');
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const principleContentHash = hashOfCanonical(principle.contentJson);
|
|
81
|
+
if (principleContentHash !== inputManifest.expectedSourceHashes.principleContentHash) problems.push('principle content hash != input-manifest expected');
|
|
82
|
+
if (principleContentHash !== mapping.principleContentHash) problems.push('mapping.principleContentHash mismatch');
|
|
83
|
+
|
|
84
|
+
if (!verifyCanonicalHash(mapping, mapping.mappingHash, { stripFields: ['mappingHash'] }).ok) problems.push('mappingHash drift');
|
|
85
|
+
const graderHash = hashOfCanonical(scoringContract);
|
|
86
|
+
if (!verifyCanonicalHash(manifest, manifest.suiteHash, { stripFields: ['suiteHash'] }).ok) problems.push('suiteHash drift');
|
|
87
|
+
if (manifest.mappingHash !== mapping.mappingHash) problems.push('manifest↔mapping mappingHash mismatch');
|
|
88
|
+
if (manifest.graderHash !== graderHash) problems.push('manifest↔scoring-contract graderHash mismatch');
|
|
89
|
+
|
|
90
|
+
for (const c of manifest.cases) {
|
|
91
|
+
const judgeFile = path.join(studyDir, 'judge-only', `${c.caseId}.json`);
|
|
92
|
+
const visibleFile = path.join(studyDir, 'actor-visible', `${c.caseId}.json`);
|
|
93
|
+
if (!fs.existsSync(judgeFile) || !fs.existsSync(visibleFile)) { problems.push(`case ${c.caseId}: missing frozen file`); continue; }
|
|
94
|
+
const judge = readJson(judgeFile);
|
|
95
|
+
const expectedJudgeSchema = scoringContract.schema === 'claim-evidence-scoring.v3'
|
|
96
|
+
? 'claim-evidence-judge-only.v3'
|
|
97
|
+
: scoringContract.schema === 'claim-evidence-scoring.v2' ? 'claim-evidence-judge-only.v2' : 'claim-evidence-judge-only.v1';
|
|
98
|
+
if (judge.schema !== expectedJudgeSchema) {
|
|
99
|
+
problems.push(`case ${c.caseId}: scoring contract version mismatch`);
|
|
100
|
+
}
|
|
101
|
+
const visible = readJson(visibleFile);
|
|
102
|
+
if (!verifyCanonicalHash(judge, c.goldHash, { stripFields: ['inputHash', 'goldHash'] }).ok) problems.push(`case ${c.caseId}: judge-only goldHash drift`);
|
|
103
|
+
if (hashOfCanonical(visible) !== c.inputHash) problems.push(`case ${c.caseId}: actor-visible inputHash drift`);
|
|
104
|
+
if (judge.inputHash !== c.inputHash || judge.goldHash !== c.goldHash) problems.push(`case ${c.caseId}: judge-only hash fields != manifest`);
|
|
105
|
+
}
|
|
106
|
+
if (problems.length > 0) throw new FrozenStudyDriftError(problems.join('; '));
|
|
107
|
+
return {
|
|
108
|
+
studyId: manifest.studyId,
|
|
109
|
+
suiteHash: manifest.suiteHash,
|
|
110
|
+
mappingHash: mapping.mappingHash,
|
|
111
|
+
graderHash,
|
|
112
|
+
principleContentHash,
|
|
113
|
+
manifest,
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Rebuild the A/B/C arm configs from CANONICAL frozen sources and compare
|
|
119
|
+
* with the stored arm-configs.json (payload hashes + isolation audit). Catches
|
|
120
|
+
* arm-content mixing (e.g. summary text swapped into arm B).
|
|
121
|
+
*
|
|
122
|
+
* v0.9: when examiner-arm-configs.json exists the study is an EXAMINER
|
|
123
|
+
* dual-mode study — verification routes to the examiner shape (HEPS +
|
|
124
|
+
* target from frozen inputs, payloads rebuilt under the frozen budget,
|
|
125
|
+
* manifest-pinned payload hashes).
|
|
126
|
+
*/
|
|
127
|
+
export function verifyArmConfigsFrozen(studyDir) {
|
|
128
|
+
if (fs.existsSync(path.join(studyDir, 'examiner-arm-configs.json'))) {
|
|
129
|
+
return verifyExaminerArmConfigsFrozen(studyDir);
|
|
130
|
+
}
|
|
131
|
+
const principle = readJson(path.join(studyDir, 'principle-frozen.json'));
|
|
132
|
+
const summaryPayload = readJson(path.join(studyDir, 'arm-payload-experience-summary.json'));
|
|
133
|
+
const armConfigs = readJson(path.join(studyDir, 'arm-configs.json'));
|
|
134
|
+
const prereg = readJson(path.join(studyDir, 'preregistration.json'));
|
|
135
|
+
const rebuilt = buildStudyArms({
|
|
136
|
+
commonContextText: prereg.commonContext.text,
|
|
137
|
+
principleStatement: principle.contentJson.principleDraft.statement,
|
|
138
|
+
summaryText: summaryPayload.text,
|
|
139
|
+
budgetChars: prereg.delivery.budgetChars,
|
|
140
|
+
});
|
|
141
|
+
const problems = [];
|
|
142
|
+
if (armConfigs.arms.A.delivery.payloadText !== '') problems.push('stored arm A payload not empty');
|
|
143
|
+
if (armConfigs.arms.B.delivery.payloadHash !== rebuilt.arms.B.delivery.payloadHash) problems.push('arm B payloadHash != frozen principle statement');
|
|
144
|
+
if (armConfigs.arms.C.delivery.payloadHash !== rebuilt.arms.C.delivery.payloadHash) problems.push('arm C payloadHash != frozen summary');
|
|
145
|
+
if (rebuilt.arms.B.delivery.payloadHash !== sha256(principle.contentJson.principleDraft.statement)) problems.push('arm B payload is not the exact frozen statement');
|
|
146
|
+
const manifest = readJson(path.join(studyDir, 'suite-manifest.json'));
|
|
147
|
+
const firstCaseId = manifest.cases[0].caseId;
|
|
148
|
+
const audit = auditArmIsolation({
|
|
149
|
+
arms: rebuilt.arms,
|
|
150
|
+
commonContextText: prereg.commonContext.text,
|
|
151
|
+
taskText: readJson(path.join(studyDir, 'actor-visible', `${firstCaseId}.json`)).taskText,
|
|
152
|
+
normativeMarkers: summaryPayload.normativeMarkersThatMustNotAppear,
|
|
153
|
+
});
|
|
154
|
+
if (!audit.ok) problems.push(...audit.problems.map((p) => `isolation: ${p}`));
|
|
155
|
+
if (problems.length > 0) throw new FrozenStudyDriftError(problems.join('; '));
|
|
156
|
+
return {
|
|
157
|
+
arms: rebuilt.arms,
|
|
158
|
+
commonContextHash: rebuilt.commonContextHash,
|
|
159
|
+
evidence: audit.evidence,
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
export const STUB_STRATEGIES = {
|
|
164
|
+
'perfect-play': (ctx) => perfectPlayActor(ctx.caseDef),
|
|
165
|
+
'greedy-observer': () => createGreedyObserverStub(),
|
|
166
|
+
naive: () => createNaiveStub(),
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* v0.9 dual-mode arm verification: rebuild the control/treatment payloads from
|
|
171
|
+
* the frozen examiner inputs (heps-principles.json + examiner-manifest.json)
|
|
172
|
+
* under the frozen budget/ordering, compare against the stored
|
|
173
|
+
* examiner-arm-configs.json, enforce the cross-arm isolation audit, and re-pin
|
|
174
|
+
* the payload hashes to the manifest when it carries them. Returns the SAME
|
|
175
|
+
* arm shape as the legacy verifier (delivery objects) so the runner loop,
|
|
176
|
+
* prompt composition, contamination checks and injection evidence collection
|
|
177
|
+
* are shape-compatible across both study kinds.
|
|
178
|
+
*/
|
|
179
|
+
export function verifyExaminerArmConfigsFrozen(studyDir) {
|
|
180
|
+
// P1-3: cross-file preregistration binding + immutable anchor FIRST — each
|
|
181
|
+
// frozen file proving itself is not enough; the files must agree with each
|
|
182
|
+
// other and with the anchor written at freeze time.
|
|
183
|
+
const bindings = verifyExaminerStudyBindings(studyDir);
|
|
184
|
+
if (!bindings.ok) throw new FrozenStudyDriftError(`preregistration bindings: ${bindings.problems.join('; ')}`);
|
|
185
|
+
const exManifest = readJson(path.join(studyDir, 'examiner-manifest.json'));
|
|
186
|
+
const hepsInput = readJson(path.join(studyDir, 'heps-principles.json'));
|
|
187
|
+
const armConfigs = readJson(path.join(studyDir, 'examiner-arm-configs.json'));
|
|
188
|
+
const prereg = readJson(path.join(studyDir, 'preregistration.json'));
|
|
189
|
+
const problems = [];
|
|
190
|
+
const manifestCheck = verifyExaminerManifest(exManifest);
|
|
191
|
+
if (!manifestCheck.ok) problems.push(`examiner manifest drift: ${manifestCheck.problems.join('; ')}`);
|
|
192
|
+
const heps = buildHostEffectivePrincipleSet({
|
|
193
|
+
principles: hepsInput.principles,
|
|
194
|
+
targetPrincipleId: hepsInput.targetPrincipleId,
|
|
195
|
+
host: hepsInput.host,
|
|
196
|
+
});
|
|
197
|
+
if (heps.snapshotHash !== exManifest.principleSetSnapshotHash) {
|
|
198
|
+
problems.push('HEPS snapshot hash != manifest principleSetSnapshotHash — the principle set drifted after preregistration');
|
|
199
|
+
}
|
|
200
|
+
const targetFact = hepsInput.principles.find((p) => p.id === hepsInput.targetPrincipleId);
|
|
201
|
+
if (!targetFact) problems.push(`target principle '${hepsInput.targetPrincipleId}' missing from heps-principles.json`);
|
|
202
|
+
const budgetChars = Number.isInteger(armConfigs.budgetChars) ? armConfigs.budgetChars : prereg.delivery.budgetChars;
|
|
203
|
+
const rebuilt = buildDualModeArmPayloads({
|
|
204
|
+
mode: exManifest.experimentMode,
|
|
205
|
+
heps,
|
|
206
|
+
targetPrinciple: targetFact,
|
|
207
|
+
principleFacts: hepsInput.principles,
|
|
208
|
+
budgetChars,
|
|
209
|
+
ordering: armConfigs.ordering || 'id-asc',
|
|
210
|
+
});
|
|
211
|
+
const stored = armConfigs.arms || {};
|
|
212
|
+
for (const [side, armId] of [['control', 'A'], ['treatment', 'B']]) {
|
|
213
|
+
if (!stored[side] || stored[side].payloadHash !== rebuilt.arms[side].payloadHash) {
|
|
214
|
+
problems.push(`arm ${armId} (${side}) payloadHash != rebuilt dual-mode payload — arm content drift`);
|
|
215
|
+
}
|
|
216
|
+
if (stored[side] && typeof stored[side].payloadText === 'string' && sha256(stored[side].payloadText) !== stored[side].payloadHash) {
|
|
217
|
+
problems.push(`arm ${armId} (${side}) stored payloadText does not match its own payloadHash — the arm config was edited after freezing`);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
if (exManifest.armPayloadHashes) {
|
|
221
|
+
if (exManifest.armPayloadHashes.control !== rebuilt.arms.control.payloadHash) problems.push('manifest armPayloadHashes.control != rebuilt control payload');
|
|
222
|
+
if (exManifest.armPayloadHashes.treatment !== rebuilt.arms.treatment.payloadHash) problems.push('manifest armPayloadHashes.treatment != rebuilt treatment payload');
|
|
223
|
+
}
|
|
224
|
+
if (!rebuilt.isolationAudit.ok) problems.push(...rebuilt.isolationAudit.problems.map((p) => `isolation: ${p}`));
|
|
225
|
+
// The control prompt must never carry the target principle text (the A/B
|
|
226
|
+
// contrast is exactly the target — §2.1/§2.2 frozen-variable rule).
|
|
227
|
+
if (targetFact && rebuilt.arms.control.payloadText.includes(targetFact.text) && exManifest.experimentMode === 'full_library_incremental') {
|
|
228
|
+
problems.push('control payload contains the target principle text — treatment leak into control');
|
|
229
|
+
}
|
|
230
|
+
if (problems.length > 0) throw new FrozenStudyDriftError(problems.join('; '));
|
|
231
|
+
const toDeliveryArm = (side, armId) => ({
|
|
232
|
+
armId,
|
|
233
|
+
name: side === 'control' ? 'EXAMINER_CONTROL' : 'EXAMINER_TREATMENT',
|
|
234
|
+
contentMode: rebuilt.arms[side].contentMode,
|
|
235
|
+
delivery: {
|
|
236
|
+
channel: 'research-same-channel-v1',
|
|
237
|
+
armId,
|
|
238
|
+
payloadText: rebuilt.arms[side].payloadText,
|
|
239
|
+
payloadHash: rebuilt.arms[side].payloadHash,
|
|
240
|
+
deliveredBytes: Buffer.byteLength(rebuilt.arms[side].payloadText, 'utf8'),
|
|
241
|
+
position: 'after-task-text',
|
|
242
|
+
budgetChars,
|
|
243
|
+
deliveredAt: null,
|
|
244
|
+
},
|
|
245
|
+
examinerCapacity: rebuilt.capacity[side],
|
|
246
|
+
});
|
|
247
|
+
return {
|
|
248
|
+
arms: { A: toDeliveryArm('control', 'A'), B: toDeliveryArm('treatment', 'B') },
|
|
249
|
+
commonContextHash: sha256(prereg.commonContext.text),
|
|
250
|
+
evidence: {
|
|
251
|
+
mode: exManifest.experimentMode,
|
|
252
|
+
controlPayloadHash: rebuilt.arms.control.payloadHash,
|
|
253
|
+
treatmentPayloadHash: rebuilt.arms.treatment.payloadHash,
|
|
254
|
+
isolationAudit: rebuilt.isolationAudit,
|
|
255
|
+
},
|
|
256
|
+
// The control arm's FORBIDDEN content (the target principle text) so the
|
|
257
|
+
// offline injection-evidence collector can record contamination honestly
|
|
258
|
+
// (full-library controls legitimately carry the non-target library).
|
|
259
|
+
examinerTargetText: targetFact ? targetFact.text : null,
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
export const RUNNER_EXECUTION_MODES = ['offline-stub', 'F1-direct-model', 'F2-real-host'];
|
|
264
|
+
|
|
265
|
+
/**
|
|
266
|
+
* P1-4: study-level run readiness, evaluated under the study write lock and
|
|
267
|
+
* BEFORE RUN_STARTED. The console preflight is a convenience for the UI; this
|
|
268
|
+
* is the authority the execution path itself enforces, so a client that calls
|
|
269
|
+
* POST /run directly cannot skip it. Throws FrozenStudyDriftError on any
|
|
270
|
+
* blocker — the caller must then leave the ledger untouched (no RUN_STARTED,
|
|
271
|
+
* no reservation, no actor).
|
|
272
|
+
*/
|
|
273
|
+
export function verifyStudyRunReadiness(studyDir) {
|
|
274
|
+
// Examiner studies only. Legacy challenge suites keep their established
|
|
275
|
+
// per-unit SUITE_DRIFT conversion inside the dispatch loop; refusing them
|
|
276
|
+
// here would change behavior they are contractually tested for.
|
|
277
|
+
const examinerManifestPath = path.join(studyDir, 'examiner-manifest.json');
|
|
278
|
+
if (!fs.existsSync(examinerManifestPath)) return { frozen: null, armCheck: null, examiner: null };
|
|
279
|
+
const frozen = verifyFrozenStudy(studyDir);
|
|
280
|
+
const armCheck = verifyArmConfigsFrozen(studyDir); // bindings + immutable anchor
|
|
281
|
+
const exManifest = readJson(examinerManifestPath);
|
|
282
|
+
const hepsInput = readJson(path.join(studyDir, 'heps-principles.json'));
|
|
283
|
+
const rulecode = readJson(path.join(studyDir, 'rulecode-baseline.json'));
|
|
284
|
+
const problems = [];
|
|
285
|
+
let heps = null;
|
|
286
|
+
let policy = null;
|
|
287
|
+
try {
|
|
288
|
+
heps = buildHostEffectivePrincipleSet({
|
|
289
|
+
principles: hepsInput.principles,
|
|
290
|
+
targetPrincipleId: hepsInput.targetPrincipleId,
|
|
291
|
+
host: hepsInput.host,
|
|
292
|
+
});
|
|
293
|
+
if (heps.snapshotHash !== exManifest.principleSetSnapshotHash) problems.push('HEPS snapshot hash != manifest principleSetSnapshotHash');
|
|
294
|
+
if (!heps.target || heps.target.qualifies !== true) {
|
|
295
|
+
problems.push(`target principle '${String(hepsInput.targetPrincipleId)}' is not host-effective (${heps.target ? heps.target.exclusionReason : 'missing'})`);
|
|
296
|
+
}
|
|
297
|
+
} catch (e) {
|
|
298
|
+
problems.push(`HEPS rebuild failed: ${e.message}`);
|
|
299
|
+
}
|
|
300
|
+
try {
|
|
301
|
+
policy = buildRuleCodePolicy({
|
|
302
|
+
mode: exManifest.experimentMode,
|
|
303
|
+
ruleCodes: Array.isArray(rulecode.ruleCodes) ? rulecode.ruleCodes : [],
|
|
304
|
+
targetPrincipleId: hepsInput.targetPrincipleId,
|
|
305
|
+
principleRuleCodeIndependence: rulecode.principleRuleCodeIndependence,
|
|
306
|
+
targetRuleCodeCount: rulecode.targetRuleCodeCount ?? null,
|
|
307
|
+
});
|
|
308
|
+
const rcCheck = verifyRuleCodePolicy(policy);
|
|
309
|
+
if (!rcCheck.ok) problems.push(`RuleCode policy: ${rcCheck.reason}`);
|
|
310
|
+
if (policy.ruleCodeSnapshotHash !== exManifest.ruleCodeSnapshotHash) problems.push('RuleCode policy snapshot hash != manifest ruleCodeSnapshotHash');
|
|
311
|
+
if (policy.interventionType !== exManifest.interventionType) problems.push('RuleCode interventionType != manifest interventionType');
|
|
312
|
+
if (policy.executable !== true) problems.push(`INTERVENTION_NOT_IDENTIFIABLE: ${policy.couplingReason}`);
|
|
313
|
+
} catch (e) {
|
|
314
|
+
problems.push(`RuleCode policy rebuild failed: ${e.message}`);
|
|
315
|
+
}
|
|
316
|
+
if (problems.length > 0) throw new FrozenStudyDriftError(problems.join('; '));
|
|
317
|
+
return { frozen, armCheck, examiner: { heps, policy } };
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
/** Automatic dispatch stop: after this many CONSECUTIVE RESULT_UNKNOWN unit
|
|
322
|
+
* settlements in one run, every remaining unit is refused (UNKNOWN_STREAK_
|
|
323
|
+
* STOPPED) before any dispatch. Run-scoped; never mutates authorization,
|
|
324
|
+
* budget or plan. */
|
|
325
|
+
export const STAGE_R_UNKNOWN_STREAK_LIMIT = 3;
|
|
326
|
+
|
|
327
|
+
/** Capacity circuit breaker: after this many CONSECUTIVE unit settlements
|
|
328
|
+
* whose failure is a PROVEN provider-capacity condition (HTTP 429 rate
|
|
329
|
+
* limiting, or an explicit HTTP 5xx upstream-overload status), every
|
|
330
|
+
* remaining unit is refused (CAPACITY_STREAK_STOPPED) before any dispatch.
|
|
331
|
+
* Deliberately narrow: identity mismatches, output-contract violations,
|
|
332
|
+
* authorization refusals and other known failures are NOT capacity errors
|
|
333
|
+
* and do not arm this breaker. */
|
|
334
|
+
export const STAGE_R_CAPACITY_STREAK_LIMIT = 3;
|
|
335
|
+
|
|
336
|
+
/** Classify a FAILED_KNOWN detail string as a provider-capacity failure.
|
|
337
|
+
* The detail is `execution_error: <responder message>`; only responder-
|
|
338
|
+
* classified 429 (`rate_limited (HTTP 429)`) and explicit 5xx statuses
|
|
339
|
+
* (`http_500:`…`http_599:`) count — both carry server-attested evidence,
|
|
340
|
+
* never a guess. */
|
|
341
|
+
export function isCapacityFailureDetail(detail) {
|
|
342
|
+
if (typeof detail !== 'string') return false;
|
|
343
|
+
const message = detail.startsWith('execution_error: ') ? detail.slice('execution_error: '.length) : detail;
|
|
344
|
+
return message.startsWith('rate_limited (HTTP 429)') || /^http_5\d\d\b/.test(message);
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/**
|
|
348
|
+
* v0.9 §3.3 dispatch order resolution. Without plan.unitOrder the runner
|
|
349
|
+
* keeps its historical deterministic arms→cases→repetitions order (existing
|
|
350
|
+
* studies byte-identical). With plan.unitOrder (the FROZEN pair-assignment
|
|
351
|
+
* dispatch order) the order is honored — but only after proving it covers
|
|
352
|
+
* EXACTLY the planned unit set: skipping, duplicating or smuggling units is
|
|
353
|
+
* a tampered randomization and refuses the whole run before any dispatch.
|
|
354
|
+
*/
|
|
355
|
+
export function resolveDispatchUnits(plan) {
|
|
356
|
+
if (!plan || !Array.isArray(plan.arms) || !Array.isArray(plan.cases) || !Array.isArray(plan.repetitions)) {
|
|
357
|
+
throw new Error('resolveDispatchUnits: plan.arms/cases/repetitions arrays required');
|
|
358
|
+
}
|
|
359
|
+
const canonical = [];
|
|
360
|
+
for (const armId of plan.arms) {
|
|
361
|
+
for (const caseId of plan.cases) {
|
|
362
|
+
for (const repetitionId of plan.repetitions) {
|
|
363
|
+
canonical.push(`${caseId}/${armId}/r${repetitionId}`);
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
if (plan.unitOrder === undefined) {
|
|
368
|
+
return canonical.map((key) => {
|
|
369
|
+
const [caseId, armId, rep] = key.split('/');
|
|
370
|
+
return { caseId, armId, repetitionId: rep.slice(1) };
|
|
371
|
+
});
|
|
372
|
+
}
|
|
373
|
+
if (!Array.isArray(plan.unitOrder) || plan.unitOrder.length === 0) {
|
|
374
|
+
throw new Error('resolveDispatchUnits: plan.unitOrder must be a non-empty array when supplied');
|
|
375
|
+
}
|
|
376
|
+
const ordered = plan.unitOrder.map((entry, i) => {
|
|
377
|
+
if (!entry || typeof entry !== 'object' || !entry.caseId || !entry.armId || entry.repetitionId === undefined) {
|
|
378
|
+
throw new Error(`resolveDispatchUnits: plan.unitOrder[${i}] must be {caseId, armId, repetitionId}`);
|
|
379
|
+
}
|
|
380
|
+
return { caseId: entry.caseId, armId: entry.armId, repetitionId: entry.repetitionId };
|
|
381
|
+
});
|
|
382
|
+
const key = (u) => `${u.caseId}/${u.armId}/r${u.repetitionId}`;
|
|
383
|
+
const expected = new Map(canonical.map((k) => [k, 1]));
|
|
384
|
+
const seen = new Map();
|
|
385
|
+
for (const u of ordered) {
|
|
386
|
+
const k = key(u);
|
|
387
|
+
if (!expected.has(k)) throw new Error(`resolveDispatchUnits: unitOrder contains '${k}' which is NOT in the planned unit set — a schedule outside the preregistered plan refuses the run`);
|
|
388
|
+
if (seen.has(k)) throw new Error(`resolveDispatchUnits: unitOrder contains '${k}' more than once`);
|
|
389
|
+
seen.set(k, 1);
|
|
390
|
+
}
|
|
391
|
+
for (const k of expected.keys()) {
|
|
392
|
+
if (!seen.has(k)) throw new Error(`resolveDispatchUnits: unitOrder omits planned unit '${k}' — a randomization that skips units is tampering, not scheduling`);
|
|
393
|
+
}
|
|
394
|
+
return ordered;
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
/**
|
|
398
|
+
* Serial study run. One writer (lock), one unit at a time, append-only
|
|
399
|
+
* records. Each unit: verify frozen + compose the arm prompt → classify
|
|
400
|
+
* recovery state → gate (real modes, BEFORE the factory) → create actor →
|
|
401
|
+
* reserve → prepare → execute loop → score → persist attempt atomically →
|
|
402
|
+
* settle. The run is ASYNC (real F1/F2 actors answer across real
|
|
403
|
+
* transports); offline stubs are awaited transparently and byte-identical.
|
|
404
|
+
*/
|
|
405
|
+
export async function runChallengeStudy({
|
|
406
|
+
studyDir,
|
|
407
|
+
plan,
|
|
408
|
+
actorFactory,
|
|
409
|
+
maxAttemptsTotal = 999,
|
|
410
|
+
owner = 'challenge-runner',
|
|
411
|
+
deps = {},
|
|
412
|
+
execution = null,
|
|
413
|
+
}) {
|
|
414
|
+
const declaredMode = execution && execution.mode !== undefined ? execution.mode : 'offline-stub';
|
|
415
|
+
if (!RUNNER_EXECUTION_MODES.includes(declaredMode)) {
|
|
416
|
+
throw new Error(`runChallengeStudy: execution.mode must be one of ${RUNNER_EXECUTION_MODES.join(' | ')} (got '${String(declaredMode)}') — the runner, not the actor, declares the execution mode`);
|
|
417
|
+
}
|
|
418
|
+
const isRealMode = declaredMode !== 'offline-stub';
|
|
419
|
+
// P1-2: real execution reads the AUTHORITATIVE record from the study
|
|
420
|
+
// directory — the same run-authorization.json whose NOT_AUTHORIZED
|
|
421
|
+
// placeholder the gate enforces. Tests inject fixtures by writing this
|
|
422
|
+
// file inside a COPIED study dir; there is no in-memory override path.
|
|
423
|
+
const authorizationRecord = isRealMode ? readStageRAuthorization(studyDir) : null;
|
|
424
|
+
const adjudicate = deps.adjudicate || adjudicateClaimEvidence;
|
|
425
|
+
// v0.9 §3.3: a FROZEN randomized dispatch order (plan.unitOrder, produced by
|
|
426
|
+
// the examiner pair assignment) replaces the deterministic triple loop when
|
|
427
|
+
// supplied. Fail-closed validation: the order must cover EXACTLY the
|
|
428
|
+
// planned unit set — an order that skips, duplicates or smuggles in units
|
|
429
|
+
// is a tampered randomization, not a schedule.
|
|
430
|
+
// Validation runs BEFORE the lock and BEFORE RUN_STARTED: a rejected
|
|
431
|
+
// schedule must leave no ledger row and no residual write lock.
|
|
432
|
+
const dispatchUnits = resolveDispatchUnits(plan);
|
|
433
|
+
|
|
434
|
+
const ledger = createChallengeLedger(studyDir);
|
|
435
|
+
const lock = acquireStudyWriteLock(studyDir, { owner });
|
|
436
|
+
|
|
437
|
+
const summary = { studyId: null, units: [], interrupted: null, stoppedReason: null };
|
|
438
|
+
const attemptId = plan.attemptId || 'a1';
|
|
439
|
+
// Automatic dispatch stop (SPEC §7.4 discipline, made machine-enforced):
|
|
440
|
+
// after N CONSECUTIVE units settle RESULT_UNKNOWN in THIS run, no further
|
|
441
|
+
// unit may dispatch anything. The streak counts only actual settlements:
|
|
442
|
+
// an ambiguous RESULT_UNKNOWN increments it; a SUCCEEDED or FAILED_KNOWN
|
|
443
|
+
// settlement resets it; recovery-classified units, skips and refusals
|
|
444
|
+
// leave it unchanged. The stop is run-scoped — it never touches the
|
|
445
|
+
// authorization, never widens the budget, never changes the plan, and
|
|
446
|
+
// keeps every existing ledger row and attempt record intact.
|
|
447
|
+
let consecutiveUnknown = 0;
|
|
448
|
+
// Capacity circuit breaker (distinct from the unknown streak): counts only
|
|
449
|
+
// unit settlements whose failure detail is a PROVEN provider-capacity
|
|
450
|
+
// condition (429 / explicit 5xx). A SUCCEEDED settlement or a NON-capacity
|
|
451
|
+
// FAILED_KNOWN settlement breaks the streak; skips and refusals leave it
|
|
452
|
+
// unchanged. Same run-scoped discipline as the unknown streak.
|
|
453
|
+
let consecutiveCapacityFailures = 0;
|
|
454
|
+
try {
|
|
455
|
+
// P1-4 server-enforced readiness: identity/path ownership are the caller's
|
|
456
|
+
// gate (the console resolves the directory from its own enumeration and
|
|
457
|
+
// verifies the write permission marker); the FROZEN study is re-verified
|
|
458
|
+
// here, under the lock, before any ledger row exists. Any drift refuses
|
|
459
|
+
// the run with no RUN_STARTED, no reservation and no actor.
|
|
460
|
+
verifyStudyRunReadiness(studyDir);
|
|
461
|
+
ledger.appendEvent({ type: 'RUN_STARTED', owner, plan });
|
|
462
|
+
for (const dispatchUnit of dispatchUnits) {
|
|
463
|
+
const { caseId, armId, repetitionId } = dispatchUnit;
|
|
464
|
+
const unit = { caseId, armId, repetitionId, attemptId };
|
|
465
|
+
const before = ledger.classifyUnit(unit);
|
|
466
|
+
|
|
467
|
+
if (before.state === 'SUCCEEDED' || before.state === 'FAILED_KNOWN') {
|
|
468
|
+
ledger.appendEvent({ type: 'UNIT_SKIPPED', unit, reason: 'already_recorded_no_rerun', recordedResult: before.state });
|
|
469
|
+
summary.units.push({ unit, state: before.state, skipped: true });
|
|
470
|
+
continue;
|
|
471
|
+
}
|
|
472
|
+
if (before.state === 'RESULT_UNKNOWN') {
|
|
473
|
+
ledger.appendEvent({ type: 'UNIT_REFUSED', unit, reason: 'result_unknown_no_auto_retry' });
|
|
474
|
+
summary.units.push({ unit, state: 'RESULT_UNKNOWN', refused: true });
|
|
475
|
+
continue;
|
|
476
|
+
}
|
|
477
|
+
if (consecutiveUnknown >= STAGE_R_UNKNOWN_STREAK_LIMIT) {
|
|
478
|
+
// streak stop: BEFORE any input read, gate call, factory
|
|
479
|
+
// invocation or reservation — nothing downstream can dispatch.
|
|
480
|
+
if (summary.stoppedReason === null) summary.stoppedReason = `stage_r_unknown_streak_stop: ${consecutiveUnknown} consecutive RESULT_UNKNOWN units — automatic dispatch stop`;
|
|
481
|
+
ledger.appendEvent({ type: 'UNIT_REFUSED', unit, reason: 'stage_r_unknown_streak_stop', detail: `${consecutiveUnknown} consecutive RESULT_UNKNOWN units — no further dispatch this run (resume requires a new run; settled attempts are never retried)` });
|
|
482
|
+
summary.units.push({ unit, state: 'UNKNOWN_STREAK_STOPPED', refused: true, reason: 'stage_r_unknown_streak_stop' });
|
|
483
|
+
continue;
|
|
484
|
+
}
|
|
485
|
+
if (consecutiveCapacityFailures >= STAGE_R_CAPACITY_STREAK_LIMIT) {
|
|
486
|
+
// capacity breaker: same pre-everything stop shape — no input
|
|
487
|
+
// read, no gate call, no factory, no reservation. Proven 429/5xx
|
|
488
|
+
// saturation means more dispatch would only burn budget.
|
|
489
|
+
if (summary.stoppedReason === null) summary.stoppedReason = `stage_r_capacity_streak_stop: ${consecutiveCapacityFailures} consecutive provider-capacity (429/5xx) failures — automatic dispatch stop`;
|
|
490
|
+
ledger.appendEvent({ type: 'UNIT_REFUSED', unit, reason: 'stage_r_capacity_streak_stop', detail: `${consecutiveCapacityFailures} consecutive provider-capacity (429/5xx) unit failures — no further dispatch this run (failover requires a NEW authorization bound to the alternate model; settled attempts and reservations are kept)` });
|
|
491
|
+
summary.units.push({ unit, state: 'CAPACITY_STREAK_STOPPED', refused: true, reason: 'stage_r_capacity_streak_stop' });
|
|
492
|
+
continue;
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
let caseDef;
|
|
496
|
+
let caseVisible;
|
|
497
|
+
try {
|
|
498
|
+
caseDef = readJson(path.join(studyDir, 'judge-only', `${caseId}.json`));
|
|
499
|
+
caseVisible = readJson(path.join(studyDir, 'actor-visible', `${caseId}.json`));
|
|
500
|
+
} catch (e) {
|
|
501
|
+
ledger.appendEvent({ type: 'UNIT_REFUSED', unit, reason: 'unit_inputs_unreadable', detail: e.message });
|
|
502
|
+
summary.units.push({ unit, state: 'SUITE_DRIFT', refused: true, detail: e.message });
|
|
503
|
+
continue;
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
// Per-unit fail-closed frozen verification + ARM PROMPT composition
|
|
507
|
+
// FIRST (read-only, zero mutations): the composed arm prompt must
|
|
508
|
+
// exist before the gate and the actor factory — a real actor
|
|
509
|
+
// without its arm delivery would erase the A/B/C contrast, and
|
|
510
|
+
// drift must refuse before anything is constructed or reserved.
|
|
511
|
+
let frozen;
|
|
512
|
+
let armCheck;
|
|
513
|
+
let armPrompt;
|
|
514
|
+
try {
|
|
515
|
+
frozen = verifyFrozenStudy(studyDir);
|
|
516
|
+
armCheck = verifyArmConfigsFrozen(studyDir);
|
|
517
|
+
armPrompt = composeArmPrompt({ arms: armCheck.arms, armId, taskText: caseVisible.taskText });
|
|
518
|
+
} catch (e) {
|
|
519
|
+
ledger.appendEvent({ type: 'UNIT_REFUSED', unit, reason: 'frozen_study_drift', detail: e.message });
|
|
520
|
+
summary.units.push({ unit, state: 'SUITE_DRIFT', refused: true, detail: e.message });
|
|
521
|
+
continue;
|
|
522
|
+
}
|
|
523
|
+
summary.studyId = frozen.studyId;
|
|
524
|
+
|
|
525
|
+
// Stage R boundary (v0.4 §7.2): the runner-declared execution mode
|
|
526
|
+
// is authoritative and the gate runs BEFORE the actor factory is
|
|
527
|
+
// ever invoked — an unauthorized real run constructs no actor,
|
|
528
|
+
// reserves nothing and takes zero actor steps. Bindings come from
|
|
529
|
+
// the frozen verification above (freshly recomputed, never stored).
|
|
530
|
+
// `execution.now` may be a NUMBER (fixed clock, deterministic tests)
|
|
531
|
+
// or a FUNCTION (evaluated PER UNIT — the production CLI passes
|
|
532
|
+
// () => Date.now() so an authorization that expires mid-run is
|
|
533
|
+
// refused for every subsequent unit, not just at startup).
|
|
534
|
+
let gate = null;
|
|
535
|
+
if (isRealMode) {
|
|
536
|
+
gate = verifyStageRAuthorization({
|
|
537
|
+
authorizationRecord,
|
|
538
|
+
retiredAuthorizations: readAuthorizationRevocations(), // archived-evidence records refused at ANY clock, any study copy (Owner P1-1 isolation scheme)
|
|
539
|
+
bindings: {
|
|
540
|
+
studyId: frozen.studyId,
|
|
541
|
+
suiteHash: frozen.suiteHash,
|
|
542
|
+
mappingHash: frozen.mappingHash,
|
|
543
|
+
graderHash: frozen.graderHash,
|
|
544
|
+
studyPlanHash: readStudyPlanHash(studyDir),
|
|
545
|
+
},
|
|
546
|
+
plan,
|
|
547
|
+
maxAttemptsTotal,
|
|
548
|
+
now: typeof execution.now === 'function' ? execution.now() : execution.now,
|
|
549
|
+
declaredExecutionMode: declaredMode,
|
|
550
|
+
declaredRequestTimeoutMs: execution.requestTimeoutMs, // undefined = default 240000 (the gate enforces the authorization binding)
|
|
551
|
+
declaredMaxRequestsPerAttempt: execution.maxRequestsPerAttempt, // undefined = no per-attempt cap approved (the gate enforces the authorization binding)
|
|
552
|
+
});
|
|
553
|
+
if (!gate.ok) {
|
|
554
|
+
ledger.appendEvent({ type: 'UNIT_REFUSED', unit, reason: 'stage_r_not_authorized', detail: gate.reasons });
|
|
555
|
+
summary.units.push({ unit, state: 'STAGE_R_REFUSED', refused: true, reasons: gate.reasons });
|
|
556
|
+
continue;
|
|
557
|
+
}
|
|
558
|
+
// PRI-1037 deep evidence verification (F2 only, after the gate's
|
|
559
|
+
// structural probe check, BEFORE the factory): the authorization's
|
|
560
|
+
// realHostProbe.evidenceRef must resolve to a real-host-executed,
|
|
561
|
+
// all-checks-PASSED probe record bound to THIS frozen study, with
|
|
562
|
+
// every recorded session corroborated by the host's own evidence.
|
|
563
|
+
// Host dispatch follows the probe binding's hostKind: codex-cli →
|
|
564
|
+
// rollout corroboration under execution.codexHome (historical
|
|
565
|
+
// path); openclaw → exec-state corroboration with the operator-
|
|
566
|
+
// declared probe controlled root (execution.probeControlledRoot —
|
|
567
|
+
// the workRoot rounds dir the probe ran in; missing = refuse,
|
|
568
|
+
// never a silent skip). Offline-rehearsal, missing, corrupt,
|
|
569
|
+
// mismatched or uncorroborated evidence refuses F2.
|
|
570
|
+
if (declaredMode === 'F2-real-host') {
|
|
571
|
+
const probeBinding = authorizationRecord && authorizationRecord.isolationAcceptance && authorizationRecord.isolationAcceptance.realHostProbe;
|
|
572
|
+
const probeHostKind = probeBinding && probeBinding.hostKind === 'openclaw' ? 'openclaw' : 'codex-cli';
|
|
573
|
+
const { readOpenClawExecStateTrajectory } = await import('./challenge-openclaw-host.mjs');
|
|
574
|
+
const probeVerify = verifyF2ProbeEvidence({
|
|
575
|
+
studyDir,
|
|
576
|
+
evidenceRef: probeBinding
|
|
577
|
+
? probeBinding.evidenceRef
|
|
578
|
+
: null,
|
|
579
|
+
expected: { studyId: frozen.studyId, suiteHash: frozen.suiteHash },
|
|
580
|
+
codexHome: probeHostKind === 'codex-cli' ? (execution.codexHome ?? null) : null,
|
|
581
|
+
hostKind: probeHostKind,
|
|
582
|
+
...(probeHostKind === 'openclaw' ? {
|
|
583
|
+
openclawControlledRoot: execution.probeControlledRoot ?? null,
|
|
584
|
+
readOpenClawTrajectory: readOpenClawExecStateTrajectory,
|
|
585
|
+
} : {}),
|
|
586
|
+
});
|
|
587
|
+
if (!probeVerify.ok) {
|
|
588
|
+
ledger.appendEvent({ type: 'UNIT_REFUSED', unit, reason: 'stage_r_probe_evidence_invalid', detail: probeVerify.reasons });
|
|
589
|
+
summary.units.push({ unit, state: 'STAGE_R_REFUSED', refused: true, reasons: probeVerify.reasons });
|
|
590
|
+
continue;
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
let actor;
|
|
595
|
+
try {
|
|
596
|
+
actor = actorFactory({
|
|
597
|
+
armId,
|
|
598
|
+
caseId,
|
|
599
|
+
repetitionId, // unit-unique taskId segment for host transports (one host session per SPEC §6.2.8 unit)
|
|
600
|
+
caseDef,
|
|
601
|
+
caseVisible,
|
|
602
|
+
armPrompt, // SPEC §6.2 — the ONLY injection channel a real actor receives
|
|
603
|
+
frozenRefs: { suiteHash: frozen.suiteHash, mappingHash: frozen.mappingHash, graderHash: frozen.graderHash },
|
|
604
|
+
});
|
|
605
|
+
} catch (e) {
|
|
606
|
+
ledger.appendEvent({ type: 'UNIT_REFUSED', unit, reason: 'actor_factory_error', detail: e.message });
|
|
607
|
+
summary.units.push({ unit, state: 'FAILED_KNOWN', refused: true, detail: `actor_factory_error: ${e.message}` });
|
|
608
|
+
continue;
|
|
609
|
+
}
|
|
610
|
+
// Mode consistency (P1-1/P1-5): the declared mode is the authority;
|
|
611
|
+
// the produced actor must agree. A real declaration refuses stub
|
|
612
|
+
// actors, executionMode mismatches and wiring whose requested model
|
|
613
|
+
// identity is not the authorization-bound actor identity; a stub
|
|
614
|
+
// declaration refuses anything that is not a declared stub. Actor
|
|
615
|
+
// self-declaration alone can never select the execution mode.
|
|
616
|
+
if (isRealMode) {
|
|
617
|
+
const modeReasons = [];
|
|
618
|
+
if (isOfflineStubActor(actor)) modeReasons.push('actor declares itself an offline stub under a real (F1/F2) execution declaration');
|
|
619
|
+
if (actor.executionMode !== declaredMode) modeReasons.push(`actor executionMode '${actor.executionMode ?? null}' does not match the runner-declared '${declaredMode}'`);
|
|
620
|
+
if (typeof actor.requestedModelIdentity !== 'string' || actor.requestedModelIdentity.length === 0) {
|
|
621
|
+
modeReasons.push('real actor must expose requestedModelIdentity (the model identity its wiring requests)');
|
|
622
|
+
} else if (actor.requestedModelIdentity !== authorizationRecord.requestedActorIdentity
|
|
623
|
+
&& !actor.requestedModelIdentity.endsWith(`/${authorizationRecord.requestedActorIdentity}`)) {
|
|
624
|
+
modeReasons.push(`actor requestedModelIdentity '${actor.requestedModelIdentity}' is not the authorization-bound requestedActorIdentity '${authorizationRecord.requestedActorIdentity}'`);
|
|
625
|
+
}
|
|
626
|
+
if (modeReasons.length > 0) {
|
|
627
|
+
ledger.appendEvent({ type: 'UNIT_REFUSED', unit, reason: 'stage_r_mode_mismatch', detail: modeReasons });
|
|
628
|
+
summary.units.push({ unit, state: 'STAGE_R_REFUSED', refused: true, reasons: modeReasons });
|
|
629
|
+
continue;
|
|
630
|
+
}
|
|
631
|
+
} else if (!isOfflineStubActor(actor)) {
|
|
632
|
+
const reasons = ['runner declared offline-stub execution but the actorFactory produced a NON-stub actor — real execution requires execution.mode (F1-direct-model | F2-real-host) and an APPROVED run-authorization.json in the study directory'];
|
|
633
|
+
ledger.appendEvent({ type: 'UNIT_REFUSED', unit, reason: 'stage_r_not_authorized', detail: reasons });
|
|
634
|
+
summary.units.push({ unit, state: 'STAGE_R_REFUSED', refused: true, reasons });
|
|
635
|
+
continue;
|
|
636
|
+
}
|
|
637
|
+
const executionMode = isRealMode ? declaredMode : null;
|
|
638
|
+
// P1-2 A: the offline runtime must actually RECEIVE and LOAD the composed
|
|
639
|
+
// arm prompt before anything is reserved. The runtime answers with its
|
|
640
|
+
// own receipt; a runtime without that capability (legacy stubs) simply
|
|
641
|
+
// produces none, and its evidence chain can never pass SUBMITTED.
|
|
642
|
+
let runtimeReceipt = null;
|
|
643
|
+
if (!isRealMode && typeof actor.loadArmPrompt === 'function') {
|
|
644
|
+
try {
|
|
645
|
+
runtimeReceipt = actor.loadArmPrompt(armPrompt);
|
|
646
|
+
} catch (e) {
|
|
647
|
+
ledger.appendEvent({ type: 'UNIT_REFUSED', unit, reason: 'offline_runtime_prompt_load_failed', detail: e.message });
|
|
648
|
+
summary.units.push({ unit, state: 'FAILED_KNOWN', refused: true, detail: `offline_runtime_prompt_load_failed: ${e.message}` });
|
|
649
|
+
continue;
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
// NOT_STARTED or RESERVED_NOT_PREPARED (idempotent reservation)
|
|
653
|
+
const reserve = ledger.reserveAttempt({ unit, maxAttemptsTotal, reason: `unit ${caseId}/${armId}/r${repetitionId}` });
|
|
654
|
+
if (!reserve.ok) {
|
|
655
|
+
ledger.appendEvent({ type: 'UNIT_REFUSED', unit, reason: reserve.code, used: reserve.used, cap: reserve.cap });
|
|
656
|
+
summary.units.push({ unit, state: 'BUDGET_EXHAUSTED', refused: true });
|
|
657
|
+
continue;
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
ledger.markPrepared(unit);
|
|
661
|
+
ledger.appendEvent({ type: 'UNIT_PREPARED', unit, suiteHash: frozen.suiteHash });
|
|
662
|
+
|
|
663
|
+
const env = buildClaimEnvironment(caseDef);
|
|
664
|
+
const runState = newClaimRunState(env);
|
|
665
|
+
const arm = armCheck.arms[armId];
|
|
666
|
+
const prompt = armPrompt;
|
|
667
|
+
if (armId === 'A' && prompt.includes(armCheck.arms.B.delivery.payloadText) && armCheck.arms.B.delivery.payloadText !== '') {
|
|
668
|
+
ledger.appendEvent({ type: 'UNIT_REFUSED', unit, reason: 'arm_a_payload_contamination' });
|
|
669
|
+
summary.units.push({ unit, state: 'CONFOUNDED', refused: true });
|
|
670
|
+
continue;
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
const linkage = {
|
|
674
|
+
suiteHash: frozen.suiteHash,
|
|
675
|
+
mappingHash: frozen.mappingHash,
|
|
676
|
+
graderHash: frozen.graderHash,
|
|
677
|
+
principleContentHash: frozen.principleContentHash,
|
|
678
|
+
armId,
|
|
679
|
+
contentMode: arm.contentMode,
|
|
680
|
+
armPayloadHash: arm.delivery.payloadHash,
|
|
681
|
+
inputHash: caseVisible.inputHash || frozen.manifest.cases.find((c) => c.caseId === caseId).inputHash,
|
|
682
|
+
};
|
|
683
|
+
|
|
684
|
+
// Real-mode authorization linkage: echoed on EVERY attempt record
|
|
685
|
+
// (SUCCEEDED, FAILED_KNOWN and ambiguous RESULT_UNKNOWN alike) so
|
|
686
|
+
// the record itself proves which authorization produced it (probe
|
|
687
|
+
// evidence included for F2) — a failed real-host attempt is still
|
|
688
|
+
// real-host evidence and must not poison deriveRealHostBaseline as
|
|
689
|
+
// linkage-missing. Stub attempts stay byte-identical to the
|
|
690
|
+
// historical Phase 1B records — no extra fields.
|
|
691
|
+
const authorizationLinkage = executionMode ? {
|
|
692
|
+
schema: authorizationRecord.schema,
|
|
693
|
+
studyId: authorizationRecord.studyId,
|
|
694
|
+
suiteHash: authorizationRecord.suiteHash,
|
|
695
|
+
executionMode,
|
|
696
|
+
issuedBy: authorizationRecord.issuedBy,
|
|
697
|
+
isolationAcceptance: authorizationRecord.isolationAcceptance || null,
|
|
698
|
+
} : null;
|
|
699
|
+
|
|
700
|
+
// P1-7 request-level budget: real actors may answer over MANY model
|
|
701
|
+
// requests per attempt; each request reserves one ledger slot
|
|
702
|
+
// (auth.requestCap) BEFORE dispatch via the loop hook. A refusal
|
|
703
|
+
// never reaches the actor (the request is not sent), and dispatched
|
|
704
|
+
// but unanswered requests keep their slot (RESULT_UNKNOWN, no
|
|
705
|
+
// auto-retry). Stub runs pass no hook — zero behavioral change.
|
|
706
|
+
// Per-attempt cap (authorization-bound, gate-checked): when
|
|
707
|
+
// execution.maxRequestsPerAttempt is declared and approved, a step
|
|
708
|
+
// at/above the cap refuses its reservation — the loop terminates
|
|
709
|
+
// (request_budget_exhausted) BEFORE the request is sent, so one
|
|
710
|
+
// attempt can never exceed the approved per-attempt request budget.
|
|
711
|
+
const perAttemptCap = isRealMode && execution.maxRequestsPerAttempt !== undefined ? execution.maxRequestsPerAttempt : null;
|
|
712
|
+
const reserveRequest = isRealMode
|
|
713
|
+
? (step) => {
|
|
714
|
+
if (perAttemptCap !== null && step >= perAttemptCap) {
|
|
715
|
+
return { ok: false, reason: `per_attempt_request_cap: step ${step} reached the approved per-attempt request cap ${perAttemptCap} — request NOT sent; the attempt settles bounded` };
|
|
716
|
+
}
|
|
717
|
+
return ledger.reserveModelRequest({
|
|
718
|
+
key: `${attemptId}:${caseId}/${armId}/r${repetitionId}:step${step}`,
|
|
719
|
+
unit,
|
|
720
|
+
maxRequestsTotal: gate.requestCap,
|
|
721
|
+
reason: `model request (unit ${caseId}/${armId}/r${repetitionId} step ${step})`,
|
|
722
|
+
});
|
|
723
|
+
}
|
|
724
|
+
: null;
|
|
725
|
+
|
|
726
|
+
try {
|
|
727
|
+
const { trace } = await runControlledActorLoop({ env, runState, caseVisible, actor, linkage, reserveRequest, runtimeReceipt });
|
|
728
|
+
const adjudication = adjudicate({ caseDef, runState });
|
|
729
|
+
|
|
730
|
+
const attempt = {
|
|
731
|
+
schema: 'challenge-attempt.v1',
|
|
732
|
+
unit,
|
|
733
|
+
mode: executionMode || 'offline-stub',
|
|
734
|
+
...(executionMode ? {
|
|
735
|
+
authorization: authorizationLinkage,
|
|
736
|
+
observedHostEvidence: readObservedHostEvidence(actor),
|
|
737
|
+
actorRounds: Array.isArray(actor.roundLog) ? actor.roundLog.slice() : [],
|
|
738
|
+
} : {}),
|
|
739
|
+
actorKind: trace.actorKind,
|
|
740
|
+
frozenRefs: linkage,
|
|
741
|
+
promptAudit: { armId, contentMode: arm.contentMode, payloadHash: arm.delivery.payloadHash, promptHash: sha256(prompt), promptLength: prompt.length },
|
|
742
|
+
actorInputHash: linkage.inputHash,
|
|
743
|
+
trace,
|
|
744
|
+
adjudication,
|
|
745
|
+
result: 'SUCCEEDED',
|
|
746
|
+
note: 'trace is actor-side evidence (implies stripped); adjudication is the judge-side record',
|
|
747
|
+
};
|
|
748
|
+
const attemptDir = path.join(studyDir, 'runs', caseId, armId, `r${repetitionId}`);
|
|
749
|
+
fs.mkdirSync(attemptDir, { recursive: true });
|
|
750
|
+
const attemptFile = path.join(attemptDir, `${attemptId}.json`);
|
|
751
|
+
const tmp = `${attemptFile}.tmp`;
|
|
752
|
+
fs.writeFileSync(tmp, `${JSON.stringify(attempt, null, 2)}\n`, 'utf8');
|
|
753
|
+
fs.renameSync(tmp, attemptFile);
|
|
754
|
+
|
|
755
|
+
// v0.9 §7.2 injection evidence (offline-stub scope): a SEPARATE
|
|
756
|
+
// tamper-evident sidecar — the attempt record itself stays
|
|
757
|
+
// byte-identical to the historical Phase 1B shape. The chain is
|
|
758
|
+
// produced by the RUNNER from the RUNTIME's own load receipt, and is
|
|
759
|
+
// bound to this exact study/unit/attempt: without a receipt it stops at
|
|
760
|
+
// SELECTED instead of claiming a load nobody observed.
|
|
761
|
+
let offlineChainInfo = null;
|
|
762
|
+
if (!isRealMode) {
|
|
763
|
+
const chain = collectOfflineStubInjectionEvidence({
|
|
764
|
+
studyId: frozen.studyId,
|
|
765
|
+
unit: { caseId, armId, repetitionId, attemptId },
|
|
766
|
+
arm: armCheck.arms[armId],
|
|
767
|
+
armPrompt: prompt,
|
|
768
|
+
receipt: runtimeReceipt,
|
|
769
|
+
attemptRef: `runs/${caseId}/${armId}/r${repetitionId}/${attemptId}.json`,
|
|
770
|
+
// Control arms of dual-mode studies must never carry the TARGET
|
|
771
|
+
// principle text (the library itself is legitimate in mode B).
|
|
772
|
+
forbiddenContent: armId === 'A' && armCheck.examinerTargetText ? armCheck.examinerTargetText : null,
|
|
773
|
+
});
|
|
774
|
+
fs.writeFileSync(path.join(attemptDir, `injection-evidence-${attemptId}.json`), `${JSON.stringify(chain, null, 2)}\n`, 'utf8');
|
|
775
|
+
offlineChainInfo = { provenLevel: chain.provenLevel, chainHash: chain.chainHash, scope: chain.scope };
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
ledger.markResult(unit, 'SUCCEEDED', adjudication.objectiveTaskOutcome);
|
|
779
|
+
ledger.appendAdjudication({ unit, suiteHash: frozen.suiteHash, objectiveTaskOutcome: adjudication.objectiveTaskOutcome, adjudication });
|
|
780
|
+
ledger.appendEvent({ type: 'UNIT_FINISHED', unit, result: 'SUCCEEDED', objectiveTaskOutcome: adjudication.objectiveTaskOutcome });
|
|
781
|
+
summary.units.push({ unit, state: 'SUCCEEDED', objectiveTaskOutcome: adjudication.objectiveTaskOutcome, ...(offlineChainInfo ? { injectionEvidence: offlineChainInfo } : {}) });
|
|
782
|
+
consecutiveUnknown = 0; // a definite execution settlement breaks the ambiguity streak
|
|
783
|
+
consecutiveCapacityFailures = 0; // and the capacity streak (the channel answered fine)
|
|
784
|
+
} catch (e) {
|
|
785
|
+
if (e instanceof SimulatedCrashError) {
|
|
786
|
+
// Process-death simulation: no result row, no attempt file, lock
|
|
787
|
+
// stays held — the unit remains PREPARED-without-RESULT.
|
|
788
|
+
ledger.appendEvent({ type: 'UNIT_INTERRUPTED', unit, detail: e.message });
|
|
789
|
+
summary.interrupted = { unit, at: e.atStep };
|
|
790
|
+
throw e;
|
|
791
|
+
}
|
|
792
|
+
const attemptDir = path.join(studyDir, 'runs', caseId, armId, `r${repetitionId}`);
|
|
793
|
+
fs.mkdirSync(attemptDir, { recursive: true });
|
|
794
|
+
if (e && e.ambiguous === true) {
|
|
795
|
+
// Real-transport ambiguity (SPEC §7.4): the request may have
|
|
796
|
+
// reached the model/host; no usable result exists. Settle the
|
|
797
|
+
// attempt EXPLICITLY as RESULT_UNKNOWN — never success, never
|
|
798
|
+
// clean failure, never auto-retried here. The reservation
|
|
799
|
+
// (attempt AND dispatched request slots) stays consumed;
|
|
800
|
+
// recovery of this attemptId is refused and only a NEW
|
|
801
|
+
// attemptId may reserve again. The record still carries the
|
|
802
|
+
// authorization linkage + observed evidence (merged #2139
|
|
803
|
+
// discipline: every real-mode record proves its authorization).
|
|
804
|
+
const detail = `result_unknown (ambiguous transport outcome): ${e.message}`;
|
|
805
|
+
ledger.markResult(unit, 'RESULT_UNKNOWN', detail);
|
|
806
|
+
fs.writeFileSync(path.join(attemptDir, `${attemptId}.json`), `${JSON.stringify({
|
|
807
|
+
schema: 'challenge-attempt.v1',
|
|
808
|
+
unit,
|
|
809
|
+
mode: executionMode || 'offline-stub',
|
|
810
|
+
...(executionMode ? {
|
|
811
|
+
authorization: authorizationLinkage,
|
|
812
|
+
observedHostEvidence: readObservedHostEvidence(actor),
|
|
813
|
+
actorRounds: Array.isArray(actor.roundLog) ? actor.roundLog.slice() : [],
|
|
814
|
+
} : {}),
|
|
815
|
+
result: 'RESULT_UNKNOWN',
|
|
816
|
+
ambiguous: true,
|
|
817
|
+
error: detail,
|
|
818
|
+
}, null, 2)}\n`, 'utf8');
|
|
819
|
+
ledger.appendEvent({ type: 'UNIT_FINISHED', unit, result: 'RESULT_UNKNOWN', detail });
|
|
820
|
+
summary.units.push({ unit, state: 'RESULT_UNKNOWN', ambiguous: true, detail });
|
|
821
|
+
consecutiveUnknown += 1; // an ambiguous settlement extends the streak (dispatch stop at the limit)
|
|
822
|
+
continue;
|
|
823
|
+
}
|
|
824
|
+
const failureDetail = `execution_error: ${e.message}`;
|
|
825
|
+
ledger.markResult(unit, 'FAILED_KNOWN', failureDetail);
|
|
826
|
+
fs.writeFileSync(path.join(attemptDir, `${attemptId}.json`), `${JSON.stringify({
|
|
827
|
+
schema: 'challenge-attempt.v1',
|
|
828
|
+
unit,
|
|
829
|
+
mode: executionMode || 'offline-stub',
|
|
830
|
+
...(executionMode ? {
|
|
831
|
+
authorization: authorizationLinkage,
|
|
832
|
+
observedHostEvidence: readObservedHostEvidence(actor),
|
|
833
|
+
actorRounds: Array.isArray(actor.roundLog) ? actor.roundLog.slice() : [],
|
|
834
|
+
} : {}),
|
|
835
|
+
result: 'FAILED_KNOWN',
|
|
836
|
+
error: failureDetail,
|
|
837
|
+
}, null, 2)}\n`, 'utf8');
|
|
838
|
+
ledger.appendEvent({ type: 'UNIT_FINISHED', unit, result: 'FAILED_KNOWN', detail: failureDetail });
|
|
839
|
+
summary.units.push({ unit, state: 'FAILED_KNOWN', detail: failureDetail });
|
|
840
|
+
consecutiveUnknown = 0; // a definite execution settlement breaks the ambiguity streak
|
|
841
|
+
// capacity breaker: ONLY proven 429/5xx failures arm it; every
|
|
842
|
+
// other known failure (identity mismatch, output contract,
|
|
843
|
+
// authorization-class problems…) breaks the streak — they are
|
|
844
|
+
// NOT channel saturation and must surface as hard failures.
|
|
845
|
+
if (isCapacityFailureDetail(failureDetail)) {
|
|
846
|
+
consecutiveCapacityFailures += 1;
|
|
847
|
+
} else {
|
|
848
|
+
consecutiveCapacityFailures = 0;
|
|
849
|
+
}
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
// v0.9 §3.3: the ACTUAL dispatch order as executed (not the intention) —
|
|
853
|
+
// the record the isolation contract's randomization check consumes.
|
|
854
|
+
ledger.appendEvent({ type: 'RUN_FINISHED', owner, dispatchedOrder: summary.units.map((u) => `${u.unit.caseId}/${u.unit.armId}/r${u.unit.repetitionId}`) });
|
|
855
|
+
releaseStudyWriteLock(studyDir, { owner });
|
|
856
|
+
return summary;
|
|
857
|
+
} catch (e) {
|
|
858
|
+
if (e instanceof SimulatedCrashError) throw e; // crash path: lock intentionally left held
|
|
859
|
+
try { releaseStudyWriteLock(studyDir, { owner }); } catch { /* already released or held by other owner */ }
|
|
860
|
+
throw e;
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
/** Resume entrypoint: same as runChallengeStudy but refuses to start when the
|
|
865
|
+
* lock is stale-held by a crashed writer unless released by that owner. */
|
|
866
|
+
export function runChallengeStudyResume(options) {
|
|
867
|
+
return runChallengeStudy(options);
|
|
868
|
+
}
|