chati-dev 4.5.16 → 4.5.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -3
- package/bin/chati.js +124 -66
- package/framework/agents/build/dev.md +5 -5
- package/framework/agents/deploy/devops.md +7 -7
- package/framework/agents/discover/brief.md +4 -4
- package/framework/agents/discover/brownfield-wu.md +3 -3
- package/framework/agents/discover/greenfield-wu.md +3 -3
- package/framework/agents/plan/architect.md +2 -2
- package/framework/agents/plan/detail.md +4 -4
- package/framework/agents/plan/phases.md +2 -2
- package/framework/agents/plan/tasks.md +2 -2
- package/framework/agents/plan/ux.md +2 -2
- package/framework/agents/quality/qa-implementation.md +11 -10
- package/framework/agents/quality/qa-planning.md +3 -3
- package/framework/agents/quality/qa-visual.md +1 -1
- package/framework/config.yaml +3 -3
- package/framework/constitution.md +18 -18
- package/framework/context/protocols.md +1 -1
- package/framework/context/quality.md +1 -1
- package/framework/context/root.md +4 -4
- package/framework/data/entity-registry.yaml +1 -1
- package/framework/domains/agents/orchestrator.yaml +2 -2
- package/framework/domains/constitution.yaml +1 -1
- package/framework/hooks/advance-trigger.js +4 -6
- package/framework/hooks/git-push-authority.js +45 -37
- package/framework/hooks/mode-governance.js +149 -40
- package/framework/hooks/model-governance.js +13 -20
- package/framework/hooks/prism-engine.js +74 -92
- package/framework/hooks/reasoning-escalator.js +23 -40
- package/framework/hooks/session-digest.js +12 -13
- package/framework/hooks/session-reader.js +224 -0
- package/framework/hooks/session-writer.js +195 -0
- package/framework/hooks/team-quality-gate.js +34 -24
- package/framework/i18n/en.yaml +2 -2
- package/framework/i18n/es.yaml +2 -2
- package/framework/i18n/fr.yaml +2 -2
- package/framework/i18n/pt.yaml +2 -2
- package/framework/intelligence/context-engine.md +4 -4
- package/framework/intelligence/memory-layer.md +1 -1
- package/framework/manifest.json +129 -119
- package/framework/manifest.sig +1 -1
- package/framework/orchestrator/chati-router.js +252 -25
- package/framework/orchestrator/chati.md +238 -70
- package/framework/schemas/session.schema.json +1 -1
- package/framework/tasks/orchestrator-deviation.md +1 -1
- package/framework/tasks/orchestrator-escalate.md +1 -1
- package/framework/tasks/orchestrator-handoff.md +6 -6
- package/framework/tasks/orchestrator-health.md +5 -9
- package/framework/tasks/orchestrator-mode-switch.md +3 -7
- package/framework/tasks/orchestrator-resume.md +10 -14
- package/framework/tasks/orchestrator-route.md +3 -3
- package/framework/tasks/orchestrator-spawn-terminal.md +1 -1
- package/framework/tasks/orchestrator-status.md +9 -9
- package/framework/tasks/orchestrator-suggest-mode.md +1 -1
- package/framework/tasks/qa-impl-consolidate.md +2 -2
- package/framework/tasks/qa-impl-performance-test.md +4 -4
- package/framework/tasks/qa-impl-regression-check.md +4 -4
- package/framework/tasks/qa-impl-sast-scan.md +1 -1
- package/framework/tasks/qa-impl-test-execute.md +1 -1
- package/framework/tasks/qa-impl-verdict.md +2 -2
- package/framework/tasks/qa-planning-consolidate.md +3 -3
- package/framework/tasks/qa-planning-coverage-plan.md +2 -2
- package/framework/tasks/qa-planning-gate-define.md +4 -4
- package/framework/tasks/qa-planning-risk-matrix.md +4 -4
- package/framework/tasks/qa-planning-test-strategy.md +2 -2
- package/node_modules/@chati/browser-capability/src/index.js +12 -2
- package/node_modules/@chati/planning/src/index.js +24 -4
- package/node_modules/@chati/provider-registry/src/index.js +11 -0
- package/node_modules/@chati/rail/src/index.js +1967 -83
- package/node_modules/@chati/release-lane/README.md +12 -8
- package/node_modules/@chati/release-lane/package.json +1 -1
- package/node_modules/@chati/release-lane/src/index.js +1426 -58
- package/node_modules/@chati/review-council/src/index.js +63 -0
- package/node_modules/@chati/tracking-clickup/README.md +13 -0
- package/node_modules/@chati/tracking-clickup/src/index.js +690 -30
- package/package-artifact-manifest.json +1 -0
- package/package-artifact-manifest.sig +1 -0
- package/package.json +16 -7
- package/scripts/verify-real-harness-e2e.js +1581 -0
- package/src/config/framework-adapter.js +4 -4
- package/src/installer/core.js +148 -53
- package/src/installer/manifest.js +140 -9
- package/src/installer/package-artifact.js +249 -0
- package/src/installer/templates.js +65 -20
- package/src/installer-v2/catalog-client.js +531 -23
- package/src/installer-v2/index.js +91 -34
- package/src/installer-v2/installation-authority.js +327 -0
- package/src/installer-v2/provider-executable.js +247 -0
- package/src/installer-v2/wizard-installation.js +1 -1
- package/src/orchestrator/browser-runtime.js +44 -13
- package/src/orchestrator/cli.js +1545 -174
- package/src/orchestrator/clickup-projection.js +43 -1
- package/src/orchestrator/clickup-runtime.js +355 -39
- package/src/orchestrator/doctor.js +16 -3
- package/src/orchestrator/index.js +16 -0
- package/src/orchestrator/planning-runtime.js +20 -2
- package/src/orchestrator/rail-adjudication-evidence.js +234 -0
- package/src/orchestrator/rail-evidence-authority.js +147 -0
- package/src/orchestrator/rail-execution-evidence.js +45 -0
- package/src/orchestrator/rail-runtime.js +836 -56
- package/src/orchestrator/release-runtime.js +65 -6
- package/src/orchestrator/review-runtime.js +186 -41
- package/src/orchestrator/runtime-installation-v2.js +236 -20
- package/src/orchestrator/session-manager.js +1167 -53
- package/src/terminal/adapters/claude-adapter.js +3 -1
- package/src/terminal/adapters/codex-adapter.js +2 -0
- package/src/terminal/adapters/grok-adapter.js +7 -4
- package/src/terminal/handoff-parser.js +19 -1
- package/src/terminal/prompt-builder.js +9 -1
- package/src/terminal/provider-preflight.js +36 -3
- package/src/terminal/rail-execution-worktree.js +213 -0
- package/src/terminal/rail-prompts.js +169 -0
- package/src/terminal/rail-readonly-workspace.js +324 -0
- package/src/terminal/run-agent.js +384 -24
- package/src/terminal/run-parallel.js +5 -0
- package/src/terminal/run-rail-adjudication.js +323 -0
- package/src/terminal/run-rail-review.js +291 -0
- package/src/terminal/run-rail-rework.js +325 -0
- package/src/terminal/run-rail-task.js +380 -0
- package/src/terminal/run-team.js +5 -0
- package/src/terminal/spawner.js +1225 -77
- package/src/wizard/index.js +3 -2
|
@@ -1,8 +1,14 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
1
|
+
import {
|
|
2
|
+
closeSync, constants, existsSync, fstatSync, fsyncSync, ftruncateSync, lstatSync, mkdirSync,
|
|
3
|
+
openSync, readSync, renameSync, unlinkSync, writeSync,
|
|
4
|
+
} from 'node:fs';
|
|
5
|
+
import { hostname } from 'node:os';
|
|
6
|
+
import { randomUUID } from 'node:crypto';
|
|
7
|
+
import { execFileSync } from 'node:child_process';
|
|
8
|
+
import { basename, dirname, join, parse, resolve, sep } from 'node:path';
|
|
3
9
|
import { ContractError, canonicalize, sha256 } from '@chati/core';
|
|
4
10
|
import { createAttemptBinding, validateAttemptBinding, verifyLocalGitCommit } from '@chati/planning';
|
|
5
|
-
import { acceptReview, decideReviewProgress } from '@chati/review-council';
|
|
11
|
+
import { acceptReview, decideReviewProgress, selectCanonicalReviewer, validateAdjudicationDecision } from '@chati/review-council';
|
|
6
12
|
|
|
7
13
|
const RFC3339 = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/;
|
|
8
14
|
const TASK_CLOSED_STATES = new Set(['merged', 'blocked', 'cancelled']);
|
|
@@ -10,11 +16,40 @@ const EXECUTABLE_ATTEMPT_STATE = 'active';
|
|
|
10
16
|
const REVIEW_DECISIONS = new Set(['accept', 'normal-rework', 'AI_ADJUDICATION', 'blocked', 'required-human-decision']);
|
|
11
17
|
const LOCK_RETRIES = 50;
|
|
12
18
|
const LOCK_WAIT = new Int32Array(new SharedArrayBuffer(4));
|
|
19
|
+
const LOCK_STALE_AFTER_MS = 30_000;
|
|
20
|
+
const JOURNAL_MAX_BYTES = 64 * 1024 * 1024;
|
|
21
|
+
const JOURNAL_RECORD_MAX_BYTES = 4 * 1024 * 1024;
|
|
22
|
+
const RAIL_SECURITY_TEST_HOOKS = Symbol.for('chati.rail.security-test-hooks');
|
|
23
|
+
const BROWSER_ARTIFACT_KINDS = new Set(['screenshot', 'dom', 'network', 'trace', 'download', 'console']);
|
|
24
|
+
const BROWSER_EVIDENCE_KEYS = new Set([
|
|
25
|
+
'schema_version', 'manifest_type', 'evidence_class', 'browser_session_id', 'task_id',
|
|
26
|
+
'policy_ref', 'tool', 'allowed_domain', 'classification', 'commit_ref',
|
|
27
|
+
'delegation_manifest_hash', 'observed_at', 'artifacts', 'evidence_ref', 'authority_receipt',
|
|
28
|
+
'tenant_ref',
|
|
29
|
+
]);
|
|
30
|
+
const BROWSER_ARTIFACT_KEYS = new Set([
|
|
31
|
+
'artifact_id', 'kind', 'sanitized_ref', 'redaction_status', 'retention_class', 'content_sha256',
|
|
32
|
+
]);
|
|
33
|
+
|
|
34
|
+
function runSecurityTestHook(name, detail) {
|
|
35
|
+
const hook = globalThis[RAIL_SECURITY_TEST_HOOKS]?.[name];
|
|
36
|
+
if (typeof hook === 'function') hook(detail);
|
|
37
|
+
}
|
|
13
38
|
|
|
14
39
|
function fail(code, message, details = {}) {
|
|
15
40
|
throw new ContractError(code, message, details);
|
|
16
41
|
}
|
|
17
42
|
|
|
43
|
+
function trustedGitEnvironment(source = process.env) {
|
|
44
|
+
const environment = Object.fromEntries(Object.entries(source).filter(([key]) => !key.startsWith('GIT_')));
|
|
45
|
+
return {
|
|
46
|
+
...environment,
|
|
47
|
+
GIT_CONFIG_NOSYSTEM: '1',
|
|
48
|
+
GIT_CONFIG_GLOBAL: process.platform === 'win32' ? 'NUL' : '/dev/null',
|
|
49
|
+
GIT_TERMINAL_PROMPT: '0',
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
|
|
18
53
|
function assertObject(value, code, label) {
|
|
19
54
|
if (!value || typeof value !== 'object' || Array.isArray(value)) fail(code, `${label} must be an object`);
|
|
20
55
|
}
|
|
@@ -29,6 +64,197 @@ function assertStringArray(value, code, label, { allowEmpty = true } = {}) {
|
|
|
29
64
|
}
|
|
30
65
|
}
|
|
31
66
|
|
|
67
|
+
function assertAllowedKeys(value, allowed, code, label) {
|
|
68
|
+
if (Object.keys(value).some((key) => !allowed.has(key))) fail(code, `${label} contains unsupported fields`);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function browserEvidenceMaterial(evidence = {}) {
|
|
72
|
+
return canonicalize({
|
|
73
|
+
schema_version: evidence.schema_version,
|
|
74
|
+
manifest_type: evidence.manifest_type,
|
|
75
|
+
evidence_class: evidence.evidence_class,
|
|
76
|
+
browser_session_id: evidence.browser_session_id,
|
|
77
|
+
task_id: evidence.task_id,
|
|
78
|
+
policy_ref: evidence.policy_ref,
|
|
79
|
+
tool: evidence.tool,
|
|
80
|
+
allowed_domain: evidence.allowed_domain,
|
|
81
|
+
classification: evidence.classification,
|
|
82
|
+
commit_ref: evidence.commit_ref,
|
|
83
|
+
delegation_manifest_hash: evidence.delegation_manifest_hash,
|
|
84
|
+
observed_at: evidence.observed_at,
|
|
85
|
+
artifacts: evidence.artifacts,
|
|
86
|
+
...(evidence.tenant_ref === undefined ? {} : { tenant_ref: evidence.tenant_ref }),
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function validateRealBrowserEvidence({
|
|
91
|
+
evidence, task, attemptId, commitRef, acceptanceEvidenceRefs, delegationManifest, verifyAuthority,
|
|
92
|
+
}) {
|
|
93
|
+
assertObject(evidence, 'REAL_BROWSER_EVIDENCE_REQUIRED', 'browser_evidence');
|
|
94
|
+
assertAllowedKeys(evidence, BROWSER_EVIDENCE_KEYS, 'INVALID_BROWSER_EVIDENCE', 'browser_evidence');
|
|
95
|
+
if (evidence.schema_version !== 1
|
|
96
|
+
|| evidence.manifest_type !== 'BrowserRuntimeEvidenceManifest'
|
|
97
|
+
|| evidence.evidence_class !== 'real-browser') {
|
|
98
|
+
fail('REAL_BROWSER_EVIDENCE_REQUIRED', 'browser completion requires evidence classified as real-browser, not delegation or simulation');
|
|
99
|
+
}
|
|
100
|
+
for (const field of [
|
|
101
|
+
'browser_session_id', 'task_id', 'policy_ref', 'tool', 'allowed_domain', 'classification',
|
|
102
|
+
'commit_ref', 'delegation_manifest_hash', 'observed_at', 'evidence_ref',
|
|
103
|
+
]) assertString(evidence[field], 'INVALID_BROWSER_EVIDENCE', `browser_evidence.${field}`);
|
|
104
|
+
if (!RFC3339.test(evidence.observed_at) || Number.isNaN(Date.parse(evidence.observed_at))) {
|
|
105
|
+
fail('INVALID_BROWSER_EVIDENCE', 'browser_evidence.observed_at must be RFC3339');
|
|
106
|
+
}
|
|
107
|
+
if (!Array.isArray(evidence.artifacts) || evidence.artifacts.length === 0) {
|
|
108
|
+
fail('REAL_BROWSER_ARTIFACT_REQUIRED', 'real browser evidence requires at least one redacted artifact');
|
|
109
|
+
}
|
|
110
|
+
const artifactIds = new Set();
|
|
111
|
+
for (const artifact of evidence.artifacts) {
|
|
112
|
+
assertObject(artifact, 'INVALID_BROWSER_EVIDENCE', 'browser artifact');
|
|
113
|
+
if (Object.keys(artifact).length !== BROWSER_ARTIFACT_KEYS.size) {
|
|
114
|
+
fail('INVALID_BROWSER_EVIDENCE', 'browser artifact has an invalid closed schema');
|
|
115
|
+
}
|
|
116
|
+
assertAllowedKeys(artifact, BROWSER_ARTIFACT_KEYS, 'INVALID_BROWSER_EVIDENCE', 'browser artifact');
|
|
117
|
+
for (const field of ['artifact_id', 'kind', 'sanitized_ref', 'redaction_status', 'retention_class', 'content_sha256']) {
|
|
118
|
+
assertString(artifact[field], 'INVALID_BROWSER_EVIDENCE', `browser artifact.${field}`);
|
|
119
|
+
}
|
|
120
|
+
if (!BROWSER_ARTIFACT_KINDS.has(artifact.kind)
|
|
121
|
+
|| artifact.redaction_status !== 'redacted'
|
|
122
|
+
|| artifact.retention_class !== task.browser_policy.artifact_policy.retention_class
|
|
123
|
+
|| !/^[a-f0-9]{64}$/.test(artifact.content_sha256)
|
|
124
|
+
|| artifactIds.has(artifact.artifact_id)) {
|
|
125
|
+
fail('INVALID_BROWSER_EVIDENCE', 'browser artifact kind, redaction, retention, digest, and identity must be valid');
|
|
126
|
+
}
|
|
127
|
+
artifactIds.add(artifact.artifact_id);
|
|
128
|
+
if (!acceptanceEvidenceRefs.includes(artifact.sanitized_ref)) {
|
|
129
|
+
fail('BROWSER_ARTIFACT_NOT_ACCEPTED', 'every real browser artifact must be present in acceptance_evidence_refs');
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
assertObject(delegationManifest, 'BROWSER_DELEGATION_EVIDENCE_REQUIRED', 'browser delegation manifest');
|
|
133
|
+
const delegationCore = { ...delegationManifest };
|
|
134
|
+
delete delegationCore.manifest_hash;
|
|
135
|
+
if (delegationManifest.manifest_type !== 'BrowserDelegationManifest'
|
|
136
|
+
|| delegationManifest.evidence_class !== 'delegation-only'
|
|
137
|
+
|| delegationManifest.delegation?.status !== 'delegated'
|
|
138
|
+
|| delegationManifest.delegation?.evidence_status !== 'required'
|
|
139
|
+
|| delegationManifest.manifest_hash !== sha256(delegationCore)) {
|
|
140
|
+
fail('INVALID_BROWSER_DELEGATION_EVIDENCE', 'browser evidence requires the exact canonical delegated browser manifest');
|
|
141
|
+
}
|
|
142
|
+
const policy = task.browser_policy;
|
|
143
|
+
if (evidence.task_id !== task.task_id
|
|
144
|
+
|| evidence.task_id !== delegationManifest.task_id
|
|
145
|
+
|| evidence.browser_session_id !== delegationManifest.browser_session_id
|
|
146
|
+
|| evidence.policy_ref !== policy.policy_id
|
|
147
|
+
|| evidence.policy_ref !== delegationManifest.policy_ref
|
|
148
|
+
|| evidence.tool !== policy.tool
|
|
149
|
+
|| evidence.tool !== delegationManifest.tool
|
|
150
|
+
|| evidence.allowed_domain !== delegationManifest.allowed_domain
|
|
151
|
+
|| evidence.classification !== delegationManifest.classification
|
|
152
|
+
|| evidence.commit_ref !== commitRef
|
|
153
|
+
|| evidence.delegation_manifest_hash !== delegationManifest.manifest_hash
|
|
154
|
+
|| (delegationManifest.tenant_ref ?? null) !== (evidence.tenant_ref ?? null)) {
|
|
155
|
+
fail('BROWSER_EVIDENCE_BINDING_MISMATCH', 'real browser evidence is not bound to the task, delegation, project data class, and result commit');
|
|
156
|
+
}
|
|
157
|
+
const material = browserEvidenceMaterial(evidence);
|
|
158
|
+
const evidenceRef = `browser-evidence://sha256/${sha256(material)}`;
|
|
159
|
+
if (evidence.evidence_ref !== evidenceRef || !acceptanceEvidenceRefs.includes(evidenceRef)) {
|
|
160
|
+
fail('BROWSER_EVIDENCE_REF_MISMATCH', 'real browser evidence must use its content-addressed acceptance reference');
|
|
161
|
+
}
|
|
162
|
+
assertObject(evidence.authority_receipt, 'UNTRUSTED_BROWSER_EVIDENCE', 'browser_evidence.authority_receipt');
|
|
163
|
+
verifyAuthority(evidence.authority_receipt, {
|
|
164
|
+
purpose: 'rail_evidence', authority_kind: 'rail_browser', attempt_id: attemptId,
|
|
165
|
+
commit_ref: commitRef, verdict: 'passed', evidence_digest: sha256(material),
|
|
166
|
+
observed_at: evidence.observed_at,
|
|
167
|
+
});
|
|
168
|
+
return canonicalize(evidence);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function parseAdjudicationOutput(text) {
|
|
172
|
+
if (typeof text !== 'string') fail('INVALID_ADJUDICATION_RECORD', 'adjudication_output must be a string');
|
|
173
|
+
const matches = [...text.matchAll(/<CHATI_RAIL_ADJUDICATION>\s*([\s\S]*?)\s*<\/CHATI_RAIL_ADJUDICATION>/g)];
|
|
174
|
+
if (matches.length !== 1) fail('INVALID_ADJUDICATION_RECORD', 'adjudication_output must contain exactly one envelope');
|
|
175
|
+
try { return validateAdjudicationDecision(JSON.parse(matches[0][1])); }
|
|
176
|
+
catch (error) { return fail(error.code || 'INVALID_ADJUDICATION_RECORD', error.message); }
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
export function deriveAdjudicationConsensus({ handoff_id, attempt_id, evidence }) {
|
|
180
|
+
if (!adjudicationEntriesAgree(evidence)) {
|
|
181
|
+
fail('ADJUDICATION_CONSENSUS_MISMATCH', 'adjudicators must agree on both the outcome and its substantive correction material');
|
|
182
|
+
}
|
|
183
|
+
const digest = sha256(evidence.map((entry) => ({ invocation: entry.invocation, decision: entry.decision })));
|
|
184
|
+
const outcome = evidence[0].decision.outcome;
|
|
185
|
+
const decision_ref = `rail-adjudication-consensus://${handoff_id}/${attempt_id}/${digest.slice(0, 24)}`;
|
|
186
|
+
if (outcome === 'bounded-correction') {
|
|
187
|
+
return validateAdjudicationDecision({ outcome, decision_ref, correction_ref: `rail-adjudication-correction://${handoff_id}/${attempt_id}/${digest.slice(0, 24)}` });
|
|
188
|
+
}
|
|
189
|
+
if (outcome === 'handoff-revision') {
|
|
190
|
+
return validateAdjudicationDecision({ outcome, decision_ref, handoff_revision_ref: `rail-handoff-revision://${handoff_id}/${attempt_id}/${digest.slice(0, 24)}` });
|
|
191
|
+
}
|
|
192
|
+
return validateAdjudicationDecision({
|
|
193
|
+
outcome, decision_ref, authority_reason: evidence[0].decision.authority_reason,
|
|
194
|
+
authority_reason_ref: `rail-adjudication-authority://${handoff_id}/${attempt_id}/${digest.slice(0, 24)}`,
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
export function adjudicationEntriesAgree(entries) {
|
|
199
|
+
if (entries.length === 0) return false;
|
|
200
|
+
const outcome = entries[0].decision.outcome;
|
|
201
|
+
if (!entries.every((entry) => entry.decision.outcome === outcome)) return false;
|
|
202
|
+
const substantiveField = {
|
|
203
|
+
'bounded-correction': 'correction_ref',
|
|
204
|
+
'handoff-revision': 'handoff_revision_ref',
|
|
205
|
+
'blocked-authority': 'authority_reason',
|
|
206
|
+
}[outcome];
|
|
207
|
+
if (!substantiveField) return false;
|
|
208
|
+
return entries.every((entry) => entry.decision[substantiveField] === entries[0].decision[substantiveField]);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
export function adjudicationAttemptState(decision) {
|
|
212
|
+
if (decision?.outcome === 'handoff-revision') return 'cancelled';
|
|
213
|
+
if (decision?.outcome === 'blocked-authority') return 'blocked';
|
|
214
|
+
return 'active';
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function unresolvedAdjudication(records, attemptId) {
|
|
218
|
+
const latestControl = records.filter((record) => ['review_decided', 'adjudication_decided'].includes(record.type)
|
|
219
|
+
&& record.payload?.attempt_id === attemptId).at(-1);
|
|
220
|
+
return latestControl?.type === 'review_decided'
|
|
221
|
+
&& latestControl.payload?.decision?.decision === 'AI_ADJUDICATION'
|
|
222
|
+
? latestControl
|
|
223
|
+
: null;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function adjudicationRoundState(records, attemptId, reviewRef, selected) {
|
|
227
|
+
const observations = records
|
|
228
|
+
.filter((record) => record.type === 'adjudication_observed'
|
|
229
|
+
&& record.payload?.attempt_id === attemptId
|
|
230
|
+
&& record.payload?.review_ref === reviewRef)
|
|
231
|
+
.map((record) => record.payload.observation);
|
|
232
|
+
const orderedRound = (round) => selected.map((candidate) => observations.find((entry) =>
|
|
233
|
+
entry.round === round
|
|
234
|
+
&& entry.invocation.provider_id === candidate.provider_id
|
|
235
|
+
&& entry.invocation.harness_id === candidate.harness_id
|
|
236
|
+
&& sha256(entry.invocation.model_pin) === sha256(candidate.model_pin))).filter(Boolean);
|
|
237
|
+
const roundOne = orderedRound(1);
|
|
238
|
+
const roundTwo = orderedRound(2);
|
|
239
|
+
if (roundOne.length < selected.length) {
|
|
240
|
+
return { round: 1, observations, current: roundOne, prior_decisions: [], complete: false };
|
|
241
|
+
}
|
|
242
|
+
if (adjudicationEntriesAgree(roundOne)) {
|
|
243
|
+
return { round: 1, observations, current: roundOne, prior_decisions: [], complete: true };
|
|
244
|
+
}
|
|
245
|
+
const prior_decisions = roundOne.map((entry, index) => canonicalize({
|
|
246
|
+
anonymous_ref: `independent-decision-${index + 1}`,
|
|
247
|
+
decision: entry.decision,
|
|
248
|
+
}));
|
|
249
|
+
return {
|
|
250
|
+
round: 2,
|
|
251
|
+
observations,
|
|
252
|
+
current: roundTwo,
|
|
253
|
+
prior_decisions,
|
|
254
|
+
complete: roundTwo.length === selected.length,
|
|
255
|
+
};
|
|
256
|
+
}
|
|
257
|
+
|
|
32
258
|
function timestamp(clock) {
|
|
33
259
|
if (typeof clock !== 'function') fail('MISSING_CLOCK', 'a deterministic clock function is required');
|
|
34
260
|
const value = clock();
|
|
@@ -52,60 +278,350 @@ function stableEventId(sequence, event) {
|
|
|
52
278
|
function checkpointPath(journalPath) { return `${journalPath}.checkpoint`; }
|
|
53
279
|
function lockPath(journalPath) { return `${journalPath}.lock`; }
|
|
54
280
|
|
|
55
|
-
function
|
|
56
|
-
|
|
57
|
-
const
|
|
281
|
+
function ensureSafeDirectory(path, { create = false } = {}) {
|
|
282
|
+
const absolute = resolve(path);
|
|
283
|
+
const root = parse(absolute).root;
|
|
284
|
+
let current = root;
|
|
285
|
+
for (const segment of absolute.slice(root.length).split(sep).filter(Boolean)) {
|
|
286
|
+
current = resolve(current, segment);
|
|
287
|
+
let stat;
|
|
288
|
+
try { stat = lstatSync(current); }
|
|
289
|
+
catch (error) {
|
|
290
|
+
if (error?.code !== 'ENOENT' || !create) throw error;
|
|
291
|
+
try { mkdirSync(current, { mode: 0o700 }); }
|
|
292
|
+
catch (mkdirError) { if (mkdirError?.code !== 'EEXIST') throw mkdirError; }
|
|
293
|
+
stat = lstatSync(current);
|
|
294
|
+
}
|
|
295
|
+
if (stat.isSymbolicLink() || !stat.isDirectory()) {
|
|
296
|
+
fail('UNSAFE_JOURNAL_DIRECTORY', 'journal directory chain must contain only real directories', { path: current });
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
return absolute;
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
function openDirectoryForSync(path, { create = false } = {}) {
|
|
303
|
+
const absolute = resolve(path);
|
|
304
|
+
if (process.platform === 'linux') {
|
|
305
|
+
const root = parse(absolute).root;
|
|
306
|
+
const segments = absolute.slice(root.length).split(sep).filter(Boolean);
|
|
307
|
+
let fd = openSync(root, constants.O_RDONLY | (constants.O_DIRECTORY || 0) | (constants.O_NOFOLLOW || 0));
|
|
308
|
+
try {
|
|
309
|
+
for (const segment of segments) {
|
|
310
|
+
const childPath = join(`/proc/self/fd/${fd}`, segment);
|
|
311
|
+
let childFd;
|
|
312
|
+
try {
|
|
313
|
+
childFd = openSync(childPath, constants.O_RDONLY | (constants.O_DIRECTORY || 0) | (constants.O_NOFOLLOW || 0));
|
|
314
|
+
} catch (error) {
|
|
315
|
+
if (error?.code !== 'ENOENT' || !create) {
|
|
316
|
+
if (['ELOOP', 'ENOTDIR'].includes(error?.code)) {
|
|
317
|
+
fail('UNSAFE_JOURNAL_DIRECTORY', 'journal directory chain must contain only real directories');
|
|
318
|
+
}
|
|
319
|
+
throw error;
|
|
320
|
+
}
|
|
321
|
+
try { mkdirSync(childPath, { mode: 0o700 }); }
|
|
322
|
+
catch (mkdirError) { if (mkdirError?.code !== 'EEXIST') throw mkdirError; }
|
|
323
|
+
childFd = openSync(childPath, constants.O_RDONLY | (constants.O_DIRECTORY || 0) | (constants.O_NOFOLLOW || 0));
|
|
324
|
+
}
|
|
325
|
+
const childStat = fstatSync(childFd);
|
|
326
|
+
if (!childStat.isDirectory()) {
|
|
327
|
+
closeSync(childFd);
|
|
328
|
+
fail('UNSAFE_JOURNAL_DIRECTORY', 'journal directory chain must contain only real directories');
|
|
329
|
+
}
|
|
330
|
+
closeSync(fd);
|
|
331
|
+
fd = childFd;
|
|
332
|
+
}
|
|
333
|
+
return { fd, path: absolute, stat: fstatSync(fd), reference: `/proc/self/fd/${fd}` };
|
|
334
|
+
} catch (error) {
|
|
335
|
+
closeSync(fd);
|
|
336
|
+
throw error;
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
const safePath = ensureSafeDirectory(absolute, { create });
|
|
340
|
+
const fd = openSync(safePath, constants.O_RDONLY | (constants.O_DIRECTORY || 0) | (constants.O_NOFOLLOW || 0));
|
|
341
|
+
const stat = fstatSync(fd);
|
|
342
|
+
if (!stat.isDirectory()) { closeSync(fd); fail('UNSAFE_JOURNAL_DIRECTORY', 'journal parent must be a directory'); }
|
|
343
|
+
return { fd, path: safePath, stat, reference: safePath };
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
function sameIdentity(left, right) {
|
|
347
|
+
return left?.dev === right?.dev && left?.ino === right?.ino;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
function sameSnapshot(left, right) {
|
|
351
|
+
return sameIdentity(left, right)
|
|
352
|
+
&& left?.mode === right?.mode
|
|
353
|
+
&& left?.uid === right?.uid
|
|
354
|
+
&& left?.gid === right?.gid
|
|
355
|
+
&& left?.nlink === right?.nlink
|
|
356
|
+
&& left?.size === right?.size
|
|
357
|
+
&& left?.mtimeMs === right?.mtimeMs
|
|
358
|
+
&& left?.ctimeMs === right?.ctimeMs;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
function readDescriptorBounded(fd, maxBytes, code, label) {
|
|
362
|
+
const chunks = [];
|
|
363
|
+
const buffer = Buffer.allocUnsafe(64 * 1024);
|
|
364
|
+
let total = 0;
|
|
365
|
+
while (total <= maxBytes) {
|
|
366
|
+
const wanted = Math.min(buffer.length, (maxBytes + 1) - total);
|
|
367
|
+
const bytesRead = readSync(fd, buffer, 0, wanted, null);
|
|
368
|
+
if (bytesRead === 0) break;
|
|
369
|
+
chunks.push(Buffer.from(buffer.subarray(0, bytesRead)));
|
|
370
|
+
total += bytesRead;
|
|
371
|
+
}
|
|
372
|
+
if (total > maxBytes) fail(code, `${label} exceeds its safety limit`);
|
|
373
|
+
return Buffer.concat(chunks, total).toString('utf8');
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
function readRegularBoundedRecord(path, { allowMissing = false, label = 'journal', maxBytes = JOURNAL_MAX_BYTES } = {}) {
|
|
377
|
+
let fd;
|
|
378
|
+
try {
|
|
379
|
+
fd = openSync(path, constants.O_RDONLY | (constants.O_NOFOLLOW || 0) | (constants.O_NONBLOCK || 0));
|
|
380
|
+
} catch (error) {
|
|
381
|
+
if (allowMissing && error?.code === 'ENOENT') return null;
|
|
382
|
+
if (['ELOOP', 'ENXIO', 'EISDIR'].includes(error?.code)) fail('UNSAFE_JOURNAL_FILE', `${label} must not be a symlink, FIFO or directory`);
|
|
383
|
+
throw error;
|
|
384
|
+
}
|
|
385
|
+
try {
|
|
386
|
+
const stat = fstatSync(fd);
|
|
387
|
+
if (!stat.isFile() || stat.nlink !== 1) fail('UNSAFE_JOURNAL_FILE', `${label} must be a regular single-link file`);
|
|
388
|
+
if (stat.size > maxBytes) fail('JOURNAL_SIZE_LIMIT', `${label} exceeds its safety limit`);
|
|
389
|
+
runSecurityTestHook('afterJournalStateStat', { path, label, fd });
|
|
390
|
+
const content = readDescriptorBounded(fd, maxBytes, 'JOURNAL_SIZE_LIMIT', label);
|
|
391
|
+
const after = fstatSync(fd);
|
|
392
|
+
if (!sameSnapshot(stat, after)) fail('JOURNAL_STORAGE_CONFLICT', `${label} changed while it was read`);
|
|
393
|
+
return { content, stat: after };
|
|
394
|
+
} finally { closeSync(fd); }
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
function readRegularBounded(path, options) {
|
|
398
|
+
return readRegularBoundedRecord(path, options)?.content ?? null;
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
function processIsAlive(pid) {
|
|
402
|
+
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
403
|
+
try { process.kill(pid, 0); return true; } catch (error) { return error?.code === 'EPERM'; }
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
function readLockOwner(path) {
|
|
407
|
+
try {
|
|
408
|
+
const raw = readRegularBounded(path, { allowMissing: true, label: 'journal lock', maxBytes: 16 * 1024 });
|
|
409
|
+
if (raw === null) return null;
|
|
410
|
+
const owner = JSON.parse(raw);
|
|
411
|
+
if (!owner || typeof owner !== 'object' || typeof owner.token !== 'string') return null;
|
|
412
|
+
return owner;
|
|
413
|
+
} catch { return null; }
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
function recoverStaleLock(path) {
|
|
417
|
+
let stat;
|
|
418
|
+
try { stat = lstatSync(path); } catch (error) { return error?.code === 'ENOENT'; }
|
|
419
|
+
if (stat.isSymbolicLink() || !stat.isFile() || stat.nlink !== 1) return false;
|
|
420
|
+
const owner = readLockOwner(path);
|
|
421
|
+
const sameHostOwnerAlive = owner?.hostname === hostname() && processIsAlive(owner.pid);
|
|
422
|
+
if (sameHostOwnerAlive) return false;
|
|
423
|
+
// Cross-host liveness cannot be proven from a local PID namespace. Remote
|
|
424
|
+
// locks are never stolen automatically; an operator must reconcile/remove
|
|
425
|
+
// them after proving the owner is gone.
|
|
426
|
+
if (owner?.hostname && owner.hostname !== hostname()) return false;
|
|
427
|
+
if (!owner && Date.now() - stat.mtimeMs < LOCK_STALE_AFTER_MS) return false;
|
|
428
|
+
const recoveryPath = `${path}.recovery`;
|
|
429
|
+
let recoveryFd;
|
|
430
|
+
try {
|
|
431
|
+
recoveryFd = openSync(recoveryPath, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | (constants.O_NOFOLLOW || 0), 0o600);
|
|
432
|
+
const recoveryBytes = Buffer.from(JSON.stringify({ pid: process.pid, hostname: hostname(), token: randomUUID() }));
|
|
433
|
+
writeSync(recoveryFd, recoveryBytes, 0, recoveryBytes.length);
|
|
434
|
+
fsyncSync(recoveryFd);
|
|
435
|
+
} catch {
|
|
436
|
+
if (recoveryFd !== undefined) closeSync(recoveryFd);
|
|
437
|
+
return false;
|
|
438
|
+
}
|
|
439
|
+
const quarantine = `${path}.stale.${process.pid}.${Date.now()}`;
|
|
440
|
+
try {
|
|
441
|
+
const current = lstatSync(path);
|
|
442
|
+
const currentOwner = readLockOwner(path);
|
|
443
|
+
if (!sameIdentity(stat, current) || currentOwner?.token !== owner?.token) return false;
|
|
444
|
+
renameSync(path, quarantine);
|
|
445
|
+
const quarantined = lstatSync(quarantine);
|
|
446
|
+
const quarantinedOwner = readLockOwner(quarantine);
|
|
447
|
+
if (!sameIdentity(stat, quarantined) || quarantinedOwner?.token !== owner?.token) {
|
|
448
|
+
try { if (!existsSync(path)) renameSync(quarantine, path); } catch { /* preserve quarantine fail-closed */ }
|
|
449
|
+
return false;
|
|
450
|
+
}
|
|
451
|
+
unlinkSync(quarantine);
|
|
452
|
+
return true;
|
|
453
|
+
} catch (error) {
|
|
454
|
+
if (error?.code === 'ENOENT') return true;
|
|
455
|
+
return false;
|
|
456
|
+
} finally {
|
|
457
|
+
if (recoveryFd !== undefined) closeSync(recoveryFd);
|
|
458
|
+
try { unlinkSync(recoveryPath); } catch { /* fail closed if another process changed it */ }
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
function parseJournal(journalPath, { repair = false, directory = null } = {}) {
|
|
463
|
+
if (!directory) ensureSafeDirectory(dirname(journalPath));
|
|
464
|
+
const activeJournalPath = directory
|
|
465
|
+
? join(directory.reference, basename(journalPath))
|
|
466
|
+
: journalPath;
|
|
467
|
+
const journal = readRegularBoundedRecord(activeJournalPath, { allowMissing: true });
|
|
468
|
+
const rawJournal = journal?.content ?? null;
|
|
469
|
+
const raw = rawJournal ?? '';
|
|
470
|
+
const hasTail = Boolean(raw && !raw.endsWith('\n'));
|
|
471
|
+
const completeJournal = hasTail ? raw.slice(0, raw.lastIndexOf('\n') + 1) : raw;
|
|
472
|
+
const lines = completeJournal.split('\n').filter(Boolean);
|
|
58
473
|
const records = [];
|
|
59
474
|
let previous_hash = 'GENESIS';
|
|
60
|
-
|
|
475
|
+
const parseRecord = (line, index, priorHash) => {
|
|
61
476
|
let record;
|
|
62
477
|
try { record = JSON.parse(line); } catch { fail('JOURNAL_CORRUPT', 'journal contains malformed JSONL', { line: index + 1 }); }
|
|
63
478
|
assertObject(record, 'JOURNAL_CORRUPT', 'journal record');
|
|
64
479
|
const { record_hash, chain_hash, previous_hash: record_previous_hash, ...material } = record;
|
|
65
480
|
assertString(record_hash, 'JOURNAL_CORRUPT', 'record_hash');
|
|
66
481
|
assertString(chain_hash, 'JOURNAL_CORRUPT', 'chain_hash');
|
|
67
|
-
if (record_previous_hash !==
|
|
482
|
+
if (record_previous_hash !== priorHash) fail('JOURNAL_CHAIN_MISMATCH', 'journal previous hash does not match prior record', { event_id: record.event_id });
|
|
68
483
|
if (sha256(material) !== record_hash) fail('JOURNAL_INTEGRITY_MISMATCH', 'journal record hash does not match material', { event_id: record.event_id });
|
|
69
|
-
if (sha256({ previous_hash, record_hash }) !== chain_hash) fail('JOURNAL_CHAIN_MISMATCH', 'journal chain hash does not match record', { event_id: record.event_id });
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
484
|
+
if (sha256({ previous_hash: priorHash, record_hash }) !== chain_hash) fail('JOURNAL_CHAIN_MISMATCH', 'journal chain hash does not match record', { event_id: record.event_id });
|
|
485
|
+
return Object.freeze(canonicalize(record));
|
|
486
|
+
};
|
|
487
|
+
for (const [index, line] of lines.entries()) {
|
|
488
|
+
const record = parseRecord(line, index, previous_hash);
|
|
489
|
+
previous_hash = record.chain_hash;
|
|
490
|
+
records.push(record);
|
|
491
|
+
}
|
|
492
|
+
const tail = hasTail ? raw.slice(raw.lastIndexOf('\n') + 1) : null;
|
|
493
|
+
let completeTail = false;
|
|
494
|
+
if (tail !== null) {
|
|
495
|
+
try {
|
|
496
|
+
const record = parseRecord(tail, records.length, previous_hash);
|
|
497
|
+
previous_hash = record.chain_hash;
|
|
498
|
+
records.push(record);
|
|
499
|
+
completeTail = true;
|
|
500
|
+
} catch { /* the durable checkpoint decides whether partial bytes are repairable */ }
|
|
501
|
+
}
|
|
502
|
+
const checkpointFile = directory
|
|
503
|
+
? join(directory.reference, basename(checkpointPath(journalPath)))
|
|
504
|
+
: checkpointPath(journalPath);
|
|
505
|
+
const rawCheckpoint = readRegularBounded(checkpointFile, {
|
|
506
|
+
allowMissing: true, label: 'journal checkpoint', maxBytes: JOURNAL_RECORD_MAX_BYTES,
|
|
507
|
+
});
|
|
508
|
+
let checkpoint = null;
|
|
509
|
+
let validCommittedState = false;
|
|
510
|
+
let validPendingState = false;
|
|
511
|
+
if (rawCheckpoint !== null) {
|
|
512
|
+
try { checkpoint = JSON.parse(rawCheckpoint); } catch { fail('JOURNAL_CHECKPOINT_CORRUPT', 'journal checkpoint is malformed'); }
|
|
77
513
|
assertObject(checkpoint, 'JOURNAL_CHECKPOINT_CORRUPT', 'journal checkpoint');
|
|
78
|
-
|
|
79
|
-
|
|
514
|
+
const currentAnchor = {
|
|
515
|
+
record_count: records.length,
|
|
516
|
+
chain_hash: previous_hash,
|
|
517
|
+
last_event_id: records.at(-1)?.event_id ?? null,
|
|
518
|
+
};
|
|
519
|
+
const matches = (anchor) => Boolean(anchor
|
|
520
|
+
&& anchor.record_count === currentAnchor.record_count
|
|
521
|
+
&& anchor.chain_hash === currentAnchor.chain_hash
|
|
522
|
+
&& (anchor.last_event_id ?? null) === currentAnchor.last_event_id);
|
|
523
|
+
if (checkpoint.schema_version === 1) {
|
|
524
|
+
if (!matches(checkpoint)) fail('JOURNAL_TRUNCATION_DETECTED', 'journal does not match its persisted checkpoint');
|
|
525
|
+
validCommittedState = true;
|
|
526
|
+
} else if (checkpoint.schema_version === 2) {
|
|
527
|
+
assertObject(checkpoint.committed, 'JOURNAL_CHECKPOINT_CORRUPT', 'journal checkpoint committed anchor');
|
|
528
|
+
validCommittedState = matches(checkpoint.committed);
|
|
529
|
+
validPendingState = checkpoint.pending !== null && matches(checkpoint.pending);
|
|
530
|
+
if (!validCommittedState && !validPendingState) {
|
|
531
|
+
fail('JOURNAL_TRUNCATION_DETECTED', 'journal does not match either durable checkpoint state');
|
|
532
|
+
}
|
|
533
|
+
} else {
|
|
534
|
+
fail('JOURNAL_CHECKPOINT_CORRUPT', 'journal checkpoint schema is unsupported');
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
const incompleteTail = tail !== null && !completeTail;
|
|
538
|
+
const missingDelimiter = tail !== null && completeTail;
|
|
539
|
+
if (checkpoint === null && (incompleteTail || missingDelimiter)) {
|
|
540
|
+
fail('JOURNAL_CORRUPT', 'journal contains an uncheckpointed trailing record');
|
|
541
|
+
}
|
|
542
|
+
const safeIncompleteTail = incompleteTail && validCommittedState
|
|
543
|
+
&& checkpoint?.schema_version === 2 && checkpoint.pending !== null;
|
|
544
|
+
if (incompleteTail && !safeIncompleteTail) {
|
|
545
|
+
fail('JOURNAL_TRUNCATION_DETECTED', 'journal was truncated below its durable high-water mark');
|
|
546
|
+
}
|
|
547
|
+
let identity = journal?.stat ?? null;
|
|
548
|
+
if ((safeIncompleteTail || missingDelimiter) && repair) {
|
|
549
|
+
const fd = openSync(activeJournalPath, constants.O_RDWR | (constants.O_NOFOLLOW || 0) | (constants.O_NONBLOCK || 0));
|
|
550
|
+
try {
|
|
551
|
+
const before = fstatSync(fd);
|
|
552
|
+
if (!before.isFile() || before.nlink !== 1) fail('UNSAFE_JOURNAL_FILE', 'journal must be a regular single-link file');
|
|
553
|
+
if (!sameSnapshot(before, journal.stat)) fail('JOURNAL_STORAGE_CONFLICT', 'journal changed between load and tail repair');
|
|
554
|
+
const expectedContent = safeIncompleteTail ? completeJournal : `${raw}\n`;
|
|
555
|
+
const repairedBytes = Buffer.byteLength(expectedContent);
|
|
556
|
+
if (safeIncompleteTail) {
|
|
557
|
+
ftruncateSync(fd, repairedBytes);
|
|
558
|
+
} else {
|
|
559
|
+
const newline = Buffer.from('\n');
|
|
560
|
+
const written = writeSync(fd, newline, 0, newline.length, before.size);
|
|
561
|
+
if (written !== newline.length) fail('JOURNAL_STORAGE_CONFLICT', 'journal delimiter repair was incomplete');
|
|
562
|
+
}
|
|
563
|
+
fsyncSync(fd);
|
|
564
|
+
const afterRepair = fstatSync(fd);
|
|
565
|
+
if (!sameIdentity(before, afterRepair)
|
|
566
|
+
|| afterRepair.mode !== before.mode || afterRepair.uid !== before.uid || afterRepair.gid !== before.gid
|
|
567
|
+
|| afterRepair.nlink !== before.nlink || afterRepair.size !== repairedBytes) {
|
|
568
|
+
fail('JOURNAL_STORAGE_CONFLICT', 'journal changed during tail repair');
|
|
569
|
+
}
|
|
570
|
+
const verifiedContent = readDescriptorBounded(fd, JOURNAL_MAX_BYTES, 'JOURNAL_SIZE_LIMIT', 'journal');
|
|
571
|
+
const afterRead = fstatSync(fd);
|
|
572
|
+
if (verifiedContent !== expectedContent || !sameSnapshot(afterRepair, afterRead)) {
|
|
573
|
+
fail('JOURNAL_STORAGE_CONFLICT', 'journal content changed during tail repair verification');
|
|
574
|
+
}
|
|
575
|
+
identity = afterRead;
|
|
576
|
+
} finally { closeSync(fd); }
|
|
577
|
+
if (directory) fsyncSync(directory.fd);
|
|
578
|
+
else {
|
|
579
|
+
const openedDirectory = openDirectoryForSync(dirname(journalPath));
|
|
580
|
+
try { fsyncSync(openedDirectory.fd); } finally { closeSync(openedDirectory.fd); }
|
|
80
581
|
}
|
|
81
582
|
}
|
|
82
|
-
|
|
583
|
+
if (rawJournal === null && checkpoint !== null && !validCommittedState && !validPendingState) {
|
|
584
|
+
fail('JOURNAL_TRUNCATION_DETECTED', 'journal file is missing below its durable high-water mark');
|
|
585
|
+
}
|
|
586
|
+
return { records, identity };
|
|
83
587
|
}
|
|
84
588
|
|
|
85
589
|
function isExpired(claim, now) {
|
|
86
590
|
return Date.parse(claim.lease.expires_at) <= Date.parse(now);
|
|
87
591
|
}
|
|
88
592
|
|
|
89
|
-
function
|
|
90
|
-
|
|
91
|
-
if (
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
593
|
+
function claimProcessPid(attempt) {
|
|
594
|
+
const match = /^process:\/\/chati\/(\d+)$/.exec(attempt.claim.harness_process_id || '');
|
|
595
|
+
if (!match) fail('RECOVERY_PROCESS_UNRESOLVED', 'claim process identity is not locally observable');
|
|
596
|
+
return Number(match[1]);
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
function observeStoppedClaimProcess(attempt) {
|
|
600
|
+
if (attempt.claim.hostname !== hostname()) {
|
|
601
|
+
fail('RECOVERY_REMOTE_PROCESS_UNVERIFIABLE', 'automatic recovery cannot prove a process stopped on another host');
|
|
602
|
+
}
|
|
603
|
+
const pid = claimProcessPid(attempt);
|
|
604
|
+
try {
|
|
605
|
+
process.kill(pid, 0);
|
|
606
|
+
fail('RECOVERY_PROCESS_STILL_RUNNING', 'the prior claim process is still running', { pid });
|
|
607
|
+
} catch (error) {
|
|
608
|
+
if (error?.code !== 'ESRCH') throw error;
|
|
609
|
+
}
|
|
610
|
+
return pid;
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
function verifyCommitDescendsFrom({ git_dir, base_commit_ref, result_commit_ref }) {
|
|
614
|
+
verifyLocalGitCommit({ git_dir, git_commit_ref: base_commit_ref });
|
|
615
|
+
verifyLocalGitCommit({ git_dir, git_commit_ref: result_commit_ref });
|
|
616
|
+
if (base_commit_ref === result_commit_ref) {
|
|
617
|
+
fail('GIT_RESULT_COMMIT_REQUIRED', 'result commit must differ from the sealed base commit');
|
|
618
|
+
}
|
|
619
|
+
try {
|
|
620
|
+
execFileSync('git', ['-C', git_dir, 'merge-base', '--is-ancestor', base_commit_ref, result_commit_ref], {
|
|
621
|
+
stdio: 'ignore', env: trustedGitEnvironment(),
|
|
622
|
+
});
|
|
623
|
+
} catch {
|
|
624
|
+
fail('GIT_RESULT_NOT_DESCENDANT', 'result commit must descend from the sealed base commit');
|
|
109
625
|
}
|
|
110
626
|
}
|
|
111
627
|
|
|
@@ -117,35 +633,94 @@ export class RailEngine {
|
|
|
117
633
|
#journalPath;
|
|
118
634
|
#clock;
|
|
119
635
|
#records;
|
|
636
|
+
#activeDirectory;
|
|
637
|
+
#journalIdentity;
|
|
638
|
+
#authorityVerifier;
|
|
120
639
|
|
|
121
|
-
constructor({ journal_path, clock } = {}) {
|
|
640
|
+
constructor({ journal_path, clock, authority_verifier = null } = {}) {
|
|
122
641
|
assertString(journal_path, 'INVALID_JOURNAL_PATH', 'journal_path');
|
|
123
642
|
this.#journalPath = journal_path;
|
|
124
643
|
this.#clock = clock;
|
|
125
|
-
|
|
644
|
+
if (authority_verifier !== null && typeof authority_verifier !== 'function') {
|
|
645
|
+
fail('INVALID_RAIL_AUTHORITY_VERIFIER', 'authority_verifier must be a function');
|
|
646
|
+
}
|
|
647
|
+
this.#authorityVerifier = authority_verifier;
|
|
648
|
+
const loaded = parseJournal(journal_path);
|
|
649
|
+
this.#records = loaded.records;
|
|
650
|
+
this.#journalIdentity = loaded.identity;
|
|
126
651
|
}
|
|
127
652
|
|
|
128
653
|
get journalPath() { return this.#journalPath; }
|
|
129
654
|
get records() { return Object.freeze([...this.#records]); }
|
|
130
655
|
|
|
131
|
-
#
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
656
|
+
#verifyAuthority(kind, receipt, expected) {
|
|
657
|
+
if (!this.#authorityVerifier || this.#authorityVerifier({ kind, receipt, expected }) !== true) {
|
|
658
|
+
fail('UNTRUSTED_RAIL_AUTHORITY_RECEIPT', `${kind} requires a receipt from the configured terminal authority`);
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
#assertDirectoryStable() {
|
|
663
|
+
if (!this.#activeDirectory) fail('UNSAFE_JOURNAL_DIRECTORY', 'journal mutation requires a pinned parent directory');
|
|
664
|
+
let canonical;
|
|
665
|
+
try { canonical = openDirectoryForSync(this.#activeDirectory.path); }
|
|
666
|
+
catch { fail('UNSAFE_JOURNAL_DIRECTORY', 'journal directory chain changed during mutation'); }
|
|
667
|
+
try {
|
|
668
|
+
const pinned = fstatSync(this.#activeDirectory.fd);
|
|
669
|
+
if (!pinned.isDirectory() || !sameIdentity(pinned, this.#activeDirectory.stat)
|
|
670
|
+
|| pinned.mode !== this.#activeDirectory.stat.mode
|
|
671
|
+
|| pinned.uid !== this.#activeDirectory.stat.uid
|
|
672
|
+
|| pinned.gid !== this.#activeDirectory.stat.gid
|
|
673
|
+
|| !canonical.stat.isDirectory()
|
|
674
|
+
|| !sameIdentity(canonical.stat, this.#activeDirectory.stat)) {
|
|
675
|
+
fail('UNSAFE_JOURNAL_DIRECTORY', 'journal parent changed during mutation');
|
|
140
676
|
}
|
|
677
|
+
} finally {
|
|
678
|
+
closeSync(canonical.fd);
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
#activeEntry(path) {
|
|
683
|
+
if (!this.#activeDirectory) {
|
|
684
|
+
fail('UNSAFE_JOURNAL_DIRECTORY', 'journal parent changed during mutation');
|
|
141
685
|
}
|
|
142
|
-
|
|
686
|
+
return join(this.#activeDirectory.reference, basename(path));
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
#exclusive(action) {
|
|
690
|
+
const directory = openDirectoryForSync(dirname(this.#journalPath), { create: true });
|
|
691
|
+
this.#activeDirectory = directory;
|
|
692
|
+
const path = this.#activeEntry(lockPath(this.#journalPath));
|
|
693
|
+
let descriptor;
|
|
694
|
+
let lockToken;
|
|
143
695
|
try {
|
|
144
|
-
|
|
696
|
+
for (let retry = 0; retry < LOCK_RETRIES; retry += 1) {
|
|
697
|
+
try {
|
|
698
|
+
descriptor = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | (constants.O_NOFOLLOW || 0), 0o600);
|
|
699
|
+
lockToken = `${process.pid}-${Date.now()}-${randomUUID()}`;
|
|
700
|
+
const ownerBytes = Buffer.from(JSON.stringify({ schema_version: 1, pid: process.pid, hostname: hostname(), token: lockToken, acquired_at: new Date().toISOString() }));
|
|
701
|
+
writeSync(descriptor, ownerBytes, 0, ownerBytes.length);
|
|
702
|
+
fsyncSync(descriptor);
|
|
703
|
+
fsyncSync(directory.fd);
|
|
704
|
+
break;
|
|
705
|
+
} catch (error) {
|
|
706
|
+
if (descriptor !== undefined) { closeSync(descriptor); descriptor = undefined; }
|
|
707
|
+
if (error?.code !== 'EEXIST') throw error;
|
|
708
|
+
if (recoverStaleLock(path)) continue;
|
|
709
|
+
Atomics.wait(LOCK_WAIT, 0, 0, 2);
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
if (descriptor === undefined) fail('JOURNAL_LOCK_BUSY', 'journal mutation lock is already held');
|
|
713
|
+
const loaded = parseJournal(this.#journalPath, { repair: true, directory });
|
|
714
|
+
this.#records = loaded.records;
|
|
715
|
+
this.#journalIdentity = loaded.identity;
|
|
145
716
|
return action();
|
|
146
717
|
} finally {
|
|
147
|
-
|
|
148
|
-
|
|
718
|
+
if (descriptor !== undefined) {
|
|
719
|
+
closeSync(descriptor);
|
|
720
|
+
if (readLockOwner(path)?.token === lockToken) { unlinkSync(path); fsyncSync(directory.fd); }
|
|
721
|
+
}
|
|
722
|
+
this.#activeDirectory = undefined;
|
|
723
|
+
closeSync(directory.fd);
|
|
149
724
|
}
|
|
150
725
|
}
|
|
151
726
|
|
|
@@ -153,12 +728,54 @@ export class RailEngine {
|
|
|
153
728
|
if (isExpired(attempt.claim, now)) fail(code, 'operation requires a live claim lease', { attempt_id: attempt.attempt_id });
|
|
154
729
|
}
|
|
155
730
|
|
|
156
|
-
#
|
|
157
|
-
|
|
158
|
-
const target = checkpointPath(this.#journalPath);
|
|
159
|
-
const temporary = `${target}.${process.pid}.tmp`;
|
|
160
|
-
|
|
161
|
-
|
|
731
|
+
#writeCheckpointFile(checkpoint) {
|
|
732
|
+
this.#assertDirectoryStable();
|
|
733
|
+
const target = this.#activeEntry(checkpointPath(this.#journalPath));
|
|
734
|
+
const temporary = `${target}.${process.pid}.${randomUUID()}.tmp`;
|
|
735
|
+
let fd;
|
|
736
|
+
try {
|
|
737
|
+
fd = openSync(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | (constants.O_NOFOLLOW || 0), 0o600);
|
|
738
|
+
const bytes = Buffer.from(JSON.stringify(checkpoint));
|
|
739
|
+
if (bytes.length > JOURNAL_RECORD_MAX_BYTES) fail('JOURNAL_SIZE_LIMIT', 'journal checkpoint exceeds the record safety limit');
|
|
740
|
+
writeSync(fd, bytes, 0, bytes.length);
|
|
741
|
+
fsyncSync(fd);
|
|
742
|
+
closeSync(fd);
|
|
743
|
+
fd = undefined;
|
|
744
|
+
this.#assertDirectoryStable();
|
|
745
|
+
renameSync(temporary, target);
|
|
746
|
+
fsyncSync(this.#activeDirectory.fd);
|
|
747
|
+
} finally {
|
|
748
|
+
if (fd !== undefined) closeSync(fd);
|
|
749
|
+
try { if (existsSync(temporary)) unlinkSync(temporary); } catch { /* preserve primary failure */ }
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
#journalAnchor(records = this.#records) {
|
|
754
|
+
return canonicalize({
|
|
755
|
+
record_count: records.length,
|
|
756
|
+
last_event_id: records.at(-1)?.event_id ?? null,
|
|
757
|
+
chain_hash: records.at(-1)?.chain_hash ?? 'GENESIS',
|
|
758
|
+
});
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
#writePendingCheckpoint(record) {
|
|
762
|
+
this.#writeCheckpointFile(canonicalize({
|
|
763
|
+
schema_version: 2,
|
|
764
|
+
committed: this.#journalAnchor(),
|
|
765
|
+
pending: {
|
|
766
|
+
record_count: this.#records.length + 1,
|
|
767
|
+
last_event_id: record.event_id,
|
|
768
|
+
chain_hash: record.chain_hash,
|
|
769
|
+
},
|
|
770
|
+
}));
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
#writeCheckpoint() {
|
|
774
|
+
this.#writeCheckpointFile(canonicalize({
|
|
775
|
+
schema_version: 2,
|
|
776
|
+
committed: this.#journalAnchor(),
|
|
777
|
+
pending: null,
|
|
778
|
+
}));
|
|
162
779
|
}
|
|
163
780
|
|
|
164
781
|
#append(type, payload, causation_id) {
|
|
@@ -174,10 +791,50 @@ export class RailEngine {
|
|
|
174
791
|
const record_hash = sha256(material);
|
|
175
792
|
const previous_hash = this.#records.at(-1)?.chain_hash ?? 'GENESIS';
|
|
176
793
|
const record = Object.freeze(canonicalize({ ...material, record_hash, previous_hash, chain_hash: sha256({ previous_hash, record_hash }) }));
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
794
|
+
const bytes = Buffer.from(`${JSON.stringify(record)}\n`);
|
|
795
|
+
if (bytes.length > JOURNAL_RECORD_MAX_BYTES) fail('JOURNAL_SIZE_LIMIT', 'journal record exceeds the 4 MiB safety limit');
|
|
796
|
+
// Two-phase checkpointing makes both crash windows reconstructible:
|
|
797
|
+
// pending + old journal means append never happened; pending + new journal
|
|
798
|
+
// means append succeeded and only checkpoint finalization was interrupted.
|
|
799
|
+
this.#writePendingCheckpoint(record);
|
|
800
|
+
this.#assertDirectoryStable();
|
|
801
|
+
runSecurityTestHook('beforeJournalAppend', { journalPath: this.#journalPath, type });
|
|
802
|
+
this.#assertDirectoryStable();
|
|
803
|
+
const existed = this.#journalIdentity !== null;
|
|
804
|
+
let fd;
|
|
805
|
+
try {
|
|
806
|
+
fd = openSync(this.#activeEntry(this.#journalPath), constants.O_WRONLY | constants.O_APPEND
|
|
807
|
+
| (existed ? 0 : constants.O_CREAT | constants.O_EXCL)
|
|
808
|
+
| (constants.O_NOFOLLOW || 0), 0o600);
|
|
809
|
+
} catch (error) {
|
|
810
|
+
if (!existed && error?.code === 'EEXIST') fail('JOURNAL_STORAGE_CONFLICT', 'journal appeared after canonical state was loaded');
|
|
811
|
+
throw error;
|
|
812
|
+
}
|
|
813
|
+
try {
|
|
814
|
+
const stat = fstatSync(fd);
|
|
815
|
+
if (!stat.isFile() || stat.nlink !== 1) fail('UNSAFE_JOURNAL_FILE', 'journal must be a regular single-link file');
|
|
816
|
+
if (existed && !sameSnapshot(stat, this.#journalIdentity)) fail('JOURNAL_STORAGE_CONFLICT', 'journal changed between load and append');
|
|
817
|
+
if (stat.size + bytes.length > JOURNAL_MAX_BYTES) fail('JOURNAL_SIZE_LIMIT', 'journal exceeds the 64 MiB safety limit');
|
|
818
|
+
let offset = 0;
|
|
819
|
+
while (offset < bytes.length) offset += writeSync(fd, bytes, offset, bytes.length - offset);
|
|
820
|
+
fsyncSync(fd);
|
|
821
|
+
const after = fstatSync(fd);
|
|
822
|
+
if (!sameIdentity(stat, after)
|
|
823
|
+
|| after.mode !== stat.mode || after.uid !== stat.uid || after.gid !== stat.gid
|
|
824
|
+
|| after.nlink !== stat.nlink || after.size !== stat.size + bytes.length) {
|
|
825
|
+
fail('JOURNAL_STORAGE_CONFLICT', 'journal changed during append');
|
|
826
|
+
}
|
|
827
|
+
} finally { closeSync(fd); }
|
|
828
|
+
if (!existed) fsyncSync(this.#activeDirectory.fd);
|
|
829
|
+
const expectedRecords = [...this.#records, record];
|
|
830
|
+
this.#assertDirectoryStable();
|
|
831
|
+
const verified = parseJournal(this.#journalPath, { directory: this.#activeDirectory });
|
|
832
|
+
if (sha256(verified.records) !== sha256(expectedRecords)) {
|
|
833
|
+
fail('JOURNAL_STORAGE_CONFLICT', 'journal content changed during append');
|
|
834
|
+
}
|
|
835
|
+
this.#records = verified.records;
|
|
836
|
+
this.#journalIdentity = verified.identity;
|
|
837
|
+
this.#writeCheckpoint();
|
|
181
838
|
return record;
|
|
182
839
|
}
|
|
183
840
|
|
|
@@ -190,9 +847,18 @@ export class RailEngine {
|
|
|
190
847
|
const current = attempts.get(payload.attempt_id);
|
|
191
848
|
if (current) current.claim = { ...current.claim, lease: payload.lease };
|
|
192
849
|
}
|
|
850
|
+
if (type === 'attempt_relinquished') {
|
|
851
|
+
const current = attempts.get(payload.attempt_id);
|
|
852
|
+
if (current) current.claim = { ...current.claim, lease: payload.lease };
|
|
853
|
+
}
|
|
193
854
|
if (type === 'attempt_recovered') {
|
|
194
855
|
const current = attempts.get(payload.attempt_id);
|
|
195
|
-
if (current) {
|
|
856
|
+
if (current) {
|
|
857
|
+
current.claim = payload.claim;
|
|
858
|
+
current.state = payload.resume_state ?? 'active';
|
|
859
|
+
current.claim_event_id = record.event_id;
|
|
860
|
+
current.recovery_event_id = record.event_id;
|
|
861
|
+
}
|
|
196
862
|
}
|
|
197
863
|
if (type === 'attempt_completed') {
|
|
198
864
|
const current = attempts.get(payload.completion.attempt_id);
|
|
@@ -202,6 +868,10 @@ export class RailEngine {
|
|
|
202
868
|
const current = attempts.get(payload.attempt_id);
|
|
203
869
|
if (current) current.state = type === 'attempt_blocked' ? 'blocked' : 'cancelled';
|
|
204
870
|
}
|
|
871
|
+
if (type === 'adjudication_decided' && adjudicationAttemptState(payload.decision) !== 'active') {
|
|
872
|
+
const current = attempts.get(payload.attempt_id);
|
|
873
|
+
if (current) current.state = adjudicationAttemptState(payload.decision);
|
|
874
|
+
}
|
|
205
875
|
}
|
|
206
876
|
return attempts;
|
|
207
877
|
}
|
|
@@ -218,6 +888,438 @@ export class RailEngine {
|
|
|
218
888
|
return effects;
|
|
219
889
|
}
|
|
220
890
|
|
|
891
|
+
#reviewEvidence(attemptId) {
|
|
892
|
+
return this.#records
|
|
893
|
+
.filter((record) => record.type === 'review_decided' && record.payload?.attempt_id === attemptId)
|
|
894
|
+
.map((record) => {
|
|
895
|
+
if (!record.payload.review || !record.payload.invocation) {
|
|
896
|
+
fail('REVIEW_HISTORY_INCOMPLETE', 'canonical review history is missing immutable review material', {
|
|
897
|
+
attempt_id: attemptId,
|
|
898
|
+
review_ref: record.payload.review_ref,
|
|
899
|
+
});
|
|
900
|
+
}
|
|
901
|
+
return canonicalize({
|
|
902
|
+
review: record.payload.review,
|
|
903
|
+
invocation: record.payload.invocation,
|
|
904
|
+
lineage: {
|
|
905
|
+
base_commit_ref: record.payload.base_commit_ref,
|
|
906
|
+
result_commit_ref: record.payload.result_commit_ref,
|
|
907
|
+
},
|
|
908
|
+
});
|
|
909
|
+
});
|
|
910
|
+
}
|
|
911
|
+
|
|
912
|
+
#executionEvidence(attemptId, { successfulOnly = false } = {}) {
|
|
913
|
+
return this.#records
|
|
914
|
+
.filter((record) => record.type === 'execution_recorded' && record.payload?.attempt_id === attemptId)
|
|
915
|
+
.map((record) => record.payload.execution_evidence)
|
|
916
|
+
.filter((evidence) => !successfulOnly || (
|
|
917
|
+
evidence.execution?.exit_status === 0 && typeof evidence.result_commit_ref === 'string'
|
|
918
|
+
));
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
executionEvidenceForAttempt(attemptId, { successful_only = false } = {}) {
|
|
922
|
+
assertString(attemptId, 'INVALID_EXECUTION_EVIDENCE', 'attempt_id');
|
|
923
|
+
return Object.freeze(this.#executionEvidence(attemptId, { successfulOnly: successful_only })
|
|
924
|
+
.map((entry) => Object.freeze(entry)));
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
reviewEvidenceForAttempt(attemptId) {
|
|
928
|
+
assertString(attemptId, 'INVALID_REVIEW_DECISION', 'attempt_id');
|
|
929
|
+
return Object.freeze(this.#reviewEvidence(attemptId).map((entry) => Object.freeze(entry)));
|
|
930
|
+
}
|
|
931
|
+
|
|
932
|
+
reserveExecutionLaunch({
|
|
933
|
+
attempt_id, claim_id, operator_id, harness_process_id, invocation,
|
|
934
|
+
} = {}) {
|
|
935
|
+
return this.#exclusive(() => {
|
|
936
|
+
for (const [field, value] of Object.entries({
|
|
937
|
+
attempt_id, claim_id, operator_id, harness_process_id,
|
|
938
|
+
})) assertString(value, 'INVALID_EXECUTION_LAUNCH_RESERVATION', field);
|
|
939
|
+
assertObject(invocation, 'INVALID_EXECUTION_LAUNCH_RESERVATION', 'invocation');
|
|
940
|
+
assertString(invocation.invocation_id, 'INVALID_EXECUTION_LAUNCH_RESERVATION', 'invocation.invocation_id');
|
|
941
|
+
assertString(invocation.launch_nonce, 'INVALID_EXECUTION_LAUNCH_RESERVATION', 'invocation.launch_nonce');
|
|
942
|
+
assertString(invocation.prompt_digest, 'INVALID_EXECUTION_LAUNCH_RESERVATION', 'invocation.prompt_digest');
|
|
943
|
+
const attempt = this.#attempts().get(attempt_id);
|
|
944
|
+
if (!attempt || attempt.state !== EXECUTABLE_ATTEMPT_STATE) {
|
|
945
|
+
fail('ATTEMPT_NOT_ACTIVE', 'execution launch reservation requires an active attempt', { attempt_id });
|
|
946
|
+
}
|
|
947
|
+
if (attempt.claim.claim_id !== claim_id || attempt.claim.operator_id !== operator_id) {
|
|
948
|
+
fail('CLAIM_OWNER_MISMATCH', 'execution launch reservation must belong to the active claim owner', { attempt_id });
|
|
949
|
+
}
|
|
950
|
+
if (attempt.claim.harness_process_id !== harness_process_id) {
|
|
951
|
+
fail('RAIL_CLAIM_PROCESS_MISMATCH', 'execution launch reservation must belong to the active supervisor process', { attempt_id });
|
|
952
|
+
}
|
|
953
|
+
this.#assertLiveLease(attempt, timestamp(this.#clock));
|
|
954
|
+
if (invocation.attempt_id !== attempt_id || invocation.action !== 'build'
|
|
955
|
+
|| invocation.prompt_kind === 'rework') {
|
|
956
|
+
fail('EXECUTION_INVOCATION_MISMATCH', 'initial execution launch reservation is not bound to this build attempt');
|
|
957
|
+
}
|
|
958
|
+
const binding = attempt.execution_binding;
|
|
959
|
+
if (!binding
|
|
960
|
+
|| invocation.provider_id !== binding.provider_id
|
|
961
|
+
|| invocation.harness_id !== binding.harness_id
|
|
962
|
+
|| invocation.model_pin?.model_id !== binding.model_id
|
|
963
|
+
|| invocation.model_pin?.reasoning_configuration !== binding.reasoning_configuration) {
|
|
964
|
+
fail('EXECUTION_BINDING_MISMATCH', 'execution launch reservation does not match the sealed task binding');
|
|
965
|
+
}
|
|
966
|
+
const existing = this.#records.find((record) => record.type === 'execution_launch_reserved'
|
|
967
|
+
&& record.payload?.invocation?.launch_nonce === invocation.launch_nonce);
|
|
968
|
+
if (existing) {
|
|
969
|
+
if (sha256(existing.payload.invocation) !== sha256(invocation)
|
|
970
|
+
|| existing.payload.claim_event_id !== attempt.claim_event_id) {
|
|
971
|
+
fail('EXECUTION_LAUNCH_NONCE_CONFLICT', 'launch nonce is already reserved for different execution material');
|
|
972
|
+
}
|
|
973
|
+
return Object.freeze(canonicalize({ ...existing.payload, event_id: existing.event_id, replay: true }));
|
|
974
|
+
}
|
|
975
|
+
const unresolvedReservation = this.#records.find((record) => record.type === 'execution_launch_reserved'
|
|
976
|
+
&& record.payload?.attempt_id === attempt_id
|
|
977
|
+
&& !this.#records.some((candidate) => candidate.type === 'execution_dispatched'
|
|
978
|
+
&& candidate.payload?.invocation?.launch_nonce === record.payload.invocation.launch_nonce));
|
|
979
|
+
if (unresolvedReservation) {
|
|
980
|
+
fail('RAIL_INITIAL_LAUNCH_UNRESOLVED', 'a prior initial launch reservation has no authenticated provider dispatch');
|
|
981
|
+
}
|
|
982
|
+
const unresolvedDispatch = this.#records.find((record) => record.type === 'execution_dispatched'
|
|
983
|
+
&& record.payload?.attempt_id === attempt_id
|
|
984
|
+
&& record.payload?.run_kind === 'initial'
|
|
985
|
+
&& !this.#records.some((candidate) => candidate.type === 'execution_recorded'
|
|
986
|
+
&& candidate.payload?.execution_evidence?.receipt_nonce === record.payload.invocation.launch_nonce));
|
|
987
|
+
if (unresolvedDispatch) {
|
|
988
|
+
fail('RAIL_INITIAL_DISPATCH_UNRESOLVED', 'a prior initial provider launch has no canonical completion or recovery reconciliation');
|
|
989
|
+
}
|
|
990
|
+
const payload = canonicalize({
|
|
991
|
+
attempt_id, claim_id, operator_id, harness_process_id,
|
|
992
|
+
claim_event_id: attempt.claim_event_id, invocation,
|
|
993
|
+
reserved_at: timestamp(this.#clock),
|
|
994
|
+
});
|
|
995
|
+
const record = this.#append('execution_launch_reserved', payload, attempt.claim_event_id);
|
|
996
|
+
return Object.freeze(canonicalize({ ...payload, event_id: record.event_id, replay: false }));
|
|
997
|
+
});
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
recordExecutionDispatch({
|
|
1001
|
+
attempt_id, claim_id, operator_id, invocation, launch_receipt, project_dir,
|
|
1002
|
+
browser_manifest = null, run_kind = 'initial', source_decision_ref = null, rework_reservation_id = null,
|
|
1003
|
+
} = {}) {
|
|
1004
|
+
return this.#exclusive(() => {
|
|
1005
|
+
for (const [field, value] of Object.entries({ attempt_id, claim_id, operator_id })) {
|
|
1006
|
+
assertString(value, 'INVALID_EXECUTION_DISPATCH', field);
|
|
1007
|
+
}
|
|
1008
|
+
assertObject(invocation, 'INVALID_EXECUTION_DISPATCH', 'invocation');
|
|
1009
|
+
assertObject(launch_receipt, 'INVALID_EXECUTION_DISPATCH', 'launch_receipt');
|
|
1010
|
+
assertString(invocation.launch_nonce, 'INVALID_EXECUTION_DISPATCH', 'invocation.launch_nonce');
|
|
1011
|
+
assertString(invocation.prompt_digest, 'INVALID_EXECUTION_DISPATCH', 'invocation.prompt_digest');
|
|
1012
|
+
assertString(project_dir, 'INVALID_EXECUTION_DISPATCH', 'project_dir');
|
|
1013
|
+
if (browser_manifest !== null) assertObject(browser_manifest, 'INVALID_EXECUTION_DISPATCH', 'browser_manifest');
|
|
1014
|
+
const attempt = this.#attempts().get(attempt_id);
|
|
1015
|
+
if (!attempt || attempt.state !== EXECUTABLE_ATTEMPT_STATE) fail('ATTEMPT_NOT_ACTIVE', 'execution dispatch requires an active attempt', { attempt_id });
|
|
1016
|
+
this.#verifyAuthority('execution_launch', launch_receipt, {
|
|
1017
|
+
purpose: 'terminal_launch', authority_kind: 'rail_execution',
|
|
1018
|
+
invocation_id: invocation.invocation_id, launch_nonce: invocation.launch_nonce,
|
|
1019
|
+
provider_id: invocation.provider_id, harness_id: invocation.harness_id,
|
|
1020
|
+
model_id: invocation.model_pin?.model_id,
|
|
1021
|
+
reasoning_configuration: invocation.model_pin?.reasoning_configuration,
|
|
1022
|
+
task_id: attempt?.task_id,
|
|
1023
|
+
working_dir: resolve(project_dir),
|
|
1024
|
+
prompt_digest: invocation.prompt_digest,
|
|
1025
|
+
});
|
|
1026
|
+
if (attempt.claim.claim_id !== claim_id || attempt.claim.operator_id !== operator_id) {
|
|
1027
|
+
fail('CLAIM_OWNER_MISMATCH', 'execution dispatch must be recorded by the active claim owner', { attempt_id });
|
|
1028
|
+
}
|
|
1029
|
+
this.#assertLiveLease(attempt, timestamp(this.#clock));
|
|
1030
|
+
if (invocation.attempt_id !== attempt_id || invocation.action !== 'build') {
|
|
1031
|
+
fail('EXECUTION_INVOCATION_MISMATCH', 'execution dispatch invocation is not bound to this build attempt');
|
|
1032
|
+
}
|
|
1033
|
+
if (!['initial', 'rework'].includes(run_kind)) fail('INVALID_EXECUTION_DISPATCH', 'run_kind must be initial or rework');
|
|
1034
|
+
if (run_kind === 'rework') {
|
|
1035
|
+
for (const [field, value] of Object.entries({ source_decision_ref, rework_reservation_id })) {
|
|
1036
|
+
assertString(value, 'INVALID_EXECUTION_DISPATCH', field);
|
|
1037
|
+
}
|
|
1038
|
+
if (invocation.prompt_kind !== 'rework'
|
|
1039
|
+
|| invocation.source_decision_ref !== source_decision_ref
|
|
1040
|
+
|| invocation.rework_reservation_id !== rework_reservation_id) {
|
|
1041
|
+
fail('EXECUTION_REWORK_RESERVATION_MISMATCH', 'rework dispatch must bind its canonical source decision and reservation');
|
|
1042
|
+
}
|
|
1043
|
+
const reservation = [...this.#records].reverse().find((record) => record.type === 'rework_reserved'
|
|
1044
|
+
&& record.payload?.attempt_id === attempt_id
|
|
1045
|
+
&& record.payload?.source_decision_ref === source_decision_ref);
|
|
1046
|
+
const released = this.#records.some((record) => record.type === 'rework_reservation_released'
|
|
1047
|
+
&& record.payload?.reservation_id === rework_reservation_id);
|
|
1048
|
+
if (!reservation
|
|
1049
|
+
|| released
|
|
1050
|
+
|| reservation.payload.reservation_id !== rework_reservation_id
|
|
1051
|
+
|| reservation.payload.claim_event_id !== attempt.claim_event_id
|
|
1052
|
+
|| reservation.payload.claim_id !== claim_id
|
|
1053
|
+
|| reservation.payload.operator_id !== operator_id) {
|
|
1054
|
+
fail('RAIL_REWORK_RESERVATION_REQUIRED', 'rework dispatch requires the active claim-generation reservation');
|
|
1055
|
+
}
|
|
1056
|
+
const launchReservation = this.#records.find((record) => record.type === 'rework_launch_reserved'
|
|
1057
|
+
&& record.payload?.invocation?.launch_nonce === invocation.launch_nonce);
|
|
1058
|
+
if (!launchReservation
|
|
1059
|
+
|| launchReservation.payload.attempt_id !== attempt_id
|
|
1060
|
+
|| launchReservation.payload.claim_id !== claim_id
|
|
1061
|
+
|| launchReservation.payload.operator_id !== operator_id
|
|
1062
|
+
|| launchReservation.payload.claim_event_id !== attempt.claim_event_id
|
|
1063
|
+
|| launchReservation.payload.source_decision_ref !== source_decision_ref
|
|
1064
|
+
|| launchReservation.payload.reservation_id !== rework_reservation_id
|
|
1065
|
+
|| sha256(launchReservation.payload.invocation) !== sha256(invocation)) {
|
|
1066
|
+
fail('RAIL_REWORK_LAUNCH_RESERVATION_REQUIRED', 'rework dispatch requires its exact canonical pre-launch intent');
|
|
1067
|
+
}
|
|
1068
|
+
const priorDispatches = this.#records.filter((record) => record.type === 'execution_dispatched'
|
|
1069
|
+
&& record.payload?.attempt_id === attempt_id
|
|
1070
|
+
&& record.payload?.invocation?.rework_reservation_id === rework_reservation_id);
|
|
1071
|
+
const unreceipted = priorDispatches.find((dispatch) => !this.#records.some((record) => record.type === 'execution_recorded'
|
|
1072
|
+
&& record.payload?.execution_evidence?.receipt_nonce === dispatch.payload.invocation.launch_nonce));
|
|
1073
|
+
if (unreceipted) {
|
|
1074
|
+
fail('RAIL_REWORK_DISPATCH_UNRESOLVED', 'a prior launch in this rework reservation has no canonical completion evidence');
|
|
1075
|
+
}
|
|
1076
|
+
} else if (source_decision_ref !== null || rework_reservation_id !== null || invocation.prompt_kind === 'rework') {
|
|
1077
|
+
fail('INVALID_EXECUTION_DISPATCH', 'initial execution cannot carry rework reservation material');
|
|
1078
|
+
}
|
|
1079
|
+
const binding = attempt.execution_binding;
|
|
1080
|
+
if (!binding
|
|
1081
|
+
|| invocation.provider_id !== binding.provider_id
|
|
1082
|
+
|| invocation.harness_id !== binding.harness_id
|
|
1083
|
+
|| invocation.model_pin?.model_id !== binding.model_id
|
|
1084
|
+
|| invocation.model_pin?.reasoning_configuration !== binding.reasoning_configuration) {
|
|
1085
|
+
fail('EXECUTION_BINDING_MISMATCH', 'execution dispatch does not match the sealed task provider, harness, model and reasoning binding');
|
|
1086
|
+
}
|
|
1087
|
+
const expectedExecutionBase = this.#executionEvidence(attempt_id, { successfulOnly: true })
|
|
1088
|
+
.at(-1)?.result_commit_ref ?? attempt.git_commit_ref;
|
|
1089
|
+
if (launch_receipt.purpose !== 'terminal_launch'
|
|
1090
|
+
|| launch_receipt.authority_kind !== 'rail_execution'
|
|
1091
|
+
|| launch_receipt.invocation_id !== invocation.invocation_id
|
|
1092
|
+
|| launch_receipt.launch_nonce !== invocation.launch_nonce
|
|
1093
|
+
|| launch_receipt.provider_id !== invocation.provider_id
|
|
1094
|
+
|| launch_receipt.harness_id !== invocation.harness_id
|
|
1095
|
+
|| launch_receipt.model_id !== invocation.model_pin?.model_id
|
|
1096
|
+
|| launch_receipt.reasoning_configuration !== invocation.model_pin?.reasoning_configuration
|
|
1097
|
+
|| launch_receipt.task_id !== attempt.task_id
|
|
1098
|
+
|| launch_receipt.working_dir !== resolve(project_dir)
|
|
1099
|
+
|| launch_receipt.prompt_digest !== invocation.prompt_digest
|
|
1100
|
+
|| launch_receipt.execution_base_commit_ref !== expectedExecutionBase
|
|
1101
|
+
|| launch_receipt.execution_launch_head_ref !== expectedExecutionBase
|
|
1102
|
+
|| launch_receipt.execution_worktree_clean !== true
|
|
1103
|
+
|| !/^[a-f0-9]{64}$/.test(launch_receipt.execution_repository_identity_digest || '')
|
|
1104
|
+
|| !/^[a-f0-9]{64}$/.test(launch_receipt.execution_preexisting_commits_digest || '')
|
|
1105
|
+
|| !/^[a-f0-9]{64}$/.test(launch_receipt.command_digest || '')
|
|
1106
|
+
|| !Number.isInteger(launch_receipt.process_pid) || launch_receipt.process_pid <= 0
|
|
1107
|
+
|| typeof launch_receipt.process_hostname !== 'string' || !launch_receipt.process_hostname
|
|
1108
|
+
|| !(launch_receipt.process_group_id === null
|
|
1109
|
+
|| (Number.isInteger(launch_receipt.process_group_id) && launch_receipt.process_group_id > 0))
|
|
1110
|
+
|| launch_receipt.process_containment_kind !== 'bubblewrap_pid_namespace_v1'
|
|
1111
|
+
|| typeof launch_receipt.process_containment_id !== 'string' || !launch_receipt.process_containment_id
|
|
1112
|
+
|| !/^[a-f0-9]{64}$/.test(launch_receipt.process_containment_digest || '')
|
|
1113
|
+
|| typeof launch_receipt.authority_signature !== 'string') {
|
|
1114
|
+
fail('EXECUTION_LAUNCH_RECEIPT_MISMATCH', 'launch receipt does not echo the exact canonical invocation');
|
|
1115
|
+
}
|
|
1116
|
+
const existing = this.#records.find((record) => record.type === 'execution_dispatched'
|
|
1117
|
+
&& record.payload?.invocation?.launch_nonce === invocation.launch_nonce);
|
|
1118
|
+
if (existing) {
|
|
1119
|
+
if (sha256(existing.payload.invocation) !== sha256(invocation)
|
|
1120
|
+
|| sha256(existing.payload.launch_receipt) !== sha256(launch_receipt)
|
|
1121
|
+
|| sha256(existing.payload.browser_manifest ?? null) !== sha256(browser_manifest)) {
|
|
1122
|
+
fail('EXECUTION_LAUNCH_NONCE_CONFLICT', 'launch nonce is already bound to different dispatch material');
|
|
1123
|
+
}
|
|
1124
|
+
return Object.freeze(canonicalize({ ...existing.payload, event_id: existing.event_id, replay: true }));
|
|
1125
|
+
}
|
|
1126
|
+
if (run_kind === 'initial') {
|
|
1127
|
+
const reservation = this.#records.find((record) => record.type === 'execution_launch_reserved'
|
|
1128
|
+
&& record.payload?.invocation?.launch_nonce === invocation.launch_nonce);
|
|
1129
|
+
if (!reservation
|
|
1130
|
+
|| reservation.payload.attempt_id !== attempt_id
|
|
1131
|
+
|| reservation.payload.claim_id !== claim_id
|
|
1132
|
+
|| reservation.payload.operator_id !== operator_id
|
|
1133
|
+
|| reservation.payload.claim_event_id !== attempt.claim_event_id
|
|
1134
|
+
|| sha256(reservation.payload.invocation) !== sha256(invocation)) {
|
|
1135
|
+
fail('RAIL_EXECUTION_LAUNCH_RESERVATION_REQUIRED', 'initial execution dispatch requires its exact canonical pre-launch reservation');
|
|
1136
|
+
}
|
|
1137
|
+
const unresolved = this.#records.find((record) => record.type === 'execution_dispatched'
|
|
1138
|
+
&& record.payload?.attempt_id === attempt_id
|
|
1139
|
+
&& record.payload?.run_kind === 'initial'
|
|
1140
|
+
&& !this.#records.some((candidate) => candidate.type === 'execution_recorded'
|
|
1141
|
+
&& candidate.payload?.execution_evidence?.receipt_nonce === record.payload.invocation.launch_nonce));
|
|
1142
|
+
if (unresolved) {
|
|
1143
|
+
fail('RAIL_INITIAL_DISPATCH_UNRESOLVED', 'a prior initial provider launch has no canonical completion or recovery reconciliation');
|
|
1144
|
+
}
|
|
1145
|
+
}
|
|
1146
|
+
const record = this.#append('execution_dispatched', {
|
|
1147
|
+
attempt_id, claim_id, operator_id, invocation, launch_receipt,
|
|
1148
|
+
...(browser_manifest ? { browser_manifest } : {}),
|
|
1149
|
+
run_kind, ...(source_decision_ref ? { source_decision_ref } : {}),
|
|
1150
|
+
...(rework_reservation_id ? { rework_reservation_id } : {}),
|
|
1151
|
+
});
|
|
1152
|
+
return Object.freeze(canonicalize({ ...record.payload, event_id: record.event_id, replay: false }));
|
|
1153
|
+
});
|
|
1154
|
+
}
|
|
1155
|
+
|
|
1156
|
+
recordExecutionEvidence({
|
|
1157
|
+
attempt_id, claim_id, operator_id, invocation, execution, base_commit_ref,
|
|
1158
|
+
result_commit_ref = null, run_kind = 'initial', source_decision_ref = null,
|
|
1159
|
+
receipt_nonce, git_dir,
|
|
1160
|
+
} = {}) {
|
|
1161
|
+
return this.#exclusive(() => {
|
|
1162
|
+
for (const [field, value] of Object.entries({ attempt_id, claim_id, operator_id, receipt_nonce, git_dir })) {
|
|
1163
|
+
assertString(value, 'INVALID_EXECUTION_EVIDENCE', field);
|
|
1164
|
+
}
|
|
1165
|
+
assertObject(invocation, 'INVALID_EXECUTION_EVIDENCE', 'invocation');
|
|
1166
|
+
assertObject(execution, 'INVALID_EXECUTION_EVIDENCE', 'execution');
|
|
1167
|
+
this.#verifyAuthority('execution_completion', execution.terminal_authority_receipt, {
|
|
1168
|
+
purpose: 'terminal_completion', authority_kind: 'rail_execution',
|
|
1169
|
+
invocation_id: invocation.invocation_id, launch_nonce: invocation.launch_nonce,
|
|
1170
|
+
terminal_id: execution.terminal_id, provider_id: invocation.provider_id,
|
|
1171
|
+
harness_id: invocation.harness_id, model_id: invocation.model_pin?.model_id,
|
|
1172
|
+
reasoning_configuration: invocation.model_pin?.reasoning_configuration,
|
|
1173
|
+
exit_status: execution.exit_status,
|
|
1174
|
+
});
|
|
1175
|
+
if (!['initial', 'rework'].includes(run_kind)) fail('INVALID_EXECUTION_EVIDENCE', 'run_kind must be initial or rework');
|
|
1176
|
+
if (!Number.isInteger(execution.exit_status)) fail('INVALID_EXECUTION_EVIDENCE', 'execution.exit_status must be an integer');
|
|
1177
|
+
const attempt = this.#attempts().get(attempt_id);
|
|
1178
|
+
if (!attempt || attempt.state !== EXECUTABLE_ATTEMPT_STATE) fail('ATTEMPT_NOT_ACTIVE', 'execution evidence requires an active attempt', { attempt_id });
|
|
1179
|
+
if (attempt.claim.claim_id !== claim_id || attempt.claim.operator_id !== operator_id) {
|
|
1180
|
+
fail('CLAIM_OWNER_MISMATCH', 'execution evidence must be receipted by the active claim owner', { attempt_id });
|
|
1181
|
+
}
|
|
1182
|
+
this.#assertLiveLease(attempt, timestamp(this.#clock));
|
|
1183
|
+
if (invocation.attempt_id !== attempt_id) fail('EXECUTION_INVOCATION_MISMATCH', 'execution invocation is not bound to the active attempt');
|
|
1184
|
+
if (receipt_nonce !== invocation.launch_nonce) fail('EXECUTION_LAUNCH_NONCE_MISMATCH', 'execution evidence must echo the dispatched launch nonce');
|
|
1185
|
+
const dispatch = this.#records.find((record) => record.type === 'execution_dispatched'
|
|
1186
|
+
&& record.payload?.attempt_id === attempt_id
|
|
1187
|
+
&& record.payload?.claim_id === claim_id
|
|
1188
|
+
&& record.payload?.operator_id === operator_id
|
|
1189
|
+
&& record.payload?.invocation?.launch_nonce === receipt_nonce);
|
|
1190
|
+
if (!dispatch || sha256(dispatch.payload.invocation) !== sha256(invocation)) {
|
|
1191
|
+
fail('EXECUTION_DISPATCH_REQUIRED', 'execution evidence requires an exact canonical dispatch record');
|
|
1192
|
+
}
|
|
1193
|
+
if (execution.terminal_id !== dispatch.payload.launch_receipt.terminal_id
|
|
1194
|
+
|| execution.provider !== invocation.harness_id
|
|
1195
|
+
|| execution.model !== invocation.model_pin?.model_id
|
|
1196
|
+
|| execution.reasoning_configuration !== invocation.model_pin?.reasoning_configuration) {
|
|
1197
|
+
fail('EXECUTION_TERMINAL_EVIDENCE_MISMATCH', 'terminal evidence does not match the canonical dispatch receipt and model binding');
|
|
1198
|
+
}
|
|
1199
|
+
if (execution.terminal_authority_receipt?.purpose !== 'terminal_completion'
|
|
1200
|
+
|| execution.terminal_authority_receipt?.authority_kind !== 'rail_execution'
|
|
1201
|
+
|| execution.terminal_authority_receipt?.launch_receipt_digest !== sha256(dispatch.payload.launch_receipt)
|
|
1202
|
+
|| execution.terminal_authority_receipt?.launch_nonce !== receipt_nonce
|
|
1203
|
+
|| execution.terminal_authority_receipt?.terminal_id !== execution.terminal_id
|
|
1204
|
+
|| execution.terminal_authority_receipt?.exit_status !== execution.exit_status
|
|
1205
|
+
|| execution.terminal_authority_receipt?.execution_base_commit_ref !== base_commit_ref
|
|
1206
|
+
|| execution.terminal_authority_receipt?.execution_launch_head_ref !== dispatch.payload.launch_receipt.execution_launch_head_ref
|
|
1207
|
+
|| execution.terminal_authority_receipt?.execution_repository_identity_digest !== dispatch.payload.launch_receipt.execution_repository_identity_digest
|
|
1208
|
+
|| execution.terminal_authority_receipt?.execution_preexisting_commits_digest !== dispatch.payload.launch_receipt.execution_preexisting_commits_digest
|
|
1209
|
+
|| typeof execution.terminal_authority_receipt?.authority_signature !== 'string') {
|
|
1210
|
+
fail('EXECUTION_TERMINAL_AUTHORITY_MISMATCH', 'completion receipt does not bind the canonical launch and terminal result');
|
|
1211
|
+
}
|
|
1212
|
+
const resultStatus = execution.terminal_authority_receipt.execution_result_status;
|
|
1213
|
+
const invalidOutput = execution.exit_status === 0 && resultStatus === 'invalid_output';
|
|
1214
|
+
let executionFailureCode = null;
|
|
1215
|
+
let canonicalResultCommitRef = result_commit_ref;
|
|
1216
|
+
let reportedResultCommitRef = null;
|
|
1217
|
+
if (invalidOutput) {
|
|
1218
|
+
if (![
|
|
1219
|
+
'RAIL_EXECUTION_OUTPUT_INVALID',
|
|
1220
|
+
'RAIL_EXECUTION_LAUNCH_NONCE_MISMATCH',
|
|
1221
|
+
'RAIL_EXECUTION_REPOSITORY_CHANGED',
|
|
1222
|
+
'RAIL_EXECUTION_RESULT_PREEXISTED',
|
|
1223
|
+
'RAIL_EXECUTION_RESULT_HEAD_MISMATCH',
|
|
1224
|
+
'RAIL_EXECUTION_RESULT_NOT_DESCENDANT',
|
|
1225
|
+
'RAIL_EXECUTION_WORKTREE_DIRTY',
|
|
1226
|
+
'RAIL_EXECUTION_GIT_STATE_UNAVAILABLE',
|
|
1227
|
+
]
|
|
1228
|
+
.includes(execution.terminal_authority_receipt.execution_result_error_code)
|
|
1229
|
+
|| execution.terminal_authority_receipt.result_commit_ref !== undefined
|
|
1230
|
+
|| result_commit_ref !== null) {
|
|
1231
|
+
fail('EXECUTION_RESULT_RECEIPT_MISMATCH', 'invalid execution output must be signed as a result-less terminal contract failure');
|
|
1232
|
+
}
|
|
1233
|
+
executionFailureCode = execution.terminal_authority_receipt.execution_result_error_code;
|
|
1234
|
+
} else if (execution.exit_status === 0) {
|
|
1235
|
+
if (resultStatus !== undefined && resultStatus !== 'valid') {
|
|
1236
|
+
fail('EXECUTION_RESULT_RECEIPT_MISMATCH', 'successful terminal execution has an invalid signed result status');
|
|
1237
|
+
}
|
|
1238
|
+
if (execution.terminal_authority_receipt.result_commit_ref !== result_commit_ref) {
|
|
1239
|
+
fail('EXECUTION_RESULT_RECEIPT_MISMATCH', 'result commit must be the commit parsed and signed by the terminal authority');
|
|
1240
|
+
}
|
|
1241
|
+
if (execution.terminal_authority_receipt.execution_result_head_ref !== result_commit_ref
|
|
1242
|
+
|| execution.terminal_authority_receipt.execution_result_preexisted !== false
|
|
1243
|
+
|| execution.terminal_authority_receipt.execution_worktree_clean !== true) {
|
|
1244
|
+
fail('EXECUTION_RESULT_RECEIPT_MISMATCH', 'successful execution must prove a new clean final HEAD created after launch');
|
|
1245
|
+
}
|
|
1246
|
+
} else {
|
|
1247
|
+
if (resultStatus !== undefined && resultStatus !== 'process_failed') {
|
|
1248
|
+
fail('EXECUTION_RESULT_RECEIPT_MISMATCH', 'failed terminal execution has an invalid signed result status');
|
|
1249
|
+
}
|
|
1250
|
+
executionFailureCode = 'TERMINAL_EXIT_NONZERO';
|
|
1251
|
+
}
|
|
1252
|
+
if (this.#records.some((record) => record.type === 'execution_recorded'
|
|
1253
|
+
&& record.payload?.execution_evidence?.receipt_nonce === receipt_nonce)) {
|
|
1254
|
+
fail('EXECUTION_LAUNCH_ALREADY_RECEIPTED', 'this canonical launch already has execution evidence');
|
|
1255
|
+
}
|
|
1256
|
+
const successful = this.#executionEvidence(attempt_id, { successfulOnly: true });
|
|
1257
|
+
const expectedBase = successful.at(-1)?.result_commit_ref ?? attempt.git_commit_ref;
|
|
1258
|
+
if (base_commit_ref !== expectedBase) fail('EXECUTION_BASE_MISMATCH', 'execution base must equal the latest canonical successful result or sealed base');
|
|
1259
|
+
if (run_kind === 'rework') {
|
|
1260
|
+
assertString(source_decision_ref, 'INVALID_EXECUTION_EVIDENCE', 'source_decision_ref');
|
|
1261
|
+
assertString(invocation.rework_reservation_id, 'INVALID_EXECUTION_EVIDENCE', 'invocation.rework_reservation_id');
|
|
1262
|
+
if (invocation.prompt_kind !== 'rework'
|
|
1263
|
+
|| invocation.source_decision_ref !== source_decision_ref
|
|
1264
|
+
|| dispatch.payload.run_kind !== 'rework'
|
|
1265
|
+
|| dispatch.payload.source_decision_ref !== source_decision_ref
|
|
1266
|
+
|| dispatch.payload.rework_reservation_id !== invocation.rework_reservation_id) {
|
|
1267
|
+
fail('EXECUTION_REWORK_RESERVATION_MISMATCH', 'rework evidence must bind the canonical reservation and source decision');
|
|
1268
|
+
}
|
|
1269
|
+
const authorized = this.#records.some((record) => record.payload?.attempt_id === attempt_id && (
|
|
1270
|
+
(record.type === 'review_decided'
|
|
1271
|
+
&& record.payload.review_ref === source_decision_ref
|
|
1272
|
+
&& record.payload.decision?.decision === 'normal-rework')
|
|
1273
|
+
|| (record.type === 'adjudication_decided'
|
|
1274
|
+
&& record.payload.adjudication_ref === source_decision_ref
|
|
1275
|
+
&& record.payload.decision?.outcome === 'bounded-correction')
|
|
1276
|
+
));
|
|
1277
|
+
if (!authorized) fail('EXECUTION_REWORK_NOT_AUTHORIZED', 'rework execution must reference a canonical rework decision');
|
|
1278
|
+
} else if (source_decision_ref !== null) {
|
|
1279
|
+
fail('INVALID_EXECUTION_EVIDENCE', 'initial execution must not carry source_decision_ref');
|
|
1280
|
+
}
|
|
1281
|
+
if (execution.exit_status === 0 && !invalidOutput) {
|
|
1282
|
+
assertString(result_commit_ref, 'INVALID_EXECUTION_EVIDENCE', 'result_commit_ref');
|
|
1283
|
+
try {
|
|
1284
|
+
verifyCommitDescendsFrom({ git_dir, base_commit_ref, result_commit_ref });
|
|
1285
|
+
} catch (error) {
|
|
1286
|
+
if (!['GIT_COMMIT_NOT_FOUND', 'GIT_RESULT_COMMIT_REQUIRED', 'GIT_RESULT_NOT_DESCENDANT'].includes(error?.code)) throw error;
|
|
1287
|
+
executionFailureCode = error.code;
|
|
1288
|
+
reportedResultCommitRef = result_commit_ref;
|
|
1289
|
+
canonicalResultCommitRef = null;
|
|
1290
|
+
}
|
|
1291
|
+
if (run_kind === 'rework' && successful.some((entry) => entry.source_decision_ref === source_decision_ref)) {
|
|
1292
|
+
fail('RAIL_REWORK_ALREADY_EXECUTED', 'this decision already has a successful rework execution');
|
|
1293
|
+
}
|
|
1294
|
+
} else if (result_commit_ref !== null) {
|
|
1295
|
+
fail('INVALID_EXECUTION_EVIDENCE', 'failed execution cannot bind a result commit');
|
|
1296
|
+
}
|
|
1297
|
+
const core = canonicalize({
|
|
1298
|
+
schema_version: 2,
|
|
1299
|
+
attempt_id,
|
|
1300
|
+
task_id: attempt.task_id,
|
|
1301
|
+
handoff_id: attempt.handoff_id,
|
|
1302
|
+
run_kind,
|
|
1303
|
+
base_commit_ref,
|
|
1304
|
+
outcome: canonicalResultCommitRef ? 'succeeded' : 'failed',
|
|
1305
|
+
...(canonicalResultCommitRef ? { result_commit_ref: canonicalResultCommitRef } : {}),
|
|
1306
|
+
...(reportedResultCommitRef ? { reported_result_commit_ref: reportedResultCommitRef } : {}),
|
|
1307
|
+
...(executionFailureCode ? { failure_code: executionFailureCode } : {}),
|
|
1308
|
+
...(source_decision_ref ? { source_decision_ref } : {}),
|
|
1309
|
+
receipt_nonce,
|
|
1310
|
+
invocation,
|
|
1311
|
+
execution,
|
|
1312
|
+
});
|
|
1313
|
+
const evidence_id = `execution-${sha256(core).slice(0, 24)}`;
|
|
1314
|
+
const existing = this.#records.find((record) => record.type === 'execution_recorded'
|
|
1315
|
+
&& record.payload?.execution_evidence?.evidence_id === evidence_id);
|
|
1316
|
+
if (existing) return Object.freeze(canonicalize({ ...existing.payload.execution_evidence, event_id: existing.event_id, replay: true }));
|
|
1317
|
+
const evidence = canonicalize({ ...core, evidence_id, digest: sha256(core) });
|
|
1318
|
+
const record = this.#append('execution_recorded', { attempt_id, execution_evidence: evidence });
|
|
1319
|
+
return Object.freeze(canonicalize({ ...evidence, event_id: record.event_id, replay: false }));
|
|
1320
|
+
});
|
|
1321
|
+
}
|
|
1322
|
+
|
|
221
1323
|
#currentClaimForTask(taskId) {
|
|
222
1324
|
const now = timestamp(this.#clock);
|
|
223
1325
|
for (const attempt of this.#attempts().values()) {
|
|
@@ -244,6 +1346,7 @@ export class RailEngine {
|
|
|
244
1346
|
if (!latest || latest.payload.decision?.decision !== 'accept' || !uniqueReviews.has(latest.payload.review_ref)) {
|
|
245
1347
|
fail('REVIEW_NOT_ACCEPTED', 'completion requires the latest canonical review decision to accept the attempt', { attempt_id: attemptId });
|
|
246
1348
|
}
|
|
1349
|
+
return latest;
|
|
247
1350
|
}
|
|
248
1351
|
|
|
249
1352
|
claimTask({ handoff, task_id, attempt_id, operator_id, harness_process_id, git_dir, lease_ttl_ms = 300000 } = {}) {
|
|
@@ -266,6 +1369,7 @@ export class RailEngine {
|
|
|
266
1369
|
attempt_id,
|
|
267
1370
|
operator_id,
|
|
268
1371
|
harness_process_id,
|
|
1372
|
+
hostname: hostname(),
|
|
269
1373
|
lease: { acquired_at, expires_at: plusMs(acquired_at, lease_ttl_ms), heartbeat_at: acquired_at },
|
|
270
1374
|
idempotency_scope: `task:${task_id}:attempt:${attempt_id}`,
|
|
271
1375
|
});
|
|
@@ -274,22 +1378,285 @@ export class RailEngine {
|
|
|
274
1378
|
});
|
|
275
1379
|
}
|
|
276
1380
|
|
|
277
|
-
heartbeat({ attempt_id, claim_id, operator_id, lease_ttl_ms = 300000 } = {}) {
|
|
1381
|
+
heartbeat({ attempt_id, claim_id, operator_id, harness_process_id, lease_ttl_ms = 300000 } = {}) {
|
|
278
1382
|
return this.#exclusive(() => {
|
|
279
1383
|
assertString(attempt_id, 'INVALID_HEARTBEAT', 'attempt_id');
|
|
280
1384
|
assertString(claim_id, 'INVALID_HEARTBEAT', 'claim_id');
|
|
281
1385
|
assertString(operator_id, 'INVALID_HEARTBEAT', 'operator_id');
|
|
1386
|
+
assertString(harness_process_id, 'INVALID_HEARTBEAT', 'harness_process_id');
|
|
282
1387
|
const attempt = this.#attempts().get(attempt_id);
|
|
283
1388
|
if (!attempt || ![EXECUTABLE_ATTEMPT_STATE, 'ready_to_merge'].includes(attempt.state)) fail('ATTEMPT_NOT_ACTIVE', 'attempt is not active', { attempt_id });
|
|
284
1389
|
if (attempt.claim.claim_id !== claim_id || attempt.claim.operator_id !== operator_id) fail('CLAIM_OWNER_MISMATCH', 'only the active claim owner may heartbeat', { attempt_id });
|
|
1390
|
+
if (attempt.claim.harness_process_id !== harness_process_id) {
|
|
1391
|
+
fail('RAIL_CLAIM_PROCESS_MISMATCH', 'only the process bound to the active claim may heartbeat', { attempt_id });
|
|
1392
|
+
}
|
|
285
1393
|
const heartbeat_at = timestamp(this.#clock);
|
|
286
1394
|
this.#assertLiveLease(attempt, heartbeat_at);
|
|
287
|
-
const
|
|
1395
|
+
const requestedExpiry = plusMs(heartbeat_at, lease_ttl_ms);
|
|
1396
|
+
const expires_at = Date.parse(requestedExpiry) > Date.parse(attempt.claim.lease.expires_at)
|
|
1397
|
+
? requestedExpiry
|
|
1398
|
+
: attempt.claim.lease.expires_at;
|
|
1399
|
+
const lease = { ...attempt.claim.lease, heartbeat_at, expires_at };
|
|
288
1400
|
const record = this.#append('attempt_heartbeat', { attempt_id, claim_id, lease }, attempt.claim_event_id);
|
|
289
1401
|
return Object.freeze(canonicalize({ attempt_id, claim_id, lease, event_id: record.event_id }));
|
|
290
1402
|
});
|
|
291
1403
|
}
|
|
292
1404
|
|
|
1405
|
+
relinquishAttempt({ attempt_id, claim_id, operator_id, harness_process_id, reason = 'supervisor_exit' } = {}) {
|
|
1406
|
+
return this.#exclusive(() => {
|
|
1407
|
+
for (const [field, value] of Object.entries({ attempt_id, claim_id, operator_id, harness_process_id, reason })) {
|
|
1408
|
+
assertString(value, 'INVALID_RELINQUISH', field);
|
|
1409
|
+
}
|
|
1410
|
+
const attempt = this.#attempts().get(attempt_id);
|
|
1411
|
+
if (!attempt || ![EXECUTABLE_ATTEMPT_STATE, 'ready_to_merge'].includes(attempt.state)) {
|
|
1412
|
+
fail('ATTEMPT_NOT_ACTIVE', 'only an active attempt can relinquish its lease', { attempt_id });
|
|
1413
|
+
}
|
|
1414
|
+
if (attempt.claim.claim_id !== claim_id || attempt.claim.operator_id !== operator_id) {
|
|
1415
|
+
fail('CLAIM_OWNER_MISMATCH', 'only the active claim owner may relinquish its lease', { attempt_id });
|
|
1416
|
+
}
|
|
1417
|
+
if (attempt.claim.harness_process_id !== harness_process_id) {
|
|
1418
|
+
fail('RAIL_CLAIM_PROCESS_MISMATCH', 'only the process bound to the active claim may relinquish its lease', { attempt_id });
|
|
1419
|
+
}
|
|
1420
|
+
const relinquished_at = timestamp(this.#clock);
|
|
1421
|
+
const lease = { ...attempt.claim.lease, expires_at: relinquished_at };
|
|
1422
|
+
const record = this.#append('attempt_relinquished', {
|
|
1423
|
+
attempt_id, claim_id, harness_process_id, reason, relinquished_at, lease,
|
|
1424
|
+
}, attempt.claim_event_id);
|
|
1425
|
+
return Object.freeze(canonicalize({ attempt_id, claim_id, reason, relinquished_at, lease, event_id: record.event_id }));
|
|
1426
|
+
});
|
|
1427
|
+
}
|
|
1428
|
+
|
|
1429
|
+
reserveRework({
|
|
1430
|
+
attempt_id, claim_id, operator_id, harness_process_id,
|
|
1431
|
+
source_decision_ref, reservation_id,
|
|
1432
|
+
} = {}) {
|
|
1433
|
+
return this.#exclusive(() => {
|
|
1434
|
+
for (const [field, value] of Object.entries({
|
|
1435
|
+
attempt_id, claim_id, operator_id, harness_process_id, source_decision_ref, reservation_id,
|
|
1436
|
+
})) assertString(value, 'INVALID_REWORK_RESERVATION', field);
|
|
1437
|
+
const attempt = this.#attempts().get(attempt_id);
|
|
1438
|
+
if (!attempt || attempt.state !== EXECUTABLE_ATTEMPT_STATE) {
|
|
1439
|
+
fail('ATTEMPT_NOT_ACTIVE', 'rework reservation requires an active attempt', { attempt_id });
|
|
1440
|
+
}
|
|
1441
|
+
if (attempt.claim.claim_id !== claim_id || attempt.claim.operator_id !== operator_id) {
|
|
1442
|
+
fail('CLAIM_OWNER_MISMATCH', 'rework reservation must belong to the active claim owner', { attempt_id });
|
|
1443
|
+
}
|
|
1444
|
+
if (attempt.claim.harness_process_id !== harness_process_id) {
|
|
1445
|
+
fail('RAIL_CLAIM_PROCESS_MISMATCH', 'rework reservation must belong to the active supervisor process', { attempt_id });
|
|
1446
|
+
}
|
|
1447
|
+
this.#assertLiveLease(attempt, timestamp(this.#clock));
|
|
1448
|
+
const latestControl = this.#records.filter((record) => ['review_decided', 'adjudication_decided'].includes(record.type)
|
|
1449
|
+
&& record.payload?.attempt_id === attempt_id).at(-1);
|
|
1450
|
+
const authorized = (latestControl?.type === 'review_decided'
|
|
1451
|
+
&& latestControl.payload.review_ref === source_decision_ref
|
|
1452
|
+
&& latestControl.payload.decision?.decision === 'normal-rework')
|
|
1453
|
+
|| (latestControl?.type === 'adjudication_decided'
|
|
1454
|
+
&& latestControl.payload.adjudication_ref === source_decision_ref
|
|
1455
|
+
&& latestControl.payload.decision?.outcome === 'bounded-correction');
|
|
1456
|
+
if (!authorized) fail('RAIL_REWORK_NOT_AUTHORIZED', 'the latest canonical decision does not authorize this rework reservation');
|
|
1457
|
+
if (this.#executionEvidence(attempt_id, { successfulOnly: true })
|
|
1458
|
+
.some((entry) => entry.source_decision_ref === source_decision_ref)) {
|
|
1459
|
+
fail('RAIL_REWORK_ALREADY_EXECUTED', 'this decision already has a successful rework execution');
|
|
1460
|
+
}
|
|
1461
|
+
const existingReservations = this.#records.filter((record) => record.type === 'rework_reserved'
|
|
1462
|
+
&& record.payload?.attempt_id === attempt_id
|
|
1463
|
+
&& record.payload?.source_decision_ref === source_decision_ref);
|
|
1464
|
+
for (const existing of existingReservations) {
|
|
1465
|
+
const released = this.#records.find((record) => record.type === 'rework_reservation_released'
|
|
1466
|
+
&& record.payload?.reservation_id === existing.payload.reservation_id);
|
|
1467
|
+
if (existing.payload.reservation_id === reservation_id
|
|
1468
|
+
&& existing.payload.claim_event_id === attempt.claim_event_id
|
|
1469
|
+
&& existing.payload.harness_process_id === harness_process_id
|
|
1470
|
+
&& !released) {
|
|
1471
|
+
return Object.freeze(canonicalize({ ...existing.payload, event_id: existing.event_id, replay: true }));
|
|
1472
|
+
}
|
|
1473
|
+
if (!released && existing.payload.claim_event_id === attempt.claim_event_id) {
|
|
1474
|
+
fail('RAIL_REWORK_ALREADY_RESERVED', 'this decision already has an active rework reservation');
|
|
1475
|
+
}
|
|
1476
|
+
if (!released) {
|
|
1477
|
+
const launches = this.#records.filter((record) => record.type === 'rework_launch_reserved'
|
|
1478
|
+
&& record.payload?.attempt_id === attempt_id
|
|
1479
|
+
&& record.payload?.reservation_id === existing.payload.reservation_id);
|
|
1480
|
+
const dispatches = this.#records.filter((record) => record.type === 'execution_dispatched'
|
|
1481
|
+
&& record.payload?.attempt_id === attempt_id
|
|
1482
|
+
&& record.payload?.run_kind === 'rework'
|
|
1483
|
+
&& record.payload?.rework_reservation_id === existing.payload.reservation_id);
|
|
1484
|
+
const unresolvedLaunches = launches.filter((launch) => !dispatches.some((dispatch) =>
|
|
1485
|
+
dispatch.payload?.invocation?.launch_nonce === launch.payload.invocation.launch_nonce));
|
|
1486
|
+
if (unresolvedLaunches.length > 0) {
|
|
1487
|
+
fail('RAIL_REWORK_LAUNCH_UNRESOLVED', 'the previous rework reservation has a launch intent without authenticated provider dispatch', {
|
|
1488
|
+
reservation_id: existing.payload.reservation_id,
|
|
1489
|
+
launch_nonces: unresolvedLaunches.map((launch) => launch.payload.invocation.launch_nonce),
|
|
1490
|
+
});
|
|
1491
|
+
}
|
|
1492
|
+
const unresolved = dispatches.filter((dispatch) => !this.#records.some((record) => record.type === 'execution_recorded'
|
|
1493
|
+
&& record.payload?.execution_evidence?.receipt_nonce === dispatch.payload.invocation.launch_nonce));
|
|
1494
|
+
if (unresolved.length > 0) {
|
|
1495
|
+
fail('RAIL_REWORK_DISPATCH_UNRESOLVED', 'the previous rework reservation has a dispatch without canonical execution evidence', {
|
|
1496
|
+
reservation_id: existing.payload.reservation_id,
|
|
1497
|
+
launch_nonces: unresolved.map((dispatch) => dispatch.payload.invocation.launch_nonce),
|
|
1498
|
+
});
|
|
1499
|
+
}
|
|
1500
|
+
if (!attempt.recovery_event_id) {
|
|
1501
|
+
fail('RAIL_REWORK_RECONCILIATION_REQUIRED', 'a previous rework reservation may only be reconciled by a recovered supervisor');
|
|
1502
|
+
}
|
|
1503
|
+
this.#append('rework_reservation_released', canonicalize({
|
|
1504
|
+
attempt_id, claim_id, operator_id, harness_process_id,
|
|
1505
|
+
source_decision_ref, reservation_id: existing.payload.reservation_id,
|
|
1506
|
+
reason: dispatches.length > 0 ? 'recovered_after_canonical_execution' : 'recovered_before_launch_intent',
|
|
1507
|
+
reconciled_claim_event_id: existing.payload.claim_event_id,
|
|
1508
|
+
}), attempt.recovery_event_id);
|
|
1509
|
+
}
|
|
1510
|
+
}
|
|
1511
|
+
const payload = canonicalize({
|
|
1512
|
+
attempt_id, claim_id, operator_id, harness_process_id,
|
|
1513
|
+
claim_event_id: attempt.claim_event_id, source_decision_ref, reservation_id,
|
|
1514
|
+
});
|
|
1515
|
+
const record = this.#append('rework_reserved', payload);
|
|
1516
|
+
return Object.freeze(canonicalize({ ...payload, event_id: record.event_id, replay: false }));
|
|
1517
|
+
});
|
|
1518
|
+
}
|
|
1519
|
+
|
|
1520
|
+
reserveReworkLaunch({
|
|
1521
|
+
attempt_id, claim_id, operator_id, harness_process_id,
|
|
1522
|
+
source_decision_ref, reservation_id, invocation,
|
|
1523
|
+
} = {}) {
|
|
1524
|
+
return this.#exclusive(() => {
|
|
1525
|
+
for (const [field, value] of Object.entries({
|
|
1526
|
+
attempt_id, claim_id, operator_id, harness_process_id,
|
|
1527
|
+
source_decision_ref, reservation_id,
|
|
1528
|
+
})) assertString(value, 'INVALID_REWORK_LAUNCH_RESERVATION', field);
|
|
1529
|
+
assertObject(invocation, 'INVALID_REWORK_LAUNCH_RESERVATION', 'invocation');
|
|
1530
|
+
assertString(invocation.invocation_id, 'INVALID_REWORK_LAUNCH_RESERVATION', 'invocation.invocation_id');
|
|
1531
|
+
assertString(invocation.launch_nonce, 'INVALID_REWORK_LAUNCH_RESERVATION', 'invocation.launch_nonce');
|
|
1532
|
+
assertString(invocation.prompt_digest, 'INVALID_REWORK_LAUNCH_RESERVATION', 'invocation.prompt_digest');
|
|
1533
|
+
const attempt = this.#attempts().get(attempt_id);
|
|
1534
|
+
if (!attempt || attempt.state !== EXECUTABLE_ATTEMPT_STATE) {
|
|
1535
|
+
fail('ATTEMPT_NOT_ACTIVE', 'rework launch reservation requires an active attempt', { attempt_id });
|
|
1536
|
+
}
|
|
1537
|
+
if (attempt.claim.claim_id !== claim_id || attempt.claim.operator_id !== operator_id) {
|
|
1538
|
+
fail('CLAIM_OWNER_MISMATCH', 'rework launch reservation must belong to the active claim owner', { attempt_id });
|
|
1539
|
+
}
|
|
1540
|
+
if (attempt.claim.harness_process_id !== harness_process_id) {
|
|
1541
|
+
fail('RAIL_CLAIM_PROCESS_MISMATCH', 'rework launch reservation must belong to the active supervisor process', { attempt_id });
|
|
1542
|
+
}
|
|
1543
|
+
this.#assertLiveLease(attempt, timestamp(this.#clock));
|
|
1544
|
+
if (invocation.attempt_id !== attempt_id || invocation.action !== 'build'
|
|
1545
|
+
|| invocation.prompt_kind !== 'rework'
|
|
1546
|
+
|| invocation.source_decision_ref !== source_decision_ref
|
|
1547
|
+
|| invocation.rework_reservation_id !== reservation_id) {
|
|
1548
|
+
fail('EXECUTION_REWORK_RESERVATION_MISMATCH', 'rework launch intent must bind the canonical source decision and reservation');
|
|
1549
|
+
}
|
|
1550
|
+
const binding = attempt.execution_binding;
|
|
1551
|
+
if (!binding
|
|
1552
|
+
|| invocation.provider_id !== binding.provider_id
|
|
1553
|
+
|| invocation.harness_id !== binding.harness_id
|
|
1554
|
+
|| invocation.model_pin?.model_id !== binding.model_id
|
|
1555
|
+
|| invocation.model_pin?.reasoning_configuration !== binding.reasoning_configuration) {
|
|
1556
|
+
fail('EXECUTION_BINDING_MISMATCH', 'rework launch reservation does not match the sealed task binding');
|
|
1557
|
+
}
|
|
1558
|
+
const reservation = [...this.#records].reverse().find((record) => record.type === 'rework_reserved'
|
|
1559
|
+
&& record.payload?.attempt_id === attempt_id
|
|
1560
|
+
&& record.payload?.source_decision_ref === source_decision_ref);
|
|
1561
|
+
const released = this.#records.some((record) => record.type === 'rework_reservation_released'
|
|
1562
|
+
&& record.payload?.reservation_id === reservation_id);
|
|
1563
|
+
if (!reservation || released
|
|
1564
|
+
|| reservation.payload.reservation_id !== reservation_id
|
|
1565
|
+
|| reservation.payload.claim_event_id !== attempt.claim_event_id
|
|
1566
|
+
|| reservation.payload.claim_id !== claim_id
|
|
1567
|
+
|| reservation.payload.operator_id !== operator_id) {
|
|
1568
|
+
fail('RAIL_REWORK_RESERVATION_REQUIRED', 'rework launch intent requires the active claim-generation reservation');
|
|
1569
|
+
}
|
|
1570
|
+
const existing = this.#records.find((record) => record.type === 'rework_launch_reserved'
|
|
1571
|
+
&& record.payload?.invocation?.launch_nonce === invocation.launch_nonce);
|
|
1572
|
+
if (existing) {
|
|
1573
|
+
if (sha256(existing.payload.invocation) !== sha256(invocation)
|
|
1574
|
+
|| existing.payload.reservation_id !== reservation_id
|
|
1575
|
+
|| existing.payload.claim_event_id !== attempt.claim_event_id) {
|
|
1576
|
+
fail('EXECUTION_LAUNCH_NONCE_CONFLICT', 'launch nonce is already reserved for different rework material');
|
|
1577
|
+
}
|
|
1578
|
+
return Object.freeze(canonicalize({ ...existing.payload, event_id: existing.event_id, replay: true }));
|
|
1579
|
+
}
|
|
1580
|
+
const unresolvedLaunch = this.#records.find((record) => record.type === 'rework_launch_reserved'
|
|
1581
|
+
&& record.payload?.attempt_id === attempt_id
|
|
1582
|
+
&& record.payload?.reservation_id === reservation_id
|
|
1583
|
+
&& !this.#records.some((candidate) => candidate.type === 'execution_dispatched'
|
|
1584
|
+
&& candidate.payload?.invocation?.launch_nonce === record.payload.invocation.launch_nonce));
|
|
1585
|
+
if (unresolvedLaunch) {
|
|
1586
|
+
fail('RAIL_REWORK_LAUNCH_UNRESOLVED', 'a prior rework launch intent has no authenticated provider dispatch');
|
|
1587
|
+
}
|
|
1588
|
+
const unresolvedDispatch = this.#records.find((record) => record.type === 'execution_dispatched'
|
|
1589
|
+
&& record.payload?.attempt_id === attempt_id
|
|
1590
|
+
&& record.payload?.run_kind === 'rework'
|
|
1591
|
+
&& record.payload?.rework_reservation_id === reservation_id
|
|
1592
|
+
&& !this.#records.some((candidate) => candidate.type === 'execution_recorded'
|
|
1593
|
+
&& candidate.payload?.execution_evidence?.receipt_nonce === record.payload.invocation.launch_nonce));
|
|
1594
|
+
if (unresolvedDispatch) {
|
|
1595
|
+
fail('RAIL_REWORK_DISPATCH_UNRESOLVED', 'a prior rework provider launch has no canonical completion evidence');
|
|
1596
|
+
}
|
|
1597
|
+
const payload = canonicalize({
|
|
1598
|
+
attempt_id, claim_id, operator_id, harness_process_id,
|
|
1599
|
+
claim_event_id: attempt.claim_event_id, source_decision_ref,
|
|
1600
|
+
reservation_id, invocation, reserved_at: timestamp(this.#clock),
|
|
1601
|
+
});
|
|
1602
|
+
const record = this.#append('rework_launch_reserved', payload, reservation.event_id);
|
|
1603
|
+
return Object.freeze(canonicalize({ ...payload, event_id: record.event_id, replay: false }));
|
|
1604
|
+
});
|
|
1605
|
+
}
|
|
1606
|
+
|
|
1607
|
+
releaseReworkReservation({
|
|
1608
|
+
attempt_id, claim_id, operator_id, harness_process_id,
|
|
1609
|
+
source_decision_ref, reservation_id, reason = 'supervisor_exit',
|
|
1610
|
+
} = {}) {
|
|
1611
|
+
return this.#exclusive(() => {
|
|
1612
|
+
for (const [field, value] of Object.entries({
|
|
1613
|
+
attempt_id, claim_id, operator_id, harness_process_id,
|
|
1614
|
+
source_decision_ref, reservation_id, reason,
|
|
1615
|
+
})) assertString(value, 'INVALID_REWORK_RESERVATION_RELEASE', field);
|
|
1616
|
+
const attempt = this.#attempts().get(attempt_id);
|
|
1617
|
+
if (!attempt || attempt.state !== EXECUTABLE_ATTEMPT_STATE) {
|
|
1618
|
+
fail('ATTEMPT_NOT_ACTIVE', 'rework reservation release requires an active attempt', { attempt_id });
|
|
1619
|
+
}
|
|
1620
|
+
if (attempt.claim.claim_id !== claim_id || attempt.claim.operator_id !== operator_id) {
|
|
1621
|
+
fail('CLAIM_OWNER_MISMATCH', 'only the active claim owner may release a rework reservation', { attempt_id });
|
|
1622
|
+
}
|
|
1623
|
+
if (attempt.claim.harness_process_id !== harness_process_id) {
|
|
1624
|
+
fail('RAIL_CLAIM_PROCESS_MISMATCH', 'only the reserving supervisor process may release rework', { attempt_id });
|
|
1625
|
+
}
|
|
1626
|
+
const reservation = [...this.#records].reverse().find((record) => record.type === 'rework_reserved'
|
|
1627
|
+
&& record.payload?.attempt_id === attempt_id
|
|
1628
|
+
&& record.payload?.source_decision_ref === source_decision_ref);
|
|
1629
|
+
if (!reservation || reservation.payload.reservation_id !== reservation_id
|
|
1630
|
+
|| reservation.payload.claim_event_id !== attempt.claim_event_id) {
|
|
1631
|
+
fail('RAIL_REWORK_RESERVATION_MISMATCH', 'rework reservation release does not match the active claim generation');
|
|
1632
|
+
}
|
|
1633
|
+
const existing = this.#records.find((record) => record.type === 'rework_reservation_released'
|
|
1634
|
+
&& record.payload?.reservation_id === reservation_id);
|
|
1635
|
+
if (existing) return Object.freeze(canonicalize({ ...existing.payload, event_id: existing.event_id, replay: true }));
|
|
1636
|
+
const unresolvedLaunch = this.#records.find((record) => record.type === 'rework_launch_reserved'
|
|
1637
|
+
&& record.payload?.reservation_id === reservation_id
|
|
1638
|
+
&& !this.#records.some((candidate) => candidate.type === 'execution_dispatched'
|
|
1639
|
+
&& candidate.payload?.invocation?.launch_nonce === record.payload.invocation.launch_nonce));
|
|
1640
|
+
if (unresolvedLaunch) {
|
|
1641
|
+
fail('RAIL_REWORK_LAUNCH_UNRESOLVED', 'rework reservation cannot be released while a launch intent lacks authenticated dispatch');
|
|
1642
|
+
}
|
|
1643
|
+
const unresolvedDispatch = this.#records.find((record) => record.type === 'execution_dispatched'
|
|
1644
|
+
&& record.payload?.run_kind === 'rework'
|
|
1645
|
+
&& record.payload?.rework_reservation_id === reservation_id
|
|
1646
|
+
&& !this.#records.some((candidate) => candidate.type === 'execution_recorded'
|
|
1647
|
+
&& candidate.payload?.execution_evidence?.receipt_nonce === record.payload.invocation.launch_nonce));
|
|
1648
|
+
if (unresolvedDispatch) {
|
|
1649
|
+
fail('RAIL_REWORK_DISPATCH_UNRESOLVED', 'rework reservation cannot be released while a dispatch lacks canonical execution evidence');
|
|
1650
|
+
}
|
|
1651
|
+
const payload = canonicalize({
|
|
1652
|
+
attempt_id, claim_id, operator_id, harness_process_id,
|
|
1653
|
+
source_decision_ref, reservation_id, reason,
|
|
1654
|
+
});
|
|
1655
|
+
const record = this.#append('rework_reservation_released', payload);
|
|
1656
|
+
return Object.freeze(canonicalize({ ...payload, event_id: record.event_id, replay: false }));
|
|
1657
|
+
});
|
|
1658
|
+
}
|
|
1659
|
+
|
|
293
1660
|
prepareEffect({ attempt_id, claim_id, side_effect_class, idempotency_key, effect_ref } = {}) {
|
|
294
1661
|
return this.#exclusive(() => {
|
|
295
1662
|
for (const [field, value] of Object.entries({ attempt_id, claim_id, side_effect_class, idempotency_key, effect_ref })) assertString(value, 'INVALID_EFFECT', field);
|
|
@@ -324,39 +1691,436 @@ export class RailEngine {
|
|
|
324
1691
|
});
|
|
325
1692
|
}
|
|
326
1693
|
|
|
1694
|
+
recordReviewDispatch({ attempt_id, invocation, launch_receipt, installation, snapshot, project_dir, working_dir = project_dir } = {}) {
|
|
1695
|
+
return this.#exclusive(() => {
|
|
1696
|
+
assertString(attempt_id, 'INVALID_REVIEW_DISPATCH', 'attempt_id');
|
|
1697
|
+
assertObject(invocation, 'INVALID_REVIEW_DISPATCH', 'invocation');
|
|
1698
|
+
assertObject(launch_receipt, 'INVALID_REVIEW_DISPATCH', 'launch_receipt');
|
|
1699
|
+
assertString(invocation.invocation_id, 'INVALID_REVIEW_DISPATCH', 'invocation.invocation_id');
|
|
1700
|
+
assertString(invocation.launch_nonce, 'INVALID_REVIEW_DISPATCH', 'invocation.launch_nonce');
|
|
1701
|
+
assertString(invocation.prompt_digest, 'INVALID_REVIEW_DISPATCH', 'invocation.prompt_digest');
|
|
1702
|
+
assertString(project_dir, 'INVALID_REVIEW_DISPATCH', 'project_dir');
|
|
1703
|
+
assertString(working_dir, 'INVALID_REVIEW_DISPATCH', 'working_dir');
|
|
1704
|
+
const attempt = this.#attempts().get(attempt_id);
|
|
1705
|
+
if (!attempt || attempt.state !== EXECUTABLE_ATTEMPT_STATE) fail('ATTEMPT_NOT_ACTIVE', 'review dispatch requires an active attempt');
|
|
1706
|
+
this.#assertLiveLease(attempt, timestamp(this.#clock));
|
|
1707
|
+
if (unresolvedAdjudication(this.#records, attempt_id)) {
|
|
1708
|
+
fail('RAIL_ADJUDICATION_REQUIRED', 'a canonical AI adjudication decision is required before another review can be dispatched');
|
|
1709
|
+
}
|
|
1710
|
+
if (invocation.attempt_id !== attempt_id || invocation.action !== 'review') fail('REVIEW_INVOCATION_MISMATCH', 'review launch is not bound to this attempt');
|
|
1711
|
+
const selected = selectCanonicalReviewer({
|
|
1712
|
+
installation, snapshot,
|
|
1713
|
+
builder_identity: {
|
|
1714
|
+
provider_id: attempt.execution_binding.provider_id,
|
|
1715
|
+
model_id: attempt.execution_binding.model_id,
|
|
1716
|
+
},
|
|
1717
|
+
clock: this.#clock,
|
|
1718
|
+
});
|
|
1719
|
+
if (invocation.provider_id !== selected.provider_id
|
|
1720
|
+
|| invocation.harness_id !== selected.harness_id
|
|
1721
|
+
|| invocation.model_pin?.model_id !== selected.model_pin.model_id
|
|
1722
|
+
|| invocation.model_pin?.catalog_snapshot_ref !== selected.model_pin.catalog_snapshot_ref
|
|
1723
|
+
|| invocation.model_pin?.reasoning_configuration !== selected.model_pin.reasoning_configuration) {
|
|
1724
|
+
fail('NONCANONICAL_REVIEWER', 'review launch must use the strongest canonical independent reviewer');
|
|
1725
|
+
}
|
|
1726
|
+
this.#verifyAuthority('review_launch', launch_receipt, {
|
|
1727
|
+
purpose: 'terminal_launch', authority_kind: 'rail_review',
|
|
1728
|
+
invocation_id: invocation.invocation_id, launch_nonce: invocation.launch_nonce,
|
|
1729
|
+
provider_id: invocation.provider_id, harness_id: invocation.harness_id,
|
|
1730
|
+
model_id: invocation.model_pin?.model_id,
|
|
1731
|
+
reasoning_configuration: invocation.model_pin?.reasoning_configuration,
|
|
1732
|
+
task_id: attempt.task_id,
|
|
1733
|
+
working_dir: resolve(working_dir),
|
|
1734
|
+
prompt_digest: invocation.prompt_digest,
|
|
1735
|
+
});
|
|
1736
|
+
if (launch_receipt.task_id !== attempt.task_id
|
|
1737
|
+
|| launch_receipt.working_dir !== resolve(working_dir)
|
|
1738
|
+
|| launch_receipt.prompt_digest !== invocation.prompt_digest
|
|
1739
|
+
|| !/^[a-f0-9]{64}$/.test(launch_receipt.command_digest || '')
|
|
1740
|
+
|| launch_receipt.process_containment_kind !== 'bubblewrap_pid_namespace_v1'
|
|
1741
|
+
|| typeof launch_receipt.process_containment_id !== 'string' || !launch_receipt.process_containment_id
|
|
1742
|
+
|| !/^[a-f0-9]{64}$/.test(launch_receipt.process_containment_digest || '')) {
|
|
1743
|
+
fail('REVIEW_LAUNCH_RECEIPT_MISMATCH', 'review launch receipt must bind the canonical prompt, task, working directory and command');
|
|
1744
|
+
}
|
|
1745
|
+
const prior = this.#records.find((record) => ['execution_dispatched', 'review_dispatched', 'adjudication_dispatched'].includes(record.type)
|
|
1746
|
+
&& record.payload?.invocation?.launch_nonce === invocation.launch_nonce);
|
|
1747
|
+
if (prior) fail('RAIL_LAUNCH_NONCE_CONFLICT', 'launch nonce is already registered');
|
|
1748
|
+
const record = this.#append('review_dispatched', { attempt_id, invocation, launch_receipt });
|
|
1749
|
+
return Object.freeze(canonicalize({ ...record.payload, event_id: record.event_id }));
|
|
1750
|
+
});
|
|
1751
|
+
}
|
|
1752
|
+
|
|
1753
|
+
recordAdjudicationDispatch({ attempt_id, review_ref, invocation, launch_receipt, project_dir, working_dir = project_dir } = {}) {
|
|
1754
|
+
return this.#exclusive(() => {
|
|
1755
|
+
for (const [field, value] of Object.entries({ attempt_id, review_ref })) assertString(value, 'INVALID_ADJUDICATION_DISPATCH', field);
|
|
1756
|
+
assertObject(invocation, 'INVALID_ADJUDICATION_DISPATCH', 'invocation');
|
|
1757
|
+
assertObject(launch_receipt, 'INVALID_ADJUDICATION_DISPATCH', 'launch_receipt');
|
|
1758
|
+
assertString(invocation.invocation_id, 'INVALID_ADJUDICATION_DISPATCH', 'invocation.invocation_id');
|
|
1759
|
+
assertString(invocation.launch_nonce, 'INVALID_ADJUDICATION_DISPATCH', 'invocation.launch_nonce');
|
|
1760
|
+
assertString(invocation.prompt_digest, 'INVALID_ADJUDICATION_DISPATCH', 'invocation.prompt_digest');
|
|
1761
|
+
assertString(project_dir, 'INVALID_ADJUDICATION_DISPATCH', 'project_dir');
|
|
1762
|
+
assertString(working_dir, 'INVALID_ADJUDICATION_DISPATCH', 'working_dir');
|
|
1763
|
+
const attempt = this.#attempts().get(attempt_id);
|
|
1764
|
+
if (!attempt || attempt.state !== EXECUTABLE_ATTEMPT_STATE) fail('ATTEMPT_NOT_ACTIVE', 'adjudication dispatch requires an active attempt');
|
|
1765
|
+
this.#assertLiveLease(attempt, timestamp(this.#clock));
|
|
1766
|
+
const reviewRecord = [...this.#records].reverse().find((record) => record.type === 'review_decided' && record.payload?.attempt_id === attempt_id);
|
|
1767
|
+
if (!reviewRecord || reviewRecord.payload.review_ref !== review_ref || reviewRecord.payload.decision?.decision !== 'AI_ADJUDICATION') {
|
|
1768
|
+
fail('ADJUDICATION_NOT_AUTHORIZED', 'review does not authorize adjudication dispatch');
|
|
1769
|
+
}
|
|
1770
|
+
const selected = reviewRecord.payload.decision.adjudicators.some((candidate) =>
|
|
1771
|
+
candidate.provider_id === invocation.provider_id
|
|
1772
|
+
&& candidate.harness_id === invocation.harness_id
|
|
1773
|
+
&& sha256(candidate.model_pin) === sha256(invocation.model_pin));
|
|
1774
|
+
if (!selected || invocation.attempt_id !== attempt_id || invocation.action !== 'adjudication') {
|
|
1775
|
+
fail('UNSELECTED_ADJUDICATOR', 'adjudication launch is not an exact selected invocation');
|
|
1776
|
+
}
|
|
1777
|
+
const roundState = adjudicationRoundState(
|
|
1778
|
+
this.#records, attempt_id, review_ref, reviewRecord.payload.decision.adjudicators,
|
|
1779
|
+
);
|
|
1780
|
+
if (roundState.complete) fail('ADJUDICATION_ROUND_COMPLETE', 'the canonical adjudication round is already complete');
|
|
1781
|
+
const alreadyObserved = roundState.current.some((entry) =>
|
|
1782
|
+
entry.invocation.provider_id === invocation.provider_id
|
|
1783
|
+
&& entry.invocation.harness_id === invocation.harness_id);
|
|
1784
|
+
if (alreadyObserved) fail('ADJUDICATOR_ALREADY_OBSERVED', 'this adjudicator already has canonical evidence in the current round');
|
|
1785
|
+
if (invocation.adjudication_round !== roundState.round
|
|
1786
|
+
|| invocation.prior_decisions_digest !== sha256(roundState.prior_decisions)) {
|
|
1787
|
+
fail('NONCANONICAL_ADJUDICATION_ROUND', 'adjudication launch is not bound to the canonical round and prior decisions');
|
|
1788
|
+
}
|
|
1789
|
+
this.#verifyAuthority('adjudication_launch', launch_receipt, {
|
|
1790
|
+
purpose: 'terminal_launch', authority_kind: 'rail_adjudication',
|
|
1791
|
+
invocation_id: invocation.invocation_id, launch_nonce: invocation.launch_nonce,
|
|
1792
|
+
provider_id: invocation.provider_id, harness_id: invocation.harness_id,
|
|
1793
|
+
model_id: invocation.model_pin?.model_id,
|
|
1794
|
+
reasoning_configuration: invocation.model_pin?.reasoning_configuration,
|
|
1795
|
+
task_id: attempt.task_id,
|
|
1796
|
+
working_dir: resolve(working_dir),
|
|
1797
|
+
prompt_digest: invocation.prompt_digest,
|
|
1798
|
+
});
|
|
1799
|
+
if (launch_receipt.task_id !== attempt.task_id
|
|
1800
|
+
|| launch_receipt.working_dir !== resolve(working_dir)
|
|
1801
|
+
|| launch_receipt.prompt_digest !== invocation.prompt_digest
|
|
1802
|
+
|| !/^[a-f0-9]{64}$/.test(launch_receipt.command_digest || '')
|
|
1803
|
+
|| launch_receipt.process_containment_kind !== 'bubblewrap_pid_namespace_v1'
|
|
1804
|
+
|| typeof launch_receipt.process_containment_id !== 'string' || !launch_receipt.process_containment_id
|
|
1805
|
+
|| !/^[a-f0-9]{64}$/.test(launch_receipt.process_containment_digest || '')) {
|
|
1806
|
+
fail('ADJUDICATION_LAUNCH_RECEIPT_MISMATCH', 'adjudication launch receipt must bind the canonical prompt, task, working directory and command');
|
|
1807
|
+
}
|
|
1808
|
+
const prior = this.#records.find((record) => ['execution_dispatched', 'review_dispatched', 'adjudication_dispatched'].includes(record.type)
|
|
1809
|
+
&& record.payload?.invocation?.launch_nonce === invocation.launch_nonce);
|
|
1810
|
+
if (prior) fail('RAIL_LAUNCH_NONCE_CONFLICT', 'launch nonce is already registered');
|
|
1811
|
+
const record = this.#append('adjudication_dispatched', { attempt_id, review_ref, invocation, launch_receipt });
|
|
1812
|
+
return Object.freeze(canonicalize({ ...record.payload, event_id: record.event_id }));
|
|
1813
|
+
});
|
|
1814
|
+
}
|
|
1815
|
+
|
|
1816
|
+
/** Journals each terminal-authorized adjudicator result before another round may start. */
|
|
1817
|
+
recordAdjudicationObservation({
|
|
1818
|
+
attempt_id, review_ref, invocation, decision, adjudication_output, terminal_authority_receipt,
|
|
1819
|
+
} = {}) {
|
|
1820
|
+
return this.#exclusive(() => {
|
|
1821
|
+
for (const [field, value] of Object.entries({ attempt_id, review_ref })) assertString(value, 'INVALID_ADJUDICATION_OBSERVATION', field);
|
|
1822
|
+
assertObject(invocation, 'INVALID_ADJUDICATION_OBSERVATION', 'invocation');
|
|
1823
|
+
assertObject(terminal_authority_receipt, 'INVALID_ADJUDICATION_OBSERVATION', 'terminal_authority_receipt');
|
|
1824
|
+
const attempt = this.#attempts().get(attempt_id);
|
|
1825
|
+
if (!attempt || attempt.state !== EXECUTABLE_ATTEMPT_STATE) fail('ATTEMPT_NOT_ACTIVE', 'adjudication observation requires an active attempt');
|
|
1826
|
+
this.#assertLiveLease(attempt, timestamp(this.#clock));
|
|
1827
|
+
const reviewRecord = [...this.#records].reverse().find((record) => record.type === 'review_decided'
|
|
1828
|
+
&& record.payload?.attempt_id === attempt_id);
|
|
1829
|
+
if (!reviewRecord || reviewRecord.payload.review_ref !== review_ref
|
|
1830
|
+
|| reviewRecord.payload.decision?.decision !== 'AI_ADJUDICATION') {
|
|
1831
|
+
fail('ADJUDICATION_NOT_AUTHORIZED', 'the latest review decision does not authorize adjudication evidence');
|
|
1832
|
+
}
|
|
1833
|
+
const expected = reviewRecord.payload.decision.adjudicators;
|
|
1834
|
+
const selected = expected.find((candidate) => candidate.provider_id === invocation.provider_id
|
|
1835
|
+
&& candidate.harness_id === invocation.harness_id
|
|
1836
|
+
&& sha256(candidate.model_pin) === sha256(invocation.model_pin));
|
|
1837
|
+
if (!selected || invocation.attempt_id !== attempt_id || invocation.action !== 'adjudication') {
|
|
1838
|
+
fail('UNSELECTED_ADJUDICATOR', 'adjudication observation is not from an exact selected invocation');
|
|
1839
|
+
}
|
|
1840
|
+
const roundState = adjudicationRoundState(this.#records, attempt_id, review_ref, expected);
|
|
1841
|
+
if (roundState.complete || invocation.adjudication_round !== roundState.round
|
|
1842
|
+
|| invocation.prior_decisions_digest !== sha256(roundState.prior_decisions)) {
|
|
1843
|
+
fail('NONCANONICAL_ADJUDICATION_ROUND', 'adjudication observation is not bound to the canonical round and prior decisions');
|
|
1844
|
+
}
|
|
1845
|
+
if (roundState.current.some((entry) => entry.invocation.provider_id === invocation.provider_id
|
|
1846
|
+
&& entry.invocation.harness_id === invocation.harness_id)) {
|
|
1847
|
+
fail('ADJUDICATOR_ALREADY_OBSERVED', 'this adjudicator already has canonical evidence in the current round');
|
|
1848
|
+
}
|
|
1849
|
+
const dispatch = this.#records.find((record) => record.type === 'adjudication_dispatched'
|
|
1850
|
+
&& record.payload?.attempt_id === attempt_id
|
|
1851
|
+
&& record.payload?.review_ref === review_ref
|
|
1852
|
+
&& record.payload?.invocation?.launch_nonce === invocation.launch_nonce);
|
|
1853
|
+
if (!dispatch || sha256(dispatch.payload.invocation) !== sha256(invocation)) {
|
|
1854
|
+
fail('ADJUDICATION_DISPATCH_REQUIRED', 'adjudication observation requires its exact canonical launch');
|
|
1855
|
+
}
|
|
1856
|
+
this.#verifyAuthority('adjudication_completion', terminal_authority_receipt, {
|
|
1857
|
+
purpose: 'terminal_completion', authority_kind: 'rail_adjudication',
|
|
1858
|
+
invocation_id: invocation.invocation_id, launch_nonce: invocation.launch_nonce,
|
|
1859
|
+
provider_id: invocation.provider_id, harness_id: invocation.harness_id,
|
|
1860
|
+
model_id: invocation.model_pin?.model_id,
|
|
1861
|
+
reasoning_configuration: invocation.model_pin?.reasoning_configuration,
|
|
1862
|
+
exit_status: 0,
|
|
1863
|
+
});
|
|
1864
|
+
if (terminal_authority_receipt.launch_receipt_digest !== sha256(dispatch.payload.launch_receipt)
|
|
1865
|
+
|| terminal_authority_receipt.terminal_id !== dispatch.payload.launch_receipt.terminal_id
|
|
1866
|
+
|| terminal_authority_receipt.stdout_digest !== sha256(adjudication_output)) {
|
|
1867
|
+
fail('ADJUDICATION_TERMINAL_AUTHORITY_MISMATCH', 'adjudication observation does not bind its canonical launch and full stdout');
|
|
1868
|
+
}
|
|
1869
|
+
const parsedDecision = parseAdjudicationOutput(adjudication_output);
|
|
1870
|
+
if (sha256(parsedDecision) !== sha256(validateAdjudicationDecision(decision))) {
|
|
1871
|
+
fail('ADJUDICATION_PAYLOAD_MISMATCH', 'adjudication observation differs from terminal-authorized stdout');
|
|
1872
|
+
}
|
|
1873
|
+
const observation = canonicalize({
|
|
1874
|
+
round: roundState.round,
|
|
1875
|
+
invocation,
|
|
1876
|
+
decision: parsedDecision,
|
|
1877
|
+
adjudication_output,
|
|
1878
|
+
terminal_authority_receipt,
|
|
1879
|
+
});
|
|
1880
|
+
const existing = this.#records.find((record) => record.type === 'adjudication_observed'
|
|
1881
|
+
&& record.payload?.observation?.invocation?.launch_nonce === invocation.launch_nonce);
|
|
1882
|
+
if (existing) {
|
|
1883
|
+
if (sha256(existing.payload.observation) !== sha256(observation)) {
|
|
1884
|
+
fail('ADJUDICATION_OBSERVATION_CONFLICT', 'launch nonce already has different adjudication evidence');
|
|
1885
|
+
}
|
|
1886
|
+
return Object.freeze(canonicalize({ ...observation, event_id: existing.event_id, replay: true }));
|
|
1887
|
+
}
|
|
1888
|
+
const record = this.#append('adjudication_observed', { attempt_id, review_ref, observation });
|
|
1889
|
+
return Object.freeze(canonicalize({ ...observation, event_id: record.event_id, replay: false }));
|
|
1890
|
+
});
|
|
1891
|
+
}
|
|
1892
|
+
|
|
327
1893
|
/**
|
|
328
1894
|
* Persists the Review Council transition that controls the next RAIL step.
|
|
329
|
-
*
|
|
330
|
-
* reference
|
|
1895
|
+
* Full review and invocation evidence is journaled with the immutable
|
|
1896
|
+
* reference so review rounds cannot be reset through mutable external files.
|
|
331
1897
|
*/
|
|
332
|
-
recordReviewDecision({ attempt_id, review_ref, decision, review, invocation,
|
|
1898
|
+
recordReviewDecision({ attempt_id, review_ref, decision, review, invocation, terminal_authority_receipt, installation, snapshot, git_dir, result_commit_ref } = {}) {
|
|
333
1899
|
return this.#exclusive(() => {
|
|
334
1900
|
assertString(attempt_id, 'INVALID_REVIEW_DECISION', 'attempt_id');
|
|
335
1901
|
assertString(review_ref, 'INVALID_REVIEW_DECISION', 'review_ref');
|
|
336
1902
|
assertObject(decision, 'INVALID_REVIEW_DECISION', 'decision');
|
|
337
1903
|
if (!REVIEW_DECISIONS.has(decision.decision)) fail('INVALID_REVIEW_DECISION', 'decision is not a Review Council outcome');
|
|
1904
|
+
assertObject(terminal_authority_receipt, 'INVALID_REVIEW_DECISION', 'terminal_authority_receipt');
|
|
1905
|
+
this.#verifyAuthority('review_completion', terminal_authority_receipt, {
|
|
1906
|
+
purpose: 'terminal_completion', authority_kind: 'rail_review',
|
|
1907
|
+
invocation_id: invocation?.invocation_id, launch_nonce: invocation?.launch_nonce,
|
|
1908
|
+
provider_id: invocation?.provider_id, harness_id: invocation?.harness_id,
|
|
1909
|
+
model_id: invocation?.model_pin?.model_id,
|
|
1910
|
+
reasoning_configuration: invocation?.model_pin?.reasoning_configuration,
|
|
1911
|
+
exit_status: 0,
|
|
1912
|
+
});
|
|
1913
|
+
if (terminal_authority_receipt.purpose !== 'terminal_completion'
|
|
1914
|
+
|| terminal_authority_receipt.authority_kind !== 'rail_review'
|
|
1915
|
+
|| terminal_authority_receipt.invocation_id !== invocation?.invocation_id
|
|
1916
|
+
|| terminal_authority_receipt.launch_nonce !== invocation?.launch_nonce
|
|
1917
|
+
|| terminal_authority_receipt.exit_status !== 0
|
|
1918
|
+
|| typeof terminal_authority_receipt.authority_signature !== 'string') {
|
|
1919
|
+
fail('REVIEW_TERMINAL_AUTHORITY_MISMATCH', 'review decision requires a successful authorized reviewer completion');
|
|
1920
|
+
}
|
|
338
1921
|
const attempt = this.#attempts().get(attempt_id);
|
|
339
1922
|
if (!attempt || attempt.state !== EXECUTABLE_ATTEMPT_STATE) fail('ATTEMPT_NOT_ACTIVE', 'review decision requires an active attempt', { attempt_id });
|
|
340
1923
|
this.#assertLiveLease(attempt, timestamp(this.#clock));
|
|
1924
|
+
const dispatch = this.#records.find((record) => record.type === 'review_dispatched'
|
|
1925
|
+
&& record.payload?.attempt_id === attempt_id
|
|
1926
|
+
&& record.payload?.invocation?.launch_nonce === invocation?.launch_nonce);
|
|
1927
|
+
if (!dispatch || sha256(dispatch.payload.invocation) !== sha256(invocation)) {
|
|
1928
|
+
fail('REVIEW_DISPATCH_REQUIRED', 'review completion requires an exact canonical launch');
|
|
1929
|
+
}
|
|
1930
|
+
if (terminal_authority_receipt.launch_receipt_digest !== sha256(dispatch.payload.launch_receipt)
|
|
1931
|
+
|| terminal_authority_receipt.terminal_id !== dispatch.payload.launch_receipt.terminal_id) {
|
|
1932
|
+
fail('REVIEW_TERMINAL_AUTHORITY_MISMATCH', 'review completion does not bind the canonical launch');
|
|
1933
|
+
}
|
|
1934
|
+
assertString(git_dir, 'INVALID_REVIEW_DECISION', 'git_dir');
|
|
1935
|
+
assertString(result_commit_ref, 'INVALID_REVIEW_DECISION', 'result_commit_ref');
|
|
341
1936
|
const validated = acceptReview({ review, invocation, installation, snapshot, clock: this.#clock });
|
|
342
1937
|
if (validated.attempt_id !== attempt_id) fail('REVIEW_ATTEMPT_MISMATCH', 'review is not bound to this RAIL attempt');
|
|
343
1938
|
const expectedRef = `rail-review://${attempt.handoff_id}/${attempt_id}/${validated.review_id}`;
|
|
344
1939
|
if (review_ref !== expectedRef) fail('REVIEW_REFERENCE_MISMATCH', 'review reference is not bound to the claimed handoff and attempt');
|
|
345
|
-
const expected = decideReviewProgress({ prior_review_evidence, current_review: validated, reviewer_invocation: invocation, installation, snapshot, clock: this.#clock });
|
|
346
|
-
if (sha256(expected) !== sha256(decision)) fail('REVIEW_DECISION_MISMATCH', 'journal decision does not match validated Review Council evidence');
|
|
347
1940
|
const existing = this.#records.find((record) => record.type === 'review_decided' && record.payload?.review_ref === review_ref);
|
|
348
1941
|
if (existing) {
|
|
349
|
-
if (existing.payload.attempt_id !== attempt_id
|
|
1942
|
+
if (existing.payload.attempt_id !== attempt_id
|
|
1943
|
+
|| existing.payload.result_commit_ref !== result_commit_ref
|
|
1944
|
+
|| sha256(existing.payload.decision) !== sha256(decision)
|
|
1945
|
+
|| sha256(existing.payload.review) !== sha256(validated)
|
|
1946
|
+
|| sha256(existing.payload.invocation) !== sha256(invocation)
|
|
1947
|
+
|| sha256(existing.payload.terminal_authority_receipt) !== sha256(terminal_authority_receipt)) {
|
|
350
1948
|
fail('REVIEW_DECISION_CONFLICT', 'review reference is already bound to different decision material', { review_ref });
|
|
351
1949
|
}
|
|
352
1950
|
return Object.freeze(canonicalize({ ...existing.payload, event_id: existing.event_id, replay: true }));
|
|
353
1951
|
}
|
|
354
|
-
|
|
1952
|
+
if (unresolvedAdjudication(this.#records, attempt_id)) {
|
|
1953
|
+
fail('RAIL_ADJUDICATION_REQUIRED', 'a canonical AI adjudication decision is required before another review can be accepted');
|
|
1954
|
+
}
|
|
1955
|
+
if (this.#records.some((record) => record.type === 'review_decided'
|
|
1956
|
+
&& record.payload?.terminal_authority_receipt?.launch_nonce === invocation.launch_nonce)) {
|
|
1957
|
+
fail('REVIEW_LAUNCH_ALREADY_RECEIPTED', 'this review launch already has a canonical decision');
|
|
1958
|
+
}
|
|
1959
|
+
const executionEvidence = this.#executionEvidence(attempt_id, { successfulOnly: true }).at(-1);
|
|
1960
|
+
if (!executionEvidence || executionEvidence.result_commit_ref !== result_commit_ref) {
|
|
1961
|
+
fail('REVIEW_EXECUTION_EVIDENCE_MISMATCH', 'review must bind the latest successful execution receipted in the canonical journal');
|
|
1962
|
+
}
|
|
1963
|
+
const latestControl = this.#records.filter((record) => ['review_decided', 'adjudication_decided'].includes(record.type)
|
|
1964
|
+
&& record.payload?.attempt_id === attempt_id).at(-1);
|
|
1965
|
+
const requiredReworkSource = latestControl?.type === 'review_decided'
|
|
1966
|
+
&& latestControl.payload.decision?.decision === 'normal-rework'
|
|
1967
|
+
? latestControl.payload.review_ref
|
|
1968
|
+
: latestControl?.type === 'adjudication_decided'
|
|
1969
|
+
&& latestControl.payload.decision?.outcome === 'bounded-correction'
|
|
1970
|
+
? latestControl.payload.adjudication_ref
|
|
1971
|
+
: null;
|
|
1972
|
+
if (requiredReworkSource && executionEvidence.source_decision_ref !== requiredReworkSource) {
|
|
1973
|
+
fail('RAIL_REWORK_RESULT_REQUIRED', 'a successful canonical rework for the latest decision is required before another review');
|
|
1974
|
+
}
|
|
1975
|
+
verifyCommitDescendsFrom({ git_dir, base_commit_ref: attempt.git_commit_ref, result_commit_ref });
|
|
1976
|
+
const priorReviewEvidence = this.#reviewEvidence(attempt_id);
|
|
1977
|
+
const expected = decideReviewProgress({
|
|
1978
|
+
prior_review_evidence: priorReviewEvidence,
|
|
1979
|
+
current_review: validated,
|
|
1980
|
+
reviewer_invocation: invocation,
|
|
1981
|
+
installation,
|
|
1982
|
+
snapshot,
|
|
1983
|
+
clock: this.#clock,
|
|
1984
|
+
});
|
|
1985
|
+
if (sha256(expected) !== sha256(decision)) fail('REVIEW_DECISION_MISMATCH', 'journal decision does not match canonical Review Council history');
|
|
1986
|
+
const record = this.#append('review_decided', {
|
|
1987
|
+
attempt_id, task_id: attempt.task_id, review_ref,
|
|
1988
|
+
base_commit_ref: attempt.git_commit_ref, result_commit_ref,
|
|
1989
|
+
review: canonicalize(validated),
|
|
1990
|
+
invocation: canonicalize(invocation),
|
|
1991
|
+
terminal_authority_receipt: canonicalize(terminal_authority_receipt),
|
|
1992
|
+
review_digest: sha256({ review: validated, invocation }),
|
|
1993
|
+
decision: canonicalize(decision),
|
|
1994
|
+
});
|
|
355
1995
|
return Object.freeze(canonicalize({ attempt_id, task_id: attempt.task_id, review_ref, decision, event_id: record.event_id, replay: false }));
|
|
356
1996
|
});
|
|
357
1997
|
}
|
|
358
1998
|
|
|
359
|
-
|
|
1999
|
+
/** Persists the bounded consensus reached by the selected adjudication council. */
|
|
2000
|
+
recordAdjudicationDecision({ attempt_id, review_ref, evidence } = {}) {
|
|
2001
|
+
return this.#exclusive(() => {
|
|
2002
|
+
for (const [field, value] of Object.entries({ attempt_id, review_ref })) assertString(value, 'INVALID_ADJUDICATION_RECORD', field);
|
|
2003
|
+
if (!Array.isArray(evidence) || evidence.length === 0) fail('INVALID_ADJUDICATION_RECORD', 'adjudication evidence must be a non-empty array');
|
|
2004
|
+
const attempt = this.#attempts().get(attempt_id);
|
|
2005
|
+
if (!attempt || attempt.state !== EXECUTABLE_ATTEMPT_STATE) fail('ATTEMPT_NOT_ACTIVE', 'adjudication requires an active attempt', { attempt_id });
|
|
2006
|
+
this.#assertLiveLease(attempt, timestamp(this.#clock));
|
|
2007
|
+
const reviewRecord = [...this.#records].reverse().find((record) => record.type === 'review_decided'
|
|
2008
|
+
&& record.payload?.attempt_id === attempt_id);
|
|
2009
|
+
if (!reviewRecord || reviewRecord.payload.review_ref !== review_ref
|
|
2010
|
+
|| reviewRecord.payload.decision?.decision !== 'AI_ADJUDICATION') {
|
|
2011
|
+
fail('ADJUDICATION_NOT_AUTHORIZED', 'the latest review decision does not authorize AI adjudication');
|
|
2012
|
+
}
|
|
2013
|
+
const expected = reviewRecord.payload.decision.adjudicators;
|
|
2014
|
+
if (evidence.length !== expected.length) fail('ADJUDICATION_COUNCIL_INCOMPLETE', 'every selected adjudicator must provide evidence');
|
|
2015
|
+
const seen = new Set();
|
|
2016
|
+
const normalizedEvidence = evidence.map((entry) => {
|
|
2017
|
+
assertObject(entry, 'INVALID_ADJUDICATION_RECORD', 'adjudication evidence entry');
|
|
2018
|
+
assertObject(entry.invocation, 'INVALID_ADJUDICATION_RECORD', 'adjudication evidence invocation');
|
|
2019
|
+
assertObject(entry.terminal_authority_receipt, 'INVALID_ADJUDICATION_RECORD', 'terminal_authority_receipt');
|
|
2020
|
+
assertString(entry.invocation.invocation_id, 'INVALID_ADJUDICATION_RECORD', 'invocation.invocation_id');
|
|
2021
|
+
assertString(entry.invocation.launch_nonce, 'INVALID_ADJUDICATION_RECORD', 'invocation.launch_nonce');
|
|
2022
|
+
const key = `${entry.invocation.provider_id}/${entry.invocation.harness_id}`;
|
|
2023
|
+
if (seen.has(key)) fail('DUPLICATE_ADJUDICATOR', 'an adjudicator may appear only once');
|
|
2024
|
+
seen.add(key);
|
|
2025
|
+
const selected = expected.find((candidate) => candidate.provider_id === entry.invocation.provider_id
|
|
2026
|
+
&& candidate.harness_id === entry.invocation.harness_id
|
|
2027
|
+
&& sha256(candidate.model_pin) === sha256(entry.invocation.model_pin));
|
|
2028
|
+
if (!selected || entry.invocation.action !== 'adjudication') fail('UNSELECTED_ADJUDICATOR', 'adjudication evidence contains an unselected invocation');
|
|
2029
|
+
if (entry.invocation.attempt_id !== attempt_id) fail('ADJUDICATION_ATTEMPT_MISMATCH', 'adjudication invocation is not bound to this attempt');
|
|
2030
|
+
const dispatch = this.#records.find((record) => record.type === 'adjudication_dispatched'
|
|
2031
|
+
&& record.payload?.attempt_id === attempt_id
|
|
2032
|
+
&& record.payload?.review_ref === review_ref
|
|
2033
|
+
&& record.payload?.invocation?.launch_nonce === entry.invocation.launch_nonce);
|
|
2034
|
+
if (!dispatch || sha256(dispatch.payload.invocation) !== sha256(entry.invocation)) {
|
|
2035
|
+
fail('ADJUDICATION_DISPATCH_REQUIRED', 'adjudication completion requires an exact canonical launch');
|
|
2036
|
+
}
|
|
2037
|
+
this.#verifyAuthority('adjudication_completion', entry.terminal_authority_receipt, {
|
|
2038
|
+
purpose: 'terminal_completion', authority_kind: 'rail_adjudication',
|
|
2039
|
+
invocation_id: entry.invocation.invocation_id, launch_nonce: entry.invocation.launch_nonce,
|
|
2040
|
+
provider_id: entry.invocation.provider_id, harness_id: entry.invocation.harness_id,
|
|
2041
|
+
model_id: entry.invocation.model_pin?.model_id,
|
|
2042
|
+
reasoning_configuration: entry.invocation.model_pin?.reasoning_configuration,
|
|
2043
|
+
exit_status: 0,
|
|
2044
|
+
});
|
|
2045
|
+
if (entry.terminal_authority_receipt.launch_receipt_digest !== sha256(dispatch.payload.launch_receipt)
|
|
2046
|
+
|| entry.terminal_authority_receipt.terminal_id !== dispatch.payload.launch_receipt.terminal_id
|
|
2047
|
+
|| entry.terminal_authority_receipt.stdout_digest !== sha256(entry.adjudication_output)) {
|
|
2048
|
+
fail('ADJUDICATION_TERMINAL_AUTHORITY_MISMATCH', 'adjudication completion does not bind its canonical launch and full stdout');
|
|
2049
|
+
}
|
|
2050
|
+
const parsedDecision = parseAdjudicationOutput(entry.adjudication_output);
|
|
2051
|
+
if (sha256(parsedDecision) !== sha256(validateAdjudicationDecision(entry.decision))) {
|
|
2052
|
+
fail('ADJUDICATION_PAYLOAD_MISMATCH', 'adjudication decision differs from terminal-authorized stdout');
|
|
2053
|
+
}
|
|
2054
|
+
return canonicalize({
|
|
2055
|
+
invocation: entry.invocation,
|
|
2056
|
+
decision: parsedDecision,
|
|
2057
|
+
terminal_authority_receipt: entry.terminal_authority_receipt,
|
|
2058
|
+
});
|
|
2059
|
+
});
|
|
2060
|
+
const roundState = adjudicationRoundState(this.#records, attempt_id, review_ref, expected);
|
|
2061
|
+
const canonicalEvidence = roundState.current.map((entry) => canonicalize({
|
|
2062
|
+
invocation: entry.invocation,
|
|
2063
|
+
decision: entry.decision,
|
|
2064
|
+
terminal_authority_receipt: entry.terminal_authority_receipt,
|
|
2065
|
+
}));
|
|
2066
|
+
if (!roundState.complete || sha256(normalizedEvidence) !== sha256(canonicalEvidence)) {
|
|
2067
|
+
fail('ADJUDICATION_OBSERVATIONS_REQUIRED', 'final adjudication must equal the complete canonical observed round');
|
|
2068
|
+
}
|
|
2069
|
+
const outcome = normalizedEvidence[0].decision.outcome;
|
|
2070
|
+
if (!adjudicationEntriesAgree(normalizedEvidence)) {
|
|
2071
|
+
fail('ADJUDICATION_CONSENSUS_MISMATCH', 'the final decision is not unanimous across selected adjudicators');
|
|
2072
|
+
}
|
|
2073
|
+
const decision = deriveAdjudicationConsensus({ handoff_id: attempt.handoff_id, attempt_id, evidence: normalizedEvidence });
|
|
2074
|
+
const adjudication_ref = `rail-adjudication://${attempt.handoff_id}/${attempt_id}/adjudication-${sha256({ review_ref, evidence: normalizedEvidence, decision }).slice(0, 24)}`;
|
|
2075
|
+
const existing = this.#records.find((record) => record.type === 'adjudication_decided' && record.payload?.attempt_id === attempt_id);
|
|
2076
|
+
if (existing) {
|
|
2077
|
+
if (existing.payload.review_ref !== review_ref || existing.payload.adjudication_ref !== adjudication_ref
|
|
2078
|
+
|| sha256(existing.payload.decision) !== sha256(decision)
|
|
2079
|
+
|| sha256(existing.payload.evidence) !== sha256(normalizedEvidence)) {
|
|
2080
|
+
fail('ADJUDICATION_RECORD_CONFLICT', 'attempt already has different adjudication material');
|
|
2081
|
+
}
|
|
2082
|
+
return Object.freeze(canonicalize({ ...existing.payload, event_id: existing.event_id, replay: true }));
|
|
2083
|
+
}
|
|
2084
|
+
const record = this.#append('adjudication_decided', {
|
|
2085
|
+
attempt_id, task_id: attempt.task_id, review_ref, adjudication_ref,
|
|
2086
|
+
decision, evidence: normalizedEvidence,
|
|
2087
|
+
});
|
|
2088
|
+
let terminalTransition;
|
|
2089
|
+
const terminalState = adjudicationAttemptState(decision);
|
|
2090
|
+
if (terminalState === 'cancelled') {
|
|
2091
|
+
terminalTransition = this.#append('attempt_cancelled', {
|
|
2092
|
+
attempt_id,
|
|
2093
|
+
task_id: attempt.task_id,
|
|
2094
|
+
claim_id: attempt.claim.claim_id,
|
|
2095
|
+
reason: 'handoff_revision_required',
|
|
2096
|
+
adjudication_ref,
|
|
2097
|
+
handoff_revision_ref: decision.handoff_revision_ref,
|
|
2098
|
+
cancelled_at: timestamp(this.#clock),
|
|
2099
|
+
}, record.event_id);
|
|
2100
|
+
} else if (terminalState === 'blocked') {
|
|
2101
|
+
terminalTransition = this.#append('attempt_blocked', {
|
|
2102
|
+
attempt_id,
|
|
2103
|
+
task_id: attempt.task_id,
|
|
2104
|
+
claim_id: attempt.claim.claim_id,
|
|
2105
|
+
reason: 'adjudication_authority_required',
|
|
2106
|
+
adjudication_ref,
|
|
2107
|
+
authority_reason: decision.authority_reason,
|
|
2108
|
+
authority_reason_ref: decision.authority_reason_ref,
|
|
2109
|
+
blocked_at: timestamp(this.#clock),
|
|
2110
|
+
}, record.event_id);
|
|
2111
|
+
}
|
|
2112
|
+
return Object.freeze(canonicalize({
|
|
2113
|
+
attempt_id, task_id: attempt.task_id, review_ref, adjudication_ref,
|
|
2114
|
+
decision, evidence: normalizedEvidence, event_id: record.event_id,
|
|
2115
|
+
...(terminalTransition
|
|
2116
|
+
? { [terminalState === 'cancelled' ? 'attempt_cancellation_event_id' : 'attempt_block_event_id']: terminalTransition.event_id }
|
|
2117
|
+
: {}),
|
|
2118
|
+
replay: false,
|
|
2119
|
+
}));
|
|
2120
|
+
});
|
|
2121
|
+
}
|
|
2122
|
+
|
|
2123
|
+
recoverAttempt({ attempt_id, operator_id, harness_process_id, git_dir, lease_ttl_ms = 300000 } = {}) {
|
|
360
2124
|
return this.#exclusive(() => {
|
|
361
2125
|
for (const [field, value] of Object.entries({ attempt_id, operator_id, harness_process_id })) assertString(value, 'INVALID_RECOVERY', field);
|
|
362
2126
|
assertString(git_dir, 'INVALID_RECOVERY', 'git_dir');
|
|
@@ -364,7 +2128,79 @@ export class RailEngine {
|
|
|
364
2128
|
if (!attempt || ![EXECUTABLE_ATTEMPT_STATE, 'ready_to_merge'].includes(attempt.state)) fail('ATTEMPT_NOT_RECOVERABLE', 'attempt is not recoverable', { attempt_id });
|
|
365
2129
|
const now = timestamp(this.#clock);
|
|
366
2130
|
if (!isExpired(attempt.claim, now)) fail('LEASE_STILL_ACTIVE', 'active lease cannot be recovered', { attempt_id });
|
|
367
|
-
|
|
2131
|
+
const latestClaimMutation = [...this.#records].reverse().find((record) =>
|
|
2132
|
+
['attempt_claimed', 'attempt_heartbeat', 'attempt_relinquished', 'attempt_recovered'].includes(record.type)
|
|
2133
|
+
&& (record.payload?.attempt_id === attempt_id || record.payload?.attempt?.attempt_id === attempt_id));
|
|
2134
|
+
const explicitlyRelinquished = latestClaimMutation?.type === 'attempt_relinquished';
|
|
2135
|
+
const stoppedPid = explicitlyRelinquished
|
|
2136
|
+
? claimProcessPid(attempt)
|
|
2137
|
+
: observeStoppedClaimProcess(attempt);
|
|
2138
|
+
const successfulExecution = this.#executionEvidence(attempt_id, { successfulOnly: true }).at(-1);
|
|
2139
|
+
const expectedRecoveryCommit = successfulExecution?.result_commit_ref ?? attempt.git_commit_ref;
|
|
2140
|
+
verifyLocalGitCommit({ git_dir, git_commit_ref: attempt.git_commit_ref });
|
|
2141
|
+
verifyLocalGitCommit({ git_dir, git_commit_ref: expectedRecoveryCommit });
|
|
2142
|
+
const unresolvedLaunchReservations = this.#records.filter((record) => record.type === 'execution_launch_reserved'
|
|
2143
|
+
&& record.payload?.attempt_id === attempt_id
|
|
2144
|
+
&& !this.#records.some((candidate) => candidate.type === 'execution_dispatched'
|
|
2145
|
+
&& candidate.payload?.invocation?.launch_nonce === record.payload.invocation.launch_nonce));
|
|
2146
|
+
if (unresolvedLaunchReservations.length > 0) {
|
|
2147
|
+
fail('RECOVERY_EXECUTION_LAUNCH_UNRESOLVED', 'automatic recovery cannot prove whether a reserved provider launch started before the supervisor stopped', {
|
|
2148
|
+
launch_nonces: unresolvedLaunchReservations.map((record) => record.payload.invocation.launch_nonce),
|
|
2149
|
+
});
|
|
2150
|
+
}
|
|
2151
|
+
const unresolvedInitialDispatches = this.#records.filter((record) => record.type === 'execution_dispatched'
|
|
2152
|
+
&& record.payload?.attempt_id === attempt_id
|
|
2153
|
+
&& record.payload?.run_kind === 'initial'
|
|
2154
|
+
&& !this.#records.some((candidate) => candidate.type === 'execution_recorded'
|
|
2155
|
+
&& candidate.payload?.execution_evidence?.receipt_nonce === record.payload.invocation.launch_nonce));
|
|
2156
|
+
if (unresolvedInitialDispatches.length > 0) {
|
|
2157
|
+
fail('RECOVERY_EXECUTION_DISPATCH_UNRESOLVED', 'automatic recovery cannot authorize another mutating provider after an interrupted initial dispatch', {
|
|
2158
|
+
launch_nonces: unresolvedInitialDispatches.map((record) => record.payload.invocation.launch_nonce),
|
|
2159
|
+
});
|
|
2160
|
+
}
|
|
2161
|
+
const unresolvedReworkLaunches = this.#records.filter((record) => record.type === 'rework_launch_reserved'
|
|
2162
|
+
&& record.payload?.attempt_id === attempt_id
|
|
2163
|
+
&& !this.#records.some((candidate) => candidate.type === 'execution_dispatched'
|
|
2164
|
+
&& candidate.payload?.invocation?.launch_nonce === record.payload.invocation.launch_nonce));
|
|
2165
|
+
if (unresolvedReworkLaunches.length > 0) {
|
|
2166
|
+
fail('RECOVERY_REWORK_LAUNCH_UNRESOLVED', 'automatic recovery cannot prove whether a reserved rework launch started before the supervisor stopped', {
|
|
2167
|
+
launch_nonces: unresolvedReworkLaunches.map((record) => record.payload.invocation.launch_nonce),
|
|
2168
|
+
});
|
|
2169
|
+
}
|
|
2170
|
+
const unresolvedReworkDispatches = this.#records.filter((record) => record.type === 'execution_dispatched'
|
|
2171
|
+
&& record.payload?.attempt_id === attempt_id
|
|
2172
|
+
&& record.payload?.run_kind === 'rework'
|
|
2173
|
+
&& !this.#records.some((candidate) => candidate.type === 'execution_recorded'
|
|
2174
|
+
&& candidate.payload?.execution_evidence?.receipt_nonce === record.payload.invocation.launch_nonce));
|
|
2175
|
+
if (unresolvedReworkDispatches.length > 0) {
|
|
2176
|
+
fail('RECOVERY_REWORK_DISPATCH_UNRESOLVED', 'automatic recovery cannot authorize another mutating provider after an interrupted rework dispatch', {
|
|
2177
|
+
launch_nonces: unresolvedReworkDispatches.map((record) => record.payload.invocation.launch_nonce),
|
|
2178
|
+
});
|
|
2179
|
+
}
|
|
2180
|
+
const recoveryCausation = attempt.claim_event_id;
|
|
2181
|
+
const effects = [...this.#effects().values()].filter((effect) => effect.attempt_id === attempt_id);
|
|
2182
|
+
const unresolved = effects.filter((effect) => !effect.receipt_ref);
|
|
2183
|
+
if (unresolved.length > 0) {
|
|
2184
|
+
fail('RECOVERY_EFFECTS_UNRESOLVED', 'prepared effects without canonical receipts require external reconciliation before recovery', {
|
|
2185
|
+
idempotency_keys: unresolved.map((effect) => effect.idempotency_key),
|
|
2186
|
+
});
|
|
2187
|
+
}
|
|
2188
|
+
const priorCompletion = [...this.#records].reverse().find((record) => record.type === 'attempt_completed'
|
|
2189
|
+
&& record.payload?.completion?.attempt_id === attempt_id);
|
|
2190
|
+
const reconciliation = canonicalize({
|
|
2191
|
+
schema_version: 2,
|
|
2192
|
+
status: 'safe_to_resume',
|
|
2193
|
+
process: explicitlyRelinquished
|
|
2194
|
+
? { status: 'relinquished', pid: stoppedPid, evidence_refs: [`rail-observation://process/${attempt.claim.claim_id}/relinquished`] }
|
|
2195
|
+
: { status: 'not_running', pid: stoppedPid, evidence_refs: [`rail-observation://process/${attempt.claim.claim_id}/not-running`] },
|
|
2196
|
+
heartbeat: { status: 'expired', observed_at: now, evidence_refs: [`rail-observation://lease/${attempt.claim.claim_id}/expired`] },
|
|
2197
|
+
git: { status: 'matched', commit_ref: expectedRecoveryCommit, evidence_refs: [`rail-observation://git/${expectedRecoveryCommit}`] },
|
|
2198
|
+
ci: priorCompletion
|
|
2199
|
+
? { status: priorCompletion.payload.completion.ci.verdict, evidence_refs: priorCompletion.payload.completion.ci.evidence_refs }
|
|
2200
|
+
: { status: 'not_started', evidence_refs: [`rail-observation://ci/${attempt_id}/not-started`] },
|
|
2201
|
+
effects: effects.map((effect) => ({ idempotency_key: effect.idempotency_key, status: 'receipted', receipt_ref: effect.receipt_ref })),
|
|
2202
|
+
execution_dispatches: [],
|
|
2203
|
+
});
|
|
368
2204
|
const claim = canonicalize({
|
|
369
2205
|
...attempt.claim,
|
|
370
2206
|
operator_id,
|
|
@@ -372,12 +2208,16 @@ export class RailEngine {
|
|
|
372
2208
|
lease: { acquired_at: now, heartbeat_at: now, expires_at: plusMs(now, lease_ttl_ms) },
|
|
373
2209
|
});
|
|
374
2210
|
const resume_state = attempt.state;
|
|
375
|
-
const record = this.#append('attempt_recovered', { attempt_id, claim, reconciliation, resume_state },
|
|
2211
|
+
const record = this.#append('attempt_recovered', { attempt_id, claim, reconciliation, resume_state }, recoveryCausation);
|
|
376
2212
|
return Object.freeze(canonicalize({ attempt_id, claim, resume_state, event_id: record.event_id }));
|
|
377
2213
|
});
|
|
378
2214
|
}
|
|
379
2215
|
|
|
380
|
-
completeAttempt({
|
|
2216
|
+
completeAttempt({
|
|
2217
|
+
completion_id, attempt_id, claim_id, operator_id, state, git, ci, reviews,
|
|
2218
|
+
acceptance_evidence_refs, acceptance_authority_receipt, browser_evidence,
|
|
2219
|
+
completed_at, metrics, handoff,
|
|
2220
|
+
} = {}) {
|
|
381
2221
|
return this.#exclusive(() => {
|
|
382
2222
|
assertString(completion_id, 'INVALID_COMPLETION', 'completion_id');
|
|
383
2223
|
assertString(attempt_id, 'INVALID_COMPLETION', 'attempt_id');
|
|
@@ -410,22 +2250,66 @@ export class RailEngine {
|
|
|
410
2250
|
const allowedTransition = (attempt?.state === EXECUTABLE_ATTEMPT_STATE && state === 'ready_to_merge') || (attempt?.state === 'ready_to_merge' && state === 'merged');
|
|
411
2251
|
if (!allowedTransition) fail('INVALID_COMPLETION_TRANSITION', 'completion transition is not allowed for attempt state', { attempt_id, state, current_state: attempt?.state });
|
|
412
2252
|
validateAttemptBinding({ attempt, handoff });
|
|
2253
|
+
const task = handoff.tasks.find((candidate) => candidate.task_id === attempt.task_id);
|
|
2254
|
+
let verifiedBrowserEvidence;
|
|
2255
|
+
if (task?.requires_browser === true) {
|
|
2256
|
+
const browserDispatch = [...this.#records].reverse().find((record) => record.type === 'execution_dispatched'
|
|
2257
|
+
&& record.payload?.attempt_id === attempt_id
|
|
2258
|
+
&& record.payload?.run_kind === 'initial'
|
|
2259
|
+
&& record.payload?.browser_manifest);
|
|
2260
|
+
verifiedBrowserEvidence = validateRealBrowserEvidence({
|
|
2261
|
+
evidence: browser_evidence,
|
|
2262
|
+
task,
|
|
2263
|
+
attemptId: attempt_id,
|
|
2264
|
+
commitRef: git.result_ref,
|
|
2265
|
+
acceptanceEvidenceRefs: acceptance_evidence_refs,
|
|
2266
|
+
delegationManifest: browserDispatch?.payload?.browser_manifest,
|
|
2267
|
+
verifyAuthority: (receipt, expected) => this.#verifyAuthority('browser_evidence', receipt, expected),
|
|
2268
|
+
});
|
|
2269
|
+
} else if (browser_evidence !== undefined) {
|
|
2270
|
+
fail('UNEXPECTED_BROWSER_EVIDENCE', 'non-browser task completion must not claim real browser evidence');
|
|
2271
|
+
}
|
|
413
2272
|
if (attempt.claim.claim_id !== claim_id || attempt.claim.operator_id !== operator_id) fail('CLAIM_OWNER_MISMATCH', 'only the active claim owner may complete an attempt', { attempt_id });
|
|
414
2273
|
this.#assertLiveLease(attempt, timestamp(this.#clock));
|
|
415
|
-
|
|
416
|
-
|
|
2274
|
+
assertString(git.base_ref, 'INVALID_COMPLETION', 'git.base_ref');
|
|
2275
|
+
assertString(git.result_ref, 'INVALID_COMPLETION', 'git.result_ref');
|
|
2276
|
+
if (git.base_ref !== attempt.git_commit_ref) fail('GIT_BASE_MISMATCH', 'completion base_ref must equal the sealed attempt commit', { attempt_id });
|
|
2277
|
+
if (!git.commit_refs.includes(git.base_ref) || !git.commit_refs.includes(git.result_ref)) {
|
|
2278
|
+
fail('GIT_COMMIT_MISMATCH', 'completion commit_refs must include both sealed base and reviewed result commits', { attempt_id });
|
|
2279
|
+
}
|
|
417
2280
|
for (const commit of git.commit_refs) verifyLocalGitCommit({ git_dir: git.git_dir, git_commit_ref: commit });
|
|
418
|
-
|
|
2281
|
+
verifyCommitDescendsFrom({ git_dir: git.git_dir, base_commit_ref: git.base_ref, result_commit_ref: git.result_ref });
|
|
2282
|
+
assertString(ci.commit_ref, 'INVALID_COMPLETION', 'ci.commit_ref');
|
|
2283
|
+
if (ci.commit_ref !== git.result_ref) fail('CI_SHA_MISMATCH', 'CI evidence must be bound to the reviewed result commit', { attempt_id });
|
|
2284
|
+
assertObject(ci.authority_receipt, 'UNTRUSTED_CI_EVIDENCE', 'ci.authority_receipt');
|
|
2285
|
+
this.#verifyAuthority('ci_evidence', ci.authority_receipt, {
|
|
2286
|
+
purpose: 'rail_evidence', authority_kind: 'rail_ci', attempt_id,
|
|
2287
|
+
commit_ref: git.result_ref, verdict: 'passed', evidence_digest: sha256(ci.evidence_refs),
|
|
2288
|
+
});
|
|
2289
|
+
assertObject(acceptance_authority_receipt, 'UNTRUSTED_ACCEPTANCE_EVIDENCE', 'acceptance_authority_receipt');
|
|
2290
|
+
this.#verifyAuthority('acceptance_evidence', acceptance_authority_receipt, {
|
|
2291
|
+
purpose: 'rail_evidence', authority_kind: 'rail_acceptance', attempt_id,
|
|
2292
|
+
commit_ref: git.result_ref, verdict: 'passed', evidence_digest: sha256(acceptance_evidence_refs),
|
|
2293
|
+
});
|
|
2294
|
+
if (unresolvedAdjudication(this.#records, attempt_id)) {
|
|
2295
|
+
fail('RAIL_ADJUDICATION_REQUIRED', 'attempt completion requires the pending AI adjudication to be decided first');
|
|
2296
|
+
}
|
|
2297
|
+
const acceptedReview = this.#assertAcceptedReviews(attempt_id, reviews);
|
|
2298
|
+
if (acceptedReview.payload.result_commit_ref !== git.result_ref) {
|
|
2299
|
+
fail('REVIEW_GIT_MISMATCH', 'accepted review must be bound to the completion result commit', { attempt_id });
|
|
2300
|
+
}
|
|
419
2301
|
const now = timestamp(this.#clock);
|
|
420
2302
|
const completion = canonicalize({
|
|
421
2303
|
completion_id,
|
|
422
2304
|
attempt_id,
|
|
423
2305
|
task_id: attempt.task_id,
|
|
424
2306
|
state,
|
|
425
|
-
git: {
|
|
426
|
-
ci: { evidence_refs: ci.evidence_refs, verdict: ci.verdict,
|
|
2307
|
+
git: { base_ref: git.base_ref, result_ref: git.result_ref, commit_refs: git.commit_refs, ...(git.pr_ref ? { pr_ref: git.pr_ref } : {}) },
|
|
2308
|
+
ci: { evidence_refs: ci.evidence_refs, verdict: ci.verdict, commit_ref: ci.commit_ref, authority_receipt: ci.authority_receipt },
|
|
427
2309
|
reviews,
|
|
428
2310
|
acceptance_evidence_refs,
|
|
2311
|
+
acceptance_authority_receipt,
|
|
2312
|
+
...(verifiedBrowserEvidence === undefined ? {} : { browser_evidence: verifiedBrowserEvidence }),
|
|
429
2313
|
...(metrics === undefined ? {} : { metrics }),
|
|
430
2314
|
completed_at: completed_at ?? now,
|
|
431
2315
|
});
|