chati-dev 4.5.28 → 4.5.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/chati.js +151 -123
- package/framework/agents/build/dev.md +4 -4
- package/framework/agents/deploy/devops.md +3 -3
- package/framework/agents/discover/brief.md +18 -6
- package/framework/agents/discover/brownfield-wu.md +3 -3
- package/framework/agents/discover/greenfield-wu.md +4 -4
- package/framework/agents/plan/architect.md +2 -2
- package/framework/agents/plan/detail.md +4 -4
- package/framework/agents/plan/phases.md +3 -3
- package/framework/agents/plan/tasks.md +47 -19
- package/framework/agents/plan/ux-brand-architect.md +1 -1
- package/framework/agents/plan/ux.md +26 -2
- package/framework/agents/quality/qa-implementation.md +8 -8
- package/framework/agents/quality/qa-planning.md +58 -4
- package/framework/agents/quality/qa-visual.md +4 -4
- package/framework/config.yaml +2 -2
- package/framework/constitution.md +12 -1
- package/framework/context/governance.md +2 -1
- package/framework/context/root.md +3 -1
- package/framework/domains/constitution.yaml +1 -1
- package/framework/domains/global.yaml +1 -1
- package/framework/domains/workflows/standard-flow.yaml +7 -8
- package/framework/hooks/advance-trigger.js +3 -2
- package/framework/hooks/brief-validator.js +1 -1
- package/framework/hooks/post-dev.js +1 -1
- package/framework/hooks/team-quality-gate.js +9 -2
- package/framework/i18n/en.yaml +1 -0
- package/framework/i18n/es.yaml +1 -0
- package/framework/i18n/fr.yaml +1 -0
- package/framework/i18n/pt.yaml +1 -0
- package/framework/manifest.json +83 -83
- package/framework/manifest.sig +1 -1
- package/framework/orchestrator/chati-router.js +14 -6
- package/framework/orchestrator/chati.md +61 -23
- package/framework/rules/human-writing-style.md +7 -0
- package/framework/schemas/session.schema.json +25 -2
- package/framework/schemas/task.schema.json +26 -0
- package/framework/tasks/orchestrator-handoff.md +10 -10
- package/framework/tasks/orchestrator-resume.md +3 -3
- package/framework/workflows/brownfield-fullstack.yaml +6 -3
- package/framework/workflows/brownfield-service.yaml +5 -3
- package/framework/workflows/brownfield-ui.yaml +5 -3
- package/framework/workflows/greenfield-fullstack.yaml +6 -3
- package/framework/workflows/standard-flow.yaml +8 -3
- package/node_modules/@chati/core/native/browser-proof-wire.h +29 -0
- package/node_modules/@chati/core/native/darwin-browser-listener.c +36 -0
- package/node_modules/@chati/core/native/darwin-browser-listener.h +32 -0
- package/node_modules/@chati/core/native/darwin-browser-proof.h +86 -0
- package/node_modules/@chati/core/native/darwin-code-identity.h +35 -0
- package/node_modules/@chati/core/native/darwin-launch-messages.h +37 -0
- package/node_modules/@chati/core/native/darwin-launcher.c +312 -0
- package/node_modules/@chati/core/native/descriptor-fs.c +407 -0
- package/node_modules/@chati/core/native/execution-admission.h +59 -0
- package/node_modules/@chati/core/native/execution-gate.c +41 -0
- package/node_modules/@chati/core/native/linux-process.c +41 -0
- package/node_modules/@chati/core/native/prebuilds/darwin-arm64/darwin-launcher +0 -0
- package/node_modules/@chati/core/native/prebuilds/darwin-arm64/descriptor-fs.node +0 -0
- package/node_modules/@chati/core/native/prebuilds/darwin-arm64/execution-gate +0 -0
- package/node_modules/@chati/core/native/prebuilds/linux-x64/descriptor-fs.node +0 -0
- package/node_modules/@chati/core/native/prebuilds/linux-x64/execution-gate +0 -0
- package/node_modules/@chati/core/native/process-info-pipe.c +91 -0
- package/node_modules/@chati/core/package.json +2 -1
- package/node_modules/@chati/core/scripts/build-native.js +36 -0
- package/node_modules/@chati/core/src/darwin-browser-proof.js +87 -0
- package/node_modules/@chati/core/src/darwin-host-boot.js +58 -0
- package/node_modules/@chati/core/src/darwin-process.js +79 -0
- package/node_modules/@chati/core/src/darwin-supervisor-channels.js +82 -0
- package/node_modules/@chati/core/src/darwin-terminal-receipt.js +111 -0
- package/node_modules/@chati/core/src/descriptor-fs.js +222 -0
- package/node_modules/@chati/core/src/execution-gate.js +8 -0
- package/node_modules/@chati/core/src/host-boot-marker.js +45 -0
- package/node_modules/@chati/core/src/index.js +18 -0
- package/node_modules/@chati/core/src/linux-host-boot.js +218 -0
- package/node_modules/@chati/core/src/linux-process.js +133 -0
- package/node_modules/@chati/planning/src/browser-verification-plan.js +71 -0
- package/node_modules/@chati/planning/src/index.js +33 -0
- package/node_modules/@chati/planning/src/planning-material.js +64 -0
- package/node_modules/@chati/planning/src/verification-plan.js +57 -0
- package/node_modules/@chati/rail/package.json +1 -0
- package/node_modules/@chati/rail/src/execution-restoration.js +92 -0
- package/node_modules/@chati/rail/src/index.js +812 -167
- package/node_modules/@chati/rail/src/provider-processing.js +118 -0
- package/node_modules/@chati/rail/src/terminal-containment.js +23 -0
- package/node_modules/@chati/release-lane/src/index.js +40 -43
- package/node_modules/@chati/review-council/src/index.js +223 -0
- package/node_modules/@chati/tracking-clickup/src/index.js +67 -58
- package/package-artifact-manifest.json +1 -1
- package/package-artifact-manifest.sig +1 -1
- package/package.json +3 -2
- package/scripts/verify-real-harness-e2e.js +20 -8
- package/src/installer/core.js +147 -107
- package/src/installer/package-artifact.js +7 -0
- package/src/installer/preflight.js +13 -0
- package/src/installer/provider-overlay.js +1 -2
- package/src/installer/templates.js +58 -0
- package/src/installer-v2/catalog-client.js +31 -1
- package/src/installer-v2/clickup-connection-storage.js +92 -0
- package/src/installer-v2/clickup-mcp-client.js +155 -0
- package/src/installer-v2/clickup-oauth-loopback.js +96 -0
- package/src/installer-v2/clickup-oauth.js +233 -0
- package/src/installer-v2/index.js +141 -15
- package/src/installer-v2/installation-authority.js +98 -4
- package/src/installer-v2/provider-executable.js +117 -12
- package/src/installer-v2/wizard-installation.js +11 -0
- package/src/orchestrator/agent-selector.js +39 -1
- package/src/orchestrator/browser-artifact-store.js +131 -0
- package/src/orchestrator/cli.js +762 -205
- package/src/orchestrator/clickup-mcp-observer.js +226 -0
- package/src/orchestrator/clickup-runtime.js +45 -24
- package/src/orchestrator/council-gate-runtime.js +903 -0
- package/src/orchestrator/evidence-authority-enrollment.js +75 -0
- package/src/orchestrator/evidence-authority-request.js +46 -0
- package/src/orchestrator/handoff-engine.js +4 -11
- package/src/orchestrator/pipeline-manager.js +42 -4
- package/src/orchestrator/planning-contract-revision.js +711 -0
- package/src/orchestrator/planning-reconciliation.js +302 -0
- package/src/orchestrator/planning-runtime.js +476 -29
- package/src/orchestrator/rail-adjudication-evidence.js +15 -24
- package/src/orchestrator/rail-human-effort-authority.js +46 -0
- package/src/orchestrator/rail-human-effort.js +149 -0
- package/src/orchestrator/rail-runtime.js +291 -47
- package/src/orchestrator/rail-verification.js +278 -0
- package/src/orchestrator/review-runtime.js +37 -2
- package/src/orchestrator/runtime-installation-v2.js +143 -24
- package/src/orchestrator/session-manager.js +1457 -29
- package/src/tasks/handoff.js +57 -33
- package/src/terminal/adapters/claude-adapter.js +3 -0
- package/src/terminal/adapters/codex-adapter.js +13 -0
- package/src/terminal/adapters/grok-adapter.js +1 -0
- package/src/terminal/agent-attempt-budget.js +35 -0
- package/src/terminal/browser-endpoint-proxy.js +174 -0
- package/src/terminal/browser-observer-driver.js +94 -0
- package/src/terminal/browser-observer-supervisor.js +73 -0
- package/src/terminal/browser-observer.js +95 -0
- package/src/terminal/browser-process-facts.js +75 -0
- package/src/terminal/browser-runtime.js +131 -0
- package/src/terminal/darwin-authority-policy.js +213 -0
- package/src/terminal/darwin-browser-application.js +116 -0
- package/src/terminal/darwin-browser-observer.js +57 -0
- package/src/terminal/darwin-execution.js +185 -0
- package/src/terminal/executable-snapshot.js +202 -0
- package/src/terminal/handoff-parser.js +48 -2
- package/src/terminal/isolation.js +14 -13
- package/src/terminal/prompt-builder.js +176 -15
- package/src/terminal/provider-preflight.js +47 -27
- package/src/terminal/rail-execution-worktree.js +22 -6
- package/src/terminal/rail-process-observer.js +10 -0
- package/src/terminal/rail-prompts.js +21 -3
- package/src/terminal/rail-readonly-workspace.js +54 -5
- package/src/terminal/rail-worktree-checkpoint.js +315 -0
- package/src/terminal/rail-worktree-restoration.js +485 -0
- package/src/terminal/run-agent.js +164 -42
- package/src/terminal/run-council-gate.js +338 -0
- package/src/terminal/run-planning-reconciliation.js +49 -0
- package/src/terminal/run-rail-adjudication.js +29 -7
- package/src/terminal/run-rail-recovery.js +97 -0
- package/src/terminal/run-rail-review.js +36 -15
- package/src/terminal/run-rail-rework.js +26 -15
- package/src/terminal/run-rail-task.js +22 -9
- package/src/terminal/runtime-native-helper.js +85 -0
- package/src/terminal/spawner.js +899 -142
- package/src/terminal/verification-process-facts.js +49 -0
- package/src/upgrade/entrypoints.js +68 -0
- package/src/upgrade/generation.js +194 -0
- package/src/upgrade/maintenance.js +79 -0
- package/src/upgrade/paths.js +81 -0
- package/src/upgrade/upgrade.js +267 -0
- package/src/utils/execution-role.js +15 -0
- package/src/utils/handoff-paths.js +59 -0
- package/src/wizard/i18n.js +1 -0
- package/src/wizard/index.js +17 -5
- package/src/wizard/questions.js +24 -12
|
@@ -1,14 +1,24 @@
|
|
|
1
1
|
import {
|
|
2
|
-
closeSync, constants,
|
|
3
|
-
openSync, readSync,
|
|
2
|
+
closeSync, constants, fstatSync, fsyncSync, ftruncateSync,
|
|
3
|
+
openSync, readSync, writeSync,
|
|
4
4
|
} from 'node:fs';
|
|
5
5
|
import { hostname } from 'node:os';
|
|
6
6
|
import { randomUUID } from 'node:crypto';
|
|
7
7
|
import { execFileSync } from 'node:child_process';
|
|
8
8
|
import { basename, dirname, join, parse, resolve, sep } from 'node:path';
|
|
9
|
-
import {
|
|
9
|
+
import {
|
|
10
|
+
ContractError, canonicalize, sha256, directoryPathSync, openAtSync,
|
|
11
|
+
mkdirAtSync, lstatAtSync, renameAtSync, unlinkAtSync, assertLinuxNamespaceIdentity, observeLinuxNamespace,
|
|
12
|
+
assertDarwinHostBootIdentity, captureDarwinHostBootIdentity, compareDarwinHostBootIdentity, observeDarwinPreviousBoot,
|
|
13
|
+
tryExclusiveFileLockSync, captureHostBootMarker, hostBootMarkerChanged,
|
|
14
|
+
} from '@chati/core';
|
|
10
15
|
import { createAttemptBinding, validateAttemptBinding, verifyLocalGitCommit } from '@chati/planning';
|
|
11
16
|
import { acceptReview, decideReviewProgress, selectCanonicalReviewer, validateAdjudicationDecision } from '@chati/review-council';
|
|
17
|
+
import { assertEligibleInvocation } from '@chati/provider-registry';
|
|
18
|
+
import { assertProviderProcessOutput, assertProviderProcessReceipt, collectProviderProcessing, providerProcessDispatches, providerProcessingMetric } from './provider-processing.js';
|
|
19
|
+
import { assertExecutionInterruptionLifetime, executionCheckpointBinding, restoredExecutionProof, isWellFormedTerminationAttestation } from './execution-restoration.js';
|
|
20
|
+
import { assertTerminalLaunchContainment, assertTerminalCompletionContainment } from './terminal-containment.js';
|
|
21
|
+
export { collectProviderProcessing, providerProcessingMetric } from './provider-processing.js';
|
|
12
22
|
|
|
13
23
|
const RFC3339 = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/;
|
|
14
24
|
const TASK_CLOSED_STATES = new Set(['merged', 'blocked', 'cancelled']);
|
|
@@ -68,6 +78,25 @@ function assertAllowedKeys(value, allowed, code, label) {
|
|
|
68
78
|
if (Object.keys(value).some((key) => !allowed.has(key))) fail(code, `${label} contains unsupported fields`);
|
|
69
79
|
}
|
|
70
80
|
|
|
81
|
+
/** Acceptance must certify the values as well as the underlying source refs. */
|
|
82
|
+
export function completionMetricsReference({ attempt_id, commit_ref, metrics } = {}) {
|
|
83
|
+
assertString(attempt_id, 'INVALID_COMPLETION_METRICS', 'attempt_id');
|
|
84
|
+
assertString(commit_ref, 'INVALID_COMPLETION_METRICS', 'commit_ref');
|
|
85
|
+
assertObject(metrics, 'INVALID_COMPLETION_METRICS', 'metrics');
|
|
86
|
+
return `rail-metrics://sha256/${sha256({ schema_version: 1, attempt_id, commit_ref, metrics })}`;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function effortJournalDigest(records, attempt_id) {
|
|
90
|
+
const types = new Set(['human_effort_observed', 'provider_process_observed', 'execution_dispatched',
|
|
91
|
+
'execution_recorded', 'review_dispatched', 'review_decided', 'adjudication_dispatched',
|
|
92
|
+
'adjudication_observed', 'adjudication_decided', 'execution_interruption_observed',
|
|
93
|
+
'execution_checkpoint_preserved', 'execution_worktree_restored', 'attempt_recovered']);
|
|
94
|
+
return sha256(records.filter((record) => types.has(record.type) && record.payload?.attempt_id === attempt_id
|
|
95
|
+
// Ordinary ownership renewal adds no measurement. Only consumption of an
|
|
96
|
+
// interrupted dispatch changes processing coverage.
|
|
97
|
+
&& (record.type !== 'attempt_recovered' || record.payload.reconciliation?.execution_dispatches?.length > 0)));
|
|
98
|
+
}
|
|
99
|
+
|
|
71
100
|
export function browserEvidenceMaterial(evidence = {}) {
|
|
72
101
|
return canonicalize({
|
|
73
102
|
schema_version: evidence.schema_version,
|
|
@@ -164,7 +193,8 @@ function validateRealBrowserEvidence({
|
|
|
164
193
|
purpose: 'rail_evidence', authority_kind: 'rail_browser', attempt_id: attemptId,
|
|
165
194
|
commit_ref: commitRef, verdict: 'passed', evidence_digest: sha256(material),
|
|
166
195
|
observed_at: evidence.observed_at,
|
|
167
|
-
}
|
|
196
|
+
}, { artifacts: evidence.artifacts, policy_ref: evidence.policy_ref, browser_session_id: evidence.browser_session_id,
|
|
197
|
+
captured_at: evidence.observed_at });
|
|
168
198
|
return canonicalize(evidence);
|
|
169
199
|
}
|
|
170
200
|
|
|
@@ -278,69 +308,43 @@ function stableEventId(sequence, event) {
|
|
|
278
308
|
function checkpointPath(journalPath) { return `${journalPath}.checkpoint`; }
|
|
279
309
|
function lockPath(journalPath) { return `${journalPath}.lock`; }
|
|
280
310
|
|
|
281
|
-
function ensureSafeDirectory(path, { create = false } = {}) {
|
|
282
|
-
const absolute = resolve(path);
|
|
283
|
-
const root = parse(absolute).root;
|
|
284
|
-
let current = root;
|
|
285
|
-
for (const segment of absolute.slice(root.length).split(sep).filter(Boolean)) {
|
|
286
|
-
current = resolve(current, segment);
|
|
287
|
-
let stat;
|
|
288
|
-
try { stat = lstatSync(current); }
|
|
289
|
-
catch (error) {
|
|
290
|
-
if (error?.code !== 'ENOENT' || !create) throw error;
|
|
291
|
-
try { mkdirSync(current, { mode: 0o700 }); }
|
|
292
|
-
catch (mkdirError) { if (mkdirError?.code !== 'EEXIST') throw mkdirError; }
|
|
293
|
-
stat = lstatSync(current);
|
|
294
|
-
}
|
|
295
|
-
if (stat.isSymbolicLink() || !stat.isDirectory()) {
|
|
296
|
-
fail('UNSAFE_JOURNAL_DIRECTORY', 'journal directory chain must contain only real directories', { path: current });
|
|
297
|
-
}
|
|
298
|
-
}
|
|
299
|
-
return absolute;
|
|
300
|
-
}
|
|
301
|
-
|
|
302
311
|
function openDirectoryForSync(path, { create = false } = {}) {
|
|
312
|
+
if (!['linux', 'darwin'].includes(process.platform)) {
|
|
313
|
+
fail('UNSUPPORTED_JOURNAL_STORAGE_PLATFORM', `journal storage requires descriptor-relative operations; unsupported platform: ${process.platform}`);
|
|
314
|
+
}
|
|
303
315
|
const absolute = resolve(path);
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
if (['ELOOP', 'ENOTDIR'].includes(error?.code)) {
|
|
317
|
-
fail('UNSAFE_JOURNAL_DIRECTORY', 'journal directory chain must contain only real directories');
|
|
318
|
-
}
|
|
319
|
-
throw error;
|
|
316
|
+
const root = parse(absolute).root;
|
|
317
|
+
const segments = absolute.slice(root.length).split(sep).filter(Boolean);
|
|
318
|
+
let fd = openSync(root, constants.O_RDONLY | (constants.O_DIRECTORY || 0) | (constants.O_NOFOLLOW || 0));
|
|
319
|
+
try {
|
|
320
|
+
for (const segment of segments) {
|
|
321
|
+
let childFd;
|
|
322
|
+
try {
|
|
323
|
+
childFd = openAtSync(fd, segment, constants.O_RDONLY | (constants.O_DIRECTORY || 0) | (constants.O_NOFOLLOW || 0));
|
|
324
|
+
} catch (error) {
|
|
325
|
+
if (error?.code !== 'ENOENT' || !create) {
|
|
326
|
+
if (['ELOOP', 'ENOTDIR'].includes(error?.code)) {
|
|
327
|
+
fail('UNSAFE_JOURNAL_DIRECTORY', 'journal directory chain must contain only real directories');
|
|
320
328
|
}
|
|
321
|
-
|
|
322
|
-
catch (mkdirError) { if (mkdirError?.code !== 'EEXIST') throw mkdirError; }
|
|
323
|
-
childFd = openSync(childPath, constants.O_RDONLY | (constants.O_DIRECTORY || 0) | (constants.O_NOFOLLOW || 0));
|
|
324
|
-
}
|
|
325
|
-
const childStat = fstatSync(childFd);
|
|
326
|
-
if (!childStat.isDirectory()) {
|
|
327
|
-
closeSync(childFd);
|
|
328
|
-
fail('UNSAFE_JOURNAL_DIRECTORY', 'journal directory chain must contain only real directories');
|
|
329
|
+
throw error;
|
|
329
330
|
}
|
|
330
|
-
|
|
331
|
-
|
|
331
|
+
try { mkdirAtSync(fd, segment, { mode: 0o700 }); }
|
|
332
|
+
catch (mkdirError) { if (mkdirError?.code !== 'EEXIST') throw mkdirError; }
|
|
333
|
+
childFd = openAtSync(fd, segment, constants.O_RDONLY | (constants.O_DIRECTORY || 0) | (constants.O_NOFOLLOW || 0));
|
|
334
|
+
}
|
|
335
|
+
const childStat = fstatSync(childFd);
|
|
336
|
+
if (!childStat.isDirectory()) {
|
|
337
|
+
closeSync(childFd);
|
|
338
|
+
fail('UNSAFE_JOURNAL_DIRECTORY', 'journal directory chain must contain only real directories');
|
|
332
339
|
}
|
|
333
|
-
return { fd, path: absolute, stat: fstatSync(fd), reference: `/proc/self/fd/${fd}` };
|
|
334
|
-
} catch (error) {
|
|
335
340
|
closeSync(fd);
|
|
336
|
-
|
|
341
|
+
fd = childFd;
|
|
337
342
|
}
|
|
343
|
+
return { fd, path: absolute, stat: fstatSync(fd) };
|
|
344
|
+
} catch (error) {
|
|
345
|
+
closeSync(fd);
|
|
346
|
+
throw error;
|
|
338
347
|
}
|
|
339
|
-
const safePath = ensureSafeDirectory(absolute, { create });
|
|
340
|
-
const fd = openSync(safePath, constants.O_RDONLY | (constants.O_DIRECTORY || 0) | (constants.O_NOFOLLOW || 0));
|
|
341
|
-
const stat = fstatSync(fd);
|
|
342
|
-
if (!stat.isDirectory()) { closeSync(fd); fail('UNSAFE_JOURNAL_DIRECTORY', 'journal parent must be a directory'); }
|
|
343
|
-
return { fd, path: safePath, stat, reference: safePath };
|
|
344
348
|
}
|
|
345
349
|
|
|
346
350
|
function sameIdentity(left, right) {
|
|
@@ -373,10 +377,10 @@ function readDescriptorBounded(fd, maxBytes, code, label) {
|
|
|
373
377
|
return Buffer.concat(chunks, total).toString('utf8');
|
|
374
378
|
}
|
|
375
379
|
|
|
376
|
-
function readRegularBoundedRecord(
|
|
380
|
+
function readRegularBoundedRecord(directory, name, { allowMissing = false, label = 'journal', maxBytes = JOURNAL_MAX_BYTES } = {}) {
|
|
377
381
|
let fd;
|
|
378
382
|
try {
|
|
379
|
-
fd =
|
|
383
|
+
fd = openAtSync(directory.fd, name, constants.O_RDONLY | (constants.O_NOFOLLOW || 0) | (constants.O_NONBLOCK || 0));
|
|
380
384
|
} catch (error) {
|
|
381
385
|
if (allowMissing && error?.code === 'ENOENT') return null;
|
|
382
386
|
if (['ELOOP', 'ENXIO', 'EISDIR'].includes(error?.code)) fail('UNSAFE_JOURNAL_FILE', `${label} must not be a symlink, FIFO or directory`);
|
|
@@ -386,7 +390,7 @@ function readRegularBoundedRecord(path, { allowMissing = false, label = 'journal
|
|
|
386
390
|
const stat = fstatSync(fd);
|
|
387
391
|
if (!stat.isFile() || stat.nlink !== 1) fail('UNSAFE_JOURNAL_FILE', `${label} must be a regular single-link file`);
|
|
388
392
|
if (stat.size > maxBytes) fail('JOURNAL_SIZE_LIMIT', `${label} exceeds its safety limit`);
|
|
389
|
-
runSecurityTestHook('afterJournalStateStat', { path, label, fd });
|
|
393
|
+
runSecurityTestHook('afterJournalStateStat', { path: join(directoryPathSync(directory.fd), name), label, fd });
|
|
390
394
|
const content = readDescriptorBounded(fd, maxBytes, 'JOURNAL_SIZE_LIMIT', label);
|
|
391
395
|
const after = fstatSync(fd);
|
|
392
396
|
if (!sameSnapshot(stat, after)) fail('JOURNAL_STORAGE_CONFLICT', `${label} changed while it was read`);
|
|
@@ -394,8 +398,8 @@ function readRegularBoundedRecord(path, { allowMissing = false, label = 'journal
|
|
|
394
398
|
} finally { closeSync(fd); }
|
|
395
399
|
}
|
|
396
400
|
|
|
397
|
-
function readRegularBounded(
|
|
398
|
-
return readRegularBoundedRecord(
|
|
401
|
+
function readRegularBounded(directory, name, options) {
|
|
402
|
+
return readRegularBoundedRecord(directory, name, options)?.content ?? null;
|
|
399
403
|
}
|
|
400
404
|
|
|
401
405
|
function processIsAlive(pid) {
|
|
@@ -403,9 +407,9 @@ function processIsAlive(pid) {
|
|
|
403
407
|
try { process.kill(pid, 0); return true; } catch (error) { return error?.code === 'EPERM'; }
|
|
404
408
|
}
|
|
405
409
|
|
|
406
|
-
function readLockOwner(
|
|
410
|
+
function readLockOwner(directory, name) {
|
|
407
411
|
try {
|
|
408
|
-
const raw = readRegularBounded(
|
|
412
|
+
const raw = readRegularBounded(directory, name, { allowMissing: true, label: 'journal lock', maxBytes: 16 * 1024 });
|
|
409
413
|
if (raw === null) return null;
|
|
410
414
|
const owner = JSON.parse(raw);
|
|
411
415
|
if (!owner || typeof owner !== 'object' || typeof owner.token !== 'string') return null;
|
|
@@ -413,58 +417,108 @@ function readLockOwner(path) {
|
|
|
413
417
|
} catch { return null; }
|
|
414
418
|
}
|
|
415
419
|
|
|
416
|
-
function
|
|
420
|
+
function lockOwnerMayBeStopped(owner, hostBootIdentity) {
|
|
421
|
+
if (owner?.darwin_host_boot_identity) {
|
|
422
|
+
try {
|
|
423
|
+
if (compareDarwinHostBootIdentity(owner.darwin_host_boot_identity, hostBootIdentity) === 'previous-boot') return true;
|
|
424
|
+
} catch { return false; }
|
|
425
|
+
}
|
|
426
|
+
if (owner?.hostname && owner.hostname !== hostname()) return false;
|
|
427
|
+
// A changed kernel boot marker proves a same-hostname owner's PID cannot be
|
|
428
|
+
// the one that wrote the lock, even if the numeric PID has since been reused.
|
|
429
|
+
// Darwin identity above is authoritative and already free of native re-reads;
|
|
430
|
+
// never spend a second native observation confirming what it already proved.
|
|
431
|
+
if (!hostBootIdentity && owner?.hostname === hostname() && hostBootMarkerChanged(owner.host_boot_marker, captureHostBootMarker())) return true;
|
|
432
|
+
return !(owner?.hostname === hostname() && processIsAlive(owner.pid));
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
function recoverStaleLock(directory, name, hostBootIdentity) {
|
|
417
436
|
let stat;
|
|
418
|
-
try { stat =
|
|
437
|
+
try { stat = lstatAtSync(directory.fd, name); } catch (error) { return error?.code === 'ENOENT'; }
|
|
419
438
|
if (stat.isSymbolicLink() || !stat.isFile() || stat.nlink !== 1) return false;
|
|
420
|
-
const owner = readLockOwner(
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
// locks are never stolen automatically; an operator must reconcile/remove
|
|
425
|
-
// them after proving the owner is gone.
|
|
426
|
-
if (owner?.hostname && owner.hostname !== hostname()) return false;
|
|
439
|
+
const owner = readLockOwner(directory, name);
|
|
440
|
+
// A native prior-boot owner is gone even if its PID has since been recycled.
|
|
441
|
+
// Remote/malformed identities remain unprovable and are never stolen.
|
|
442
|
+
if (!lockOwnerMayBeStopped(owner, hostBootIdentity)) return false;
|
|
427
443
|
if (!owner && Date.now() - stat.mtimeMs < LOCK_STALE_AFTER_MS) return false;
|
|
428
|
-
const
|
|
444
|
+
const recoveryName = `${name}.recovery`;
|
|
445
|
+
// Skip on Darwin: its identity above is already authoritative, and capturing
|
|
446
|
+
// a marker there would cost a second native host observation for nothing.
|
|
447
|
+
const bootMarker = hostBootIdentity ? null : captureHostBootMarker();
|
|
429
448
|
let recoveryFd;
|
|
430
449
|
try {
|
|
431
|
-
|
|
432
|
-
|
|
450
|
+
if (hostBootIdentity) {
|
|
451
|
+
let created = false;
|
|
452
|
+
try {
|
|
453
|
+
recoveryFd = openAtSync(directory.fd, recoveryName, constants.O_RDWR | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
|
454
|
+
created = true;
|
|
455
|
+
} catch (error) {
|
|
456
|
+
if (error.code !== 'EEXIST') throw error;
|
|
457
|
+
recoveryFd = openAtSync(directory.fd, recoveryName, constants.O_RDWR | constants.O_NONBLOCK | constants.O_NOFOLLOW);
|
|
458
|
+
}
|
|
459
|
+
const recoveryStat = fstatSync(recoveryFd);
|
|
460
|
+
if (!recoveryStat.isFile() || recoveryStat.nlink !== 1
|
|
461
|
+
|| !sameIdentity(recoveryStat, lstatAtSync(directory.fd, recoveryName))
|
|
462
|
+
|| !tryExclusiveFileLockSync(recoveryFd)) throw new Error('recovery guard is not exclusively held');
|
|
463
|
+
const priorOwner = readLockOwner(directory, recoveryName);
|
|
464
|
+
if (!created && priorOwner?.kind !== 'darwin_journal_recovery_flock_v1'
|
|
465
|
+
&& (!lockOwnerMayBeStopped(priorOwner, hostBootIdentity)
|
|
466
|
+
|| (!priorOwner && Date.now() - recoveryStat.mtimeMs < LOCK_STALE_AFTER_MS))) {
|
|
467
|
+
throw new Error('legacy recovery guard owner is unproven');
|
|
468
|
+
}
|
|
469
|
+
// This inode persists: the OS releases its flock on crash/boot. Unlinking
|
|
470
|
+
// it would let concurrent recoverers acquire different inodes as owners.
|
|
471
|
+
ftruncateSync(recoveryFd, 0);
|
|
472
|
+
} else {
|
|
473
|
+
recoveryFd = openAtSync(directory.fd, recoveryName, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | (constants.O_NOFOLLOW || 0), 0o600);
|
|
474
|
+
}
|
|
475
|
+
const recoveryBytes = Buffer.from(JSON.stringify({ pid: process.pid, hostname: hostname(), token: randomUUID(),
|
|
476
|
+
...(hostBootIdentity ? { kind: 'darwin_journal_recovery_flock_v1', darwin_host_boot_identity: hostBootIdentity } : {}),
|
|
477
|
+
...(bootMarker ? { host_boot_marker: bootMarker } : {}) }));
|
|
433
478
|
writeSync(recoveryFd, recoveryBytes, 0, recoveryBytes.length);
|
|
434
479
|
fsyncSync(recoveryFd);
|
|
435
480
|
} catch {
|
|
436
481
|
if (recoveryFd !== undefined) closeSync(recoveryFd);
|
|
437
482
|
return false;
|
|
438
483
|
}
|
|
439
|
-
const quarantine = `${
|
|
484
|
+
const quarantine = `${name}.stale.${process.pid}.${Date.now()}`;
|
|
440
485
|
try {
|
|
441
|
-
const current =
|
|
442
|
-
const currentOwner = readLockOwner(
|
|
486
|
+
const current = lstatAtSync(directory.fd, name);
|
|
487
|
+
const currentOwner = readLockOwner(directory, name);
|
|
443
488
|
if (!sameIdentity(stat, current) || currentOwner?.token !== owner?.token) return false;
|
|
444
|
-
|
|
445
|
-
const quarantined =
|
|
446
|
-
const quarantinedOwner = readLockOwner(quarantine);
|
|
489
|
+
renameAtSync(directory.fd, name, directory.fd, quarantine);
|
|
490
|
+
const quarantined = lstatAtSync(directory.fd, quarantine);
|
|
491
|
+
const quarantinedOwner = readLockOwner(directory, quarantine);
|
|
447
492
|
if (!sameIdentity(stat, quarantined) || quarantinedOwner?.token !== owner?.token) {
|
|
448
|
-
try {
|
|
493
|
+
try { lstatAtSync(directory.fd, name); }
|
|
494
|
+
catch (error) {
|
|
495
|
+
if (error?.code === 'ENOENT') {
|
|
496
|
+
try { renameAtSync(directory.fd, quarantine, directory.fd, name); } catch { /* preserve quarantine fail-closed */ }
|
|
497
|
+
}
|
|
498
|
+
}
|
|
449
499
|
return false;
|
|
450
500
|
}
|
|
451
|
-
|
|
501
|
+
unlinkAtSync(directory.fd, quarantine);
|
|
452
502
|
return true;
|
|
453
503
|
} catch (error) {
|
|
454
504
|
if (error?.code === 'ENOENT') return true;
|
|
455
505
|
return false;
|
|
456
506
|
} finally {
|
|
457
507
|
if (recoveryFd !== undefined) closeSync(recoveryFd);
|
|
458
|
-
|
|
508
|
+
if (!hostBootIdentity) {
|
|
509
|
+
try { unlinkAtSync(directory.fd, recoveryName); } catch { /* fail closed if another process changed it */ }
|
|
510
|
+
}
|
|
459
511
|
}
|
|
460
512
|
}
|
|
461
513
|
|
|
462
514
|
function parseJournal(journalPath, { repair = false, directory = null } = {}) {
|
|
463
|
-
if (!directory)
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
515
|
+
if (!directory) {
|
|
516
|
+
const openedDirectory = openDirectoryForSync(dirname(journalPath));
|
|
517
|
+
try { return parseJournal(journalPath, { repair, directory: openedDirectory }); }
|
|
518
|
+
finally { closeSync(openedDirectory.fd); }
|
|
519
|
+
}
|
|
520
|
+
const journalName = basename(journalPath);
|
|
521
|
+
const journal = readRegularBoundedRecord(directory, journalName, { allowMissing: true });
|
|
468
522
|
const rawJournal = journal?.content ?? null;
|
|
469
523
|
const raw = rawJournal ?? '';
|
|
470
524
|
const hasTail = Boolean(raw && !raw.endsWith('\n'));
|
|
@@ -499,10 +553,7 @@ function parseJournal(journalPath, { repair = false, directory = null } = {}) {
|
|
|
499
553
|
completeTail = true;
|
|
500
554
|
} catch { /* the durable checkpoint decides whether partial bytes are repairable */ }
|
|
501
555
|
}
|
|
502
|
-
const
|
|
503
|
-
? join(directory.reference, basename(checkpointPath(journalPath)))
|
|
504
|
-
: checkpointPath(journalPath);
|
|
505
|
-
const rawCheckpoint = readRegularBounded(checkpointFile, {
|
|
556
|
+
const rawCheckpoint = readRegularBounded(directory, basename(checkpointPath(journalPath)), {
|
|
506
557
|
allowMissing: true, label: 'journal checkpoint', maxBytes: JOURNAL_RECORD_MAX_BYTES,
|
|
507
558
|
});
|
|
508
559
|
let checkpoint = null;
|
|
@@ -546,7 +597,7 @@ function parseJournal(journalPath, { repair = false, directory = null } = {}) {
|
|
|
546
597
|
}
|
|
547
598
|
let identity = journal?.stat ?? null;
|
|
548
599
|
if ((safeIncompleteTail || missingDelimiter) && repair) {
|
|
549
|
-
const fd =
|
|
600
|
+
const fd = openAtSync(directory.fd, journalName, constants.O_RDWR | (constants.O_NOFOLLOW || 0) | (constants.O_NONBLOCK || 0));
|
|
550
601
|
try {
|
|
551
602
|
const before = fstatSync(fd);
|
|
552
603
|
if (!before.isFile() || before.nlink !== 1) fail('UNSAFE_JOURNAL_FILE', 'journal must be a regular single-link file');
|
|
@@ -574,11 +625,7 @@ function parseJournal(journalPath, { repair = false, directory = null } = {}) {
|
|
|
574
625
|
}
|
|
575
626
|
identity = afterRead;
|
|
576
627
|
} finally { closeSync(fd); }
|
|
577
|
-
|
|
578
|
-
else {
|
|
579
|
-
const openedDirectory = openDirectoryForSync(dirname(journalPath));
|
|
580
|
-
try { fsyncSync(openedDirectory.fd); } finally { closeSync(openedDirectory.fd); }
|
|
581
|
-
}
|
|
628
|
+
fsyncSync(directory.fd);
|
|
582
629
|
}
|
|
583
630
|
if (rawJournal === null && checkpoint !== null && !validCommittedState && !validPendingState) {
|
|
584
631
|
fail('JOURNAL_TRUNCATION_DETECTED', 'journal file is missing below its durable high-water mark');
|
|
@@ -590,6 +637,12 @@ function isExpired(claim, now) {
|
|
|
590
637
|
return Date.parse(claim.lease.expires_at) <= Date.parse(now);
|
|
591
638
|
}
|
|
592
639
|
|
|
640
|
+
export function attemptJournalRevision(records, attemptId) {
|
|
641
|
+
return [...records].reverse().find((record) => record.payload?.attempt_id === attemptId
|
|
642
|
+
|| record.payload?.attempt?.attempt_id === attemptId
|
|
643
|
+
|| record.payload?.completion?.attempt_id === attemptId)?.event_id;
|
|
644
|
+
}
|
|
645
|
+
|
|
593
646
|
function claimProcessPid(attempt) {
|
|
594
647
|
const match = /^process:\/\/chati\/(\d+)$/.exec(attempt.claim.harness_process_id || '');
|
|
595
648
|
if (!match) fail('RECOVERY_PROCESS_UNRESOLVED', 'claim process identity is not locally observable');
|
|
@@ -597,10 +650,15 @@ function claimProcessPid(attempt) {
|
|
|
597
650
|
}
|
|
598
651
|
|
|
599
652
|
function observeStoppedClaimProcess(attempt) {
|
|
653
|
+
const pid = claimProcessPid(attempt);
|
|
654
|
+
if (attempt.claim.darwin_host_boot_identity) {
|
|
655
|
+
const boot = compareDarwinHostBootIdentity(attempt.claim.darwin_host_boot_identity, captureDarwinHostBootIdentity());
|
|
656
|
+
// Do not probe a numeric PID from an earlier boot, even if now reused.
|
|
657
|
+
if (boot === 'previous-boot') return pid;
|
|
658
|
+
}
|
|
600
659
|
if (attempt.claim.hostname !== hostname()) {
|
|
601
660
|
fail('RECOVERY_REMOTE_PROCESS_UNVERIFIABLE', 'automatic recovery cannot prove a process stopped on another host');
|
|
602
661
|
}
|
|
603
|
-
const pid = claimProcessPid(attempt);
|
|
604
662
|
try {
|
|
605
663
|
process.kill(pid, 0);
|
|
606
664
|
fail('RECOVERY_PROCESS_STILL_RUNNING', 'the prior claim process is still running', { pid });
|
|
@@ -625,6 +683,16 @@ function verifyCommitDescendsFrom({ git_dir, base_commit_ref, result_commit_ref
|
|
|
625
683
|
}
|
|
626
684
|
}
|
|
627
685
|
|
|
686
|
+
/** Legacy attempts retain their original base; new bases are bound at claim. */
|
|
687
|
+
export function attemptExecutionBase(attempt) {
|
|
688
|
+
const base = attempt.execution_base_commit_ref === undefined
|
|
689
|
+
? attempt.git_commit_ref : attempt.execution_base_commit_ref;
|
|
690
|
+
if (typeof base !== 'string' || !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(base)) {
|
|
691
|
+
fail('INVALID_ATTEMPT_EXECUTION_BASE', 'attempt execution base must be a full Git object ID');
|
|
692
|
+
}
|
|
693
|
+
return base;
|
|
694
|
+
}
|
|
695
|
+
|
|
628
696
|
/**
|
|
629
697
|
* C-04/C-06 local engine. Every mutable operational fact is represented by a
|
|
630
698
|
* new journal event. The journal itself is canonical for this package.
|
|
@@ -636,8 +704,10 @@ export class RailEngine {
|
|
|
636
704
|
#activeDirectory;
|
|
637
705
|
#journalIdentity;
|
|
638
706
|
#authorityVerifier;
|
|
707
|
+
#checkpointWriter;
|
|
708
|
+
#restorationWriter;
|
|
639
709
|
|
|
640
|
-
constructor({ journal_path, clock, authority_verifier = null } = {}) {
|
|
710
|
+
constructor({ journal_path, clock, authority_verifier = null, execution_checkpoint_writer = null, execution_restoration_writer = null } = {}) {
|
|
641
711
|
assertString(journal_path, 'INVALID_JOURNAL_PATH', 'journal_path');
|
|
642
712
|
this.#journalPath = journal_path;
|
|
643
713
|
this.#clock = clock;
|
|
@@ -645,6 +715,14 @@ export class RailEngine {
|
|
|
645
715
|
fail('INVALID_RAIL_AUTHORITY_VERIFIER', 'authority_verifier must be a function');
|
|
646
716
|
}
|
|
647
717
|
this.#authorityVerifier = authority_verifier;
|
|
718
|
+
if (execution_checkpoint_writer !== null && typeof execution_checkpoint_writer !== 'function') {
|
|
719
|
+
fail('INVALID_CHECKPOINT_WRITER', 'checkpoint writer must be an explicitly configured host adapter');
|
|
720
|
+
}
|
|
721
|
+
this.#checkpointWriter = execution_checkpoint_writer;
|
|
722
|
+
if (execution_restoration_writer !== null && typeof execution_restoration_writer !== 'function') {
|
|
723
|
+
fail('INVALID_RESTORATION_WRITER', 'restoration writer must be an explicitly configured host adapter');
|
|
724
|
+
}
|
|
725
|
+
this.#restorationWriter = execution_restoration_writer;
|
|
648
726
|
const loaded = parseJournal(journal_path);
|
|
649
727
|
this.#records = loaded.records;
|
|
650
728
|
this.#journalIdentity = loaded.identity;
|
|
@@ -653,12 +731,89 @@ export class RailEngine {
|
|
|
653
731
|
get journalPath() { return this.#journalPath; }
|
|
654
732
|
get records() { return Object.freeze([...this.#records]); }
|
|
655
733
|
|
|
656
|
-
#verifyAuthority(kind, receipt, expected) {
|
|
657
|
-
if (!this.#authorityVerifier || this.#authorityVerifier({ kind, receipt, expected }) !== true) {
|
|
734
|
+
#verifyAuthority(kind, receipt, expected, context) {
|
|
735
|
+
if (!this.#authorityVerifier || this.#authorityVerifier({ kind, receipt, expected, ...(context ? { context } : {}) }) !== true) {
|
|
658
736
|
fail('UNTRUSTED_RAIL_AUTHORITY_RECEIPT', `${kind} requires a receipt from the configured terminal authority`);
|
|
659
737
|
}
|
|
660
738
|
}
|
|
661
739
|
|
|
740
|
+
#assertAttemptRevision(attemptId, expected) {
|
|
741
|
+
if (expected === undefined) return;
|
|
742
|
+
assertString(expected, 'INVALID_RECOVERY_CONTEXT', 'expected_attempt_event_id');
|
|
743
|
+
if (attemptJournalRevision(this.#records, attemptId) !== expected) {
|
|
744
|
+
fail('RECOVERY_CONTEXT_CHANGED', 'attempt changed after recovery stage selection; reload its canonical state');
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
#verifyExecutionAdmission({ attempt, invocation, receipt, project_dir, harness_process_id }) {
|
|
749
|
+
assertString(project_dir, 'INVALID_EXECUTION_ADMISSION', 'project_dir');
|
|
750
|
+
assertObject(receipt, 'INVALID_EXECUTION_ADMISSION', 'admission receipt');
|
|
751
|
+
const base = this.#executionEvidence(attempt.attempt_id, { successfulOnly: true })
|
|
752
|
+
.at(-1)?.result_commit_ref ?? attemptExecutionBase(attempt);
|
|
753
|
+
this.#verifyAuthority('execution_admission', receipt, {
|
|
754
|
+
purpose: 'terminal_execution_admission', authority_kind: 'rail_execution',
|
|
755
|
+
execution_admission_protocol: 'native-positive-go-v1',
|
|
756
|
+
invocation_id: invocation.invocation_id, launch_nonce: invocation.launch_nonce,
|
|
757
|
+
task_id: attempt.task_id, working_dir: resolve(project_dir), prompt_digest: invocation.prompt_digest,
|
|
758
|
+
provider_id: invocation.provider_id, harness_id: invocation.harness_id,
|
|
759
|
+
model_id: invocation.model_pin?.model_id,
|
|
760
|
+
reasoning_configuration: invocation.model_pin?.reasoning_configuration,
|
|
761
|
+
supervisor_process_id: harness_process_id, supervisor_hostname: attempt.claim.hostname,
|
|
762
|
+
execution_base_commit_ref: base, execution_launch_head_ref: base, execution_worktree_clean: true,
|
|
763
|
+
});
|
|
764
|
+
if (attempt.claim.darwin_host_boot_identity) {
|
|
765
|
+
assertDarwinHostBootIdentity(receipt.darwin_host_boot_identity);
|
|
766
|
+
if (sha256(receipt.darwin_host_boot_identity) !== sha256(attempt.claim.darwin_host_boot_identity)) {
|
|
767
|
+
fail('EXECUTION_ADMISSION_HOST_BOOT_MISMATCH', 'admission must belong to the exact native supervisor host and boot');
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
if (!/^[a-f0-9]{64}$/.test(receipt.execution_admission_helper_sha256 || '')
|
|
771
|
+
|| !/^[a-f0-9]{64}$/.test(receipt.execution_repository_identity_digest || '')
|
|
772
|
+
|| !/^[a-f0-9]{64}$/.test(receipt.execution_preexisting_commits_digest || '')
|
|
773
|
+
|| typeof receipt.terminal_id !== 'string' || !receipt.terminal_id) {
|
|
774
|
+
fail('INVALID_EXECUTION_ADMISSION', 'execution admission requires the pinned helper and exact repository baseline');
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
#launchReconciled(launchNonce) {
|
|
779
|
+
return this.#records.some((record) => record.type === 'attempt_recovered'
|
|
780
|
+
&& (record.payload?.reconciliation?.execution_admissions?.some((entry) =>
|
|
781
|
+
entry.status === 'not_started' && entry.launch_nonce === launchNonce)
|
|
782
|
+
|| record.payload?.reconciliation?.execution_dispatches?.some((entry) =>
|
|
783
|
+
entry.status === 'interrupted_restored' && entry.launch_nonce === launchNonce)));
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
#restorationProof(dispatch, options = {}) {
|
|
787
|
+
return restoredExecutionProof(this.#records, dispatch, { ...options,
|
|
788
|
+
verifyAuthority: (kind, receipt, expected, context) => this.#verifyAuthority(kind, receipt, expected, context) });
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
#assertCompletionNotRestored(dispatch) {
|
|
792
|
+
if (this.#records.some((record) => ['execution_checkpoint_preserved', 'execution_worktree_restored'].includes(record.type)
|
|
793
|
+
&& record.payload.dispatch_event_id === dispatch.event_id)) {
|
|
794
|
+
fail('EXECUTION_COMPLETION_AFTER_CHECKPOINT', 'checkpointed interruption cannot be replaced by a late completion');
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
#assertAdmissionDispatchBound(reservation, receipt) {
|
|
799
|
+
const admission = reservation.payload.admission_receipt;
|
|
800
|
+
if (receipt.process_containment_kind === 'darwin_scoped_process_group_v1' && !admission) {
|
|
801
|
+
fail('EXECUTION_ADMISSION_DISPATCH_MISMATCH', 'Darwin dispatch requires its canonical pre-execution admission, not only a digest label');
|
|
802
|
+
}
|
|
803
|
+
if (receipt.process_containment_kind === 'darwin_scoped_process_group_v1' && admission) {
|
|
804
|
+
assertDarwinHostBootIdentity(admission.darwin_host_boot_identity);
|
|
805
|
+
if (sha256(receipt.darwin_launch_binding.host_boot_identity) !== sha256(admission.darwin_host_boot_identity)) {
|
|
806
|
+
fail('EXECUTION_ADMISSION_DISPATCH_MISMATCH', 'Darwin dispatch must bind its admission host and boot');
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
if (admission && (receipt.execution_admission_receipt_digest !== sha256(admission)
|
|
810
|
+
|| receipt.execution_admission_protocol !== admission.execution_admission_protocol
|
|
811
|
+
|| receipt.execution_admission_helper_sha256 !== admission.execution_admission_helper_sha256
|
|
812
|
+
|| receipt.terminal_id !== admission.terminal_id)) {
|
|
813
|
+
fail('EXECUTION_ADMISSION_DISPATCH_MISMATCH', 'dispatch does not bind its exact canonical pre-execution admission');
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
|
|
662
817
|
#assertDirectoryStable() {
|
|
663
818
|
if (!this.#activeDirectory) fail('UNSAFE_JOURNAL_DIRECTORY', 'journal mutation requires a pinned parent directory');
|
|
664
819
|
let canonical;
|
|
@@ -683,10 +838,16 @@ export class RailEngine {
|
|
|
683
838
|
if (!this.#activeDirectory) {
|
|
684
839
|
fail('UNSAFE_JOURNAL_DIRECTORY', 'journal parent changed during mutation');
|
|
685
840
|
}
|
|
686
|
-
return
|
|
841
|
+
return basename(path);
|
|
687
842
|
}
|
|
688
843
|
|
|
689
844
|
#exclusive(action) {
|
|
845
|
+
// Capture before creating a lock: a failed native observation must not leave
|
|
846
|
+
// a newly created ownerless lock behind.
|
|
847
|
+
const hostBootIdentity = process.platform === 'darwin' ? captureDarwinHostBootIdentity() : null;
|
|
848
|
+
// Skip on Darwin: its identity above is already authoritative, and capturing
|
|
849
|
+
// a marker there would cost a second native host observation for nothing.
|
|
850
|
+
const hostBootMarker = hostBootIdentity ? null : captureHostBootMarker();
|
|
690
851
|
const directory = openDirectoryForSync(dirname(this.#journalPath), { create: true });
|
|
691
852
|
this.#activeDirectory = directory;
|
|
692
853
|
const path = this.#activeEntry(lockPath(this.#journalPath));
|
|
@@ -695,9 +856,11 @@ export class RailEngine {
|
|
|
695
856
|
try {
|
|
696
857
|
for (let retry = 0; retry < LOCK_RETRIES; retry += 1) {
|
|
697
858
|
try {
|
|
698
|
-
descriptor =
|
|
859
|
+
descriptor = openAtSync(directory.fd, path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | (constants.O_NOFOLLOW || 0), 0o600);
|
|
699
860
|
lockToken = `${process.pid}-${Date.now()}-${randomUUID()}`;
|
|
700
|
-
const ownerBytes = Buffer.from(JSON.stringify({ schema_version: 1, pid: process.pid, hostname: hostname(), token: lockToken,
|
|
861
|
+
const ownerBytes = Buffer.from(JSON.stringify({ schema_version: 1, pid: process.pid, hostname: hostname(), token: lockToken,
|
|
862
|
+
acquired_at: new Date().toISOString(), ...(hostBootIdentity ? { darwin_host_boot_identity: hostBootIdentity } : {}),
|
|
863
|
+
...(hostBootMarker ? { host_boot_marker: hostBootMarker } : {}) }));
|
|
701
864
|
writeSync(descriptor, ownerBytes, 0, ownerBytes.length);
|
|
702
865
|
fsyncSync(descriptor);
|
|
703
866
|
fsyncSync(directory.fd);
|
|
@@ -705,7 +868,7 @@ export class RailEngine {
|
|
|
705
868
|
} catch (error) {
|
|
706
869
|
if (descriptor !== undefined) { closeSync(descriptor); descriptor = undefined; }
|
|
707
870
|
if (error?.code !== 'EEXIST') throw error;
|
|
708
|
-
if (recoverStaleLock(path)) continue;
|
|
871
|
+
if (recoverStaleLock(directory, path, hostBootIdentity)) continue;
|
|
709
872
|
Atomics.wait(LOCK_WAIT, 0, 0, 2);
|
|
710
873
|
}
|
|
711
874
|
}
|
|
@@ -717,7 +880,7 @@ export class RailEngine {
|
|
|
717
880
|
} finally {
|
|
718
881
|
if (descriptor !== undefined) {
|
|
719
882
|
closeSync(descriptor);
|
|
720
|
-
if (readLockOwner(path)?.token === lockToken) {
|
|
883
|
+
if (readLockOwner(directory, path)?.token === lockToken) { unlinkAtSync(directory.fd, path); fsyncSync(directory.fd); }
|
|
721
884
|
}
|
|
722
885
|
this.#activeDirectory = undefined;
|
|
723
886
|
closeSync(directory.fd);
|
|
@@ -734,7 +897,7 @@ export class RailEngine {
|
|
|
734
897
|
const temporary = `${target}.${process.pid}.${randomUUID()}.tmp`;
|
|
735
898
|
let fd;
|
|
736
899
|
try {
|
|
737
|
-
fd =
|
|
900
|
+
fd = openAtSync(this.#activeDirectory.fd, temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | (constants.O_NOFOLLOW || 0), 0o600);
|
|
738
901
|
const bytes = Buffer.from(JSON.stringify(checkpoint));
|
|
739
902
|
if (bytes.length > JOURNAL_RECORD_MAX_BYTES) fail('JOURNAL_SIZE_LIMIT', 'journal checkpoint exceeds the record safety limit');
|
|
740
903
|
writeSync(fd, bytes, 0, bytes.length);
|
|
@@ -742,11 +905,11 @@ export class RailEngine {
|
|
|
742
905
|
closeSync(fd);
|
|
743
906
|
fd = undefined;
|
|
744
907
|
this.#assertDirectoryStable();
|
|
745
|
-
|
|
908
|
+
renameAtSync(this.#activeDirectory.fd, temporary, this.#activeDirectory.fd, target);
|
|
746
909
|
fsyncSync(this.#activeDirectory.fd);
|
|
747
910
|
} finally {
|
|
748
911
|
if (fd !== undefined) closeSync(fd);
|
|
749
|
-
try {
|
|
912
|
+
try { unlinkAtSync(this.#activeDirectory.fd, temporary); } catch { /* preserve primary failure */ }
|
|
750
913
|
}
|
|
751
914
|
}
|
|
752
915
|
|
|
@@ -803,7 +966,7 @@ export class RailEngine {
|
|
|
803
966
|
const existed = this.#journalIdentity !== null;
|
|
804
967
|
let fd;
|
|
805
968
|
try {
|
|
806
|
-
fd =
|
|
969
|
+
fd = openAtSync(this.#activeDirectory.fd, this.#activeEntry(this.#journalPath), constants.O_WRONLY | constants.O_APPEND
|
|
807
970
|
| (existed ? 0 : constants.O_CREAT | constants.O_EXCL)
|
|
808
971
|
| (constants.O_NOFOLLOW || 0), 0o600);
|
|
809
972
|
} catch (error) {
|
|
@@ -931,6 +1094,7 @@ export class RailEngine {
|
|
|
931
1094
|
|
|
932
1095
|
reserveExecutionLaunch({
|
|
933
1096
|
attempt_id, claim_id, operator_id, harness_process_id, invocation,
|
|
1097
|
+
admission_receipt = null, project_dir = null,
|
|
934
1098
|
} = {}) {
|
|
935
1099
|
return this.#exclusive(() => {
|
|
936
1100
|
for (const [field, value] of Object.entries({
|
|
@@ -940,6 +1104,7 @@ export class RailEngine {
|
|
|
940
1104
|
assertString(invocation.invocation_id, 'INVALID_EXECUTION_LAUNCH_RESERVATION', 'invocation.invocation_id');
|
|
941
1105
|
assertString(invocation.launch_nonce, 'INVALID_EXECUTION_LAUNCH_RESERVATION', 'invocation.launch_nonce');
|
|
942
1106
|
assertString(invocation.prompt_digest, 'INVALID_EXECUTION_LAUNCH_RESERVATION', 'invocation.prompt_digest');
|
|
1107
|
+
if (this.#launchReconciled(invocation.launch_nonce)) fail('EXECUTION_LAUNCH_ALREADY_RECONCILED', 'reconciled launch nonce may not be reused');
|
|
943
1108
|
const attempt = this.#attempts().get(attempt_id);
|
|
944
1109
|
if (!attempt || attempt.state !== EXECUTABLE_ATTEMPT_STATE) {
|
|
945
1110
|
fail('ATTEMPT_NOT_ACTIVE', 'execution launch reservation requires an active attempt', { attempt_id });
|
|
@@ -963,17 +1128,22 @@ export class RailEngine {
|
|
|
963
1128
|
|| invocation.model_pin?.reasoning_configuration !== binding.reasoning_configuration) {
|
|
964
1129
|
fail('EXECUTION_BINDING_MISMATCH', 'execution launch reservation does not match the sealed task binding');
|
|
965
1130
|
}
|
|
1131
|
+
if (admission_receipt !== null) this.#verifyExecutionAdmission({
|
|
1132
|
+
attempt, invocation, receipt: admission_receipt, project_dir, harness_process_id,
|
|
1133
|
+
});
|
|
966
1134
|
const existing = this.#records.find((record) => record.type === 'execution_launch_reserved'
|
|
967
1135
|
&& record.payload?.invocation?.launch_nonce === invocation.launch_nonce);
|
|
968
1136
|
if (existing) {
|
|
969
1137
|
if (sha256(existing.payload.invocation) !== sha256(invocation)
|
|
970
|
-
|| existing.payload.claim_event_id !== attempt.claim_event_id
|
|
1138
|
+
|| existing.payload.claim_event_id !== attempt.claim_event_id
|
|
1139
|
+
|| sha256(existing.payload.admission_receipt ?? null) !== sha256(admission_receipt)) {
|
|
971
1140
|
fail('EXECUTION_LAUNCH_NONCE_CONFLICT', 'launch nonce is already reserved for different execution material');
|
|
972
1141
|
}
|
|
973
1142
|
return Object.freeze(canonicalize({ ...existing.payload, event_id: existing.event_id, replay: true }));
|
|
974
1143
|
}
|
|
975
1144
|
const unresolvedReservation = this.#records.find((record) => record.type === 'execution_launch_reserved'
|
|
976
1145
|
&& record.payload?.attempt_id === attempt_id
|
|
1146
|
+
&& !this.#launchReconciled(record.payload.invocation.launch_nonce)
|
|
977
1147
|
&& !this.#records.some((candidate) => candidate.type === 'execution_dispatched'
|
|
978
1148
|
&& candidate.payload?.invocation?.launch_nonce === record.payload.invocation.launch_nonce));
|
|
979
1149
|
if (unresolvedReservation) {
|
|
@@ -982,6 +1152,7 @@ export class RailEngine {
|
|
|
982
1152
|
const unresolvedDispatch = this.#records.find((record) => record.type === 'execution_dispatched'
|
|
983
1153
|
&& record.payload?.attempt_id === attempt_id
|
|
984
1154
|
&& record.payload?.run_kind === 'initial'
|
|
1155
|
+
&& !this.#launchReconciled(record.payload.invocation.launch_nonce)
|
|
985
1156
|
&& !this.#records.some((candidate) => candidate.type === 'execution_recorded'
|
|
986
1157
|
&& candidate.payload?.execution_evidence?.receipt_nonce === record.payload.invocation.launch_nonce));
|
|
987
1158
|
if (unresolvedDispatch) {
|
|
@@ -990,6 +1161,7 @@ export class RailEngine {
|
|
|
990
1161
|
const payload = canonicalize({
|
|
991
1162
|
attempt_id, claim_id, operator_id, harness_process_id,
|
|
992
1163
|
claim_event_id: attempt.claim_event_id, invocation,
|
|
1164
|
+
...(admission_receipt ? { admission_receipt } : {}),
|
|
993
1165
|
reserved_at: timestamp(this.#clock),
|
|
994
1166
|
});
|
|
995
1167
|
const record = this.#append('execution_launch_reserved', payload, attempt.claim_event_id);
|
|
@@ -1008,6 +1180,7 @@ export class RailEngine {
|
|
|
1008
1180
|
assertObject(invocation, 'INVALID_EXECUTION_DISPATCH', 'invocation');
|
|
1009
1181
|
assertObject(launch_receipt, 'INVALID_EXECUTION_DISPATCH', 'launch_receipt');
|
|
1010
1182
|
assertString(invocation.launch_nonce, 'INVALID_EXECUTION_DISPATCH', 'invocation.launch_nonce');
|
|
1183
|
+
if (this.#launchReconciled(invocation.launch_nonce)) fail('EXECUTION_LAUNCH_ALREADY_RECONCILED', 'reconciled launch nonce may not dispatch');
|
|
1011
1184
|
assertString(invocation.prompt_digest, 'INVALID_EXECUTION_DISPATCH', 'invocation.prompt_digest');
|
|
1012
1185
|
assertString(project_dir, 'INVALID_EXECUTION_DISPATCH', 'project_dir');
|
|
1013
1186
|
if (browser_manifest !== null) assertObject(browser_manifest, 'INVALID_EXECUTION_DISPATCH', 'browser_manifest');
|
|
@@ -1065,10 +1238,12 @@ export class RailEngine {
|
|
|
1065
1238
|
|| sha256(launchReservation.payload.invocation) !== sha256(invocation)) {
|
|
1066
1239
|
fail('RAIL_REWORK_LAUNCH_RESERVATION_REQUIRED', 'rework dispatch requires its exact canonical pre-launch intent');
|
|
1067
1240
|
}
|
|
1241
|
+
this.#assertAdmissionDispatchBound(launchReservation, launch_receipt);
|
|
1068
1242
|
const priorDispatches = this.#records.filter((record) => record.type === 'execution_dispatched'
|
|
1069
1243
|
&& record.payload?.attempt_id === attempt_id
|
|
1070
1244
|
&& record.payload?.invocation?.rework_reservation_id === rework_reservation_id);
|
|
1071
|
-
const unreceipted = priorDispatches.find((dispatch) => !this.#
|
|
1245
|
+
const unreceipted = priorDispatches.find((dispatch) => !this.#launchReconciled(dispatch.payload.invocation.launch_nonce)
|
|
1246
|
+
&& !this.#records.some((record) => record.type === 'execution_recorded'
|
|
1072
1247
|
&& record.payload?.execution_evidence?.receipt_nonce === dispatch.payload.invocation.launch_nonce));
|
|
1073
1248
|
if (unreceipted) {
|
|
1074
1249
|
fail('RAIL_REWORK_DISPATCH_UNRESOLVED', 'a prior launch in this rework reservation has no canonical completion evidence');
|
|
@@ -1085,7 +1260,8 @@ export class RailEngine {
|
|
|
1085
1260
|
fail('EXECUTION_BINDING_MISMATCH', 'execution dispatch does not match the sealed task provider, harness, model and reasoning binding');
|
|
1086
1261
|
}
|
|
1087
1262
|
const expectedExecutionBase = this.#executionEvidence(attempt_id, { successfulOnly: true })
|
|
1088
|
-
.at(-1)?.result_commit_ref ?? attempt
|
|
1263
|
+
.at(-1)?.result_commit_ref ?? attemptExecutionBase(attempt);
|
|
1264
|
+
assertTerminalLaunchContainment(launch_receipt, 'EXECUTION_LAUNCH_RECEIPT_MISMATCH');
|
|
1089
1265
|
if (launch_receipt.purpose !== 'terminal_launch'
|
|
1090
1266
|
|| launch_receipt.authority_kind !== 'rail_execution'
|
|
1091
1267
|
|| launch_receipt.invocation_id !== invocation.invocation_id
|
|
@@ -1107,12 +1283,18 @@ export class RailEngine {
|
|
|
1107
1283
|
|| typeof launch_receipt.process_hostname !== 'string' || !launch_receipt.process_hostname
|
|
1108
1284
|
|| !(launch_receipt.process_group_id === null
|
|
1109
1285
|
|| (Number.isInteger(launch_receipt.process_group_id) && launch_receipt.process_group_id > 0))
|
|
1110
|
-
|| launch_receipt.process_containment_kind !== 'bubblewrap_pid_namespace_v1'
|
|
1111
1286
|
|| typeof launch_receipt.process_containment_id !== 'string' || !launch_receipt.process_containment_id
|
|
1112
1287
|
|| !/^[a-f0-9]{64}$/.test(launch_receipt.process_containment_digest || '')
|
|
1113
1288
|
|| typeof launch_receipt.authority_signature !== 'string') {
|
|
1114
1289
|
fail('EXECUTION_LAUNCH_RECEIPT_MISMATCH', 'launch receipt does not echo the exact canonical invocation');
|
|
1115
1290
|
}
|
|
1291
|
+
if (launch_receipt.process_namespace_identity !== undefined) {
|
|
1292
|
+
assertLinuxNamespaceIdentity(launch_receipt.process_namespace_identity);
|
|
1293
|
+
if (launch_receipt.process_namespace_identity.hostname !== launch_receipt.process_hostname
|
|
1294
|
+
|| launch_receipt.process_namespace_identity.pid === launch_receipt.process_pid) {
|
|
1295
|
+
fail('EXECUTION_LAUNCH_RECEIPT_MISMATCH', 'namespace init identity must bind the exact host and differ from the outer monitor');
|
|
1296
|
+
}
|
|
1297
|
+
}
|
|
1116
1298
|
const existing = this.#records.find((record) => record.type === 'execution_dispatched'
|
|
1117
1299
|
&& record.payload?.invocation?.launch_nonce === invocation.launch_nonce);
|
|
1118
1300
|
if (existing) {
|
|
@@ -1134,9 +1316,11 @@ export class RailEngine {
|
|
|
1134
1316
|
|| sha256(reservation.payload.invocation) !== sha256(invocation)) {
|
|
1135
1317
|
fail('RAIL_EXECUTION_LAUNCH_RESERVATION_REQUIRED', 'initial execution dispatch requires its exact canonical pre-launch reservation');
|
|
1136
1318
|
}
|
|
1319
|
+
this.#assertAdmissionDispatchBound(reservation, launch_receipt);
|
|
1137
1320
|
const unresolved = this.#records.find((record) => record.type === 'execution_dispatched'
|
|
1138
1321
|
&& record.payload?.attempt_id === attempt_id
|
|
1139
1322
|
&& record.payload?.run_kind === 'initial'
|
|
1323
|
+
&& !this.#launchReconciled(record.payload.invocation.launch_nonce)
|
|
1140
1324
|
&& !this.#records.some((candidate) => candidate.type === 'execution_recorded'
|
|
1141
1325
|
&& candidate.payload?.execution_evidence?.receipt_nonce === record.payload.invocation.launch_nonce));
|
|
1142
1326
|
if (unresolved) {
|
|
@@ -1153,12 +1337,345 @@ export class RailEngine {
|
|
|
1153
1337
|
});
|
|
1154
1338
|
}
|
|
1155
1339
|
|
|
1156
|
-
|
|
1340
|
+
recordHumanEffortObservation({ attempt_id, receipt } = {}) {
|
|
1341
|
+
return this.#exclusive(() => {
|
|
1342
|
+
const attempt = this.#attempts().get(attempt_id);
|
|
1343
|
+
if (!attempt) fail('RAIL_ATTEMPT_NOT_FOUND', 'human effort requires a canonical attempt');
|
|
1344
|
+
this.#verifyAuthority('human_effort', receipt, { purpose: 'rail_human_effort', attempt_id,
|
|
1345
|
+
handoff_id: attempt.handoff_id, task_id: attempt.task_id });
|
|
1346
|
+
if (!['opened', 'finished', 'interrupted'].includes(receipt.phase)) fail('INVALID_HUMAN_EFFORT', 'invalid human effort phase');
|
|
1347
|
+
const entries = this.#records.filter((record) => record.type === 'human_effort_observed'
|
|
1348
|
+
&& record.payload.receipt.session_id === receipt.session_id);
|
|
1349
|
+
const existing = entries.find((record) => (record.payload.receipt.phase === 'opened') === (receipt.phase === 'opened'));
|
|
1350
|
+
if (existing) {
|
|
1351
|
+
if (sha256(existing.payload.receipt) !== sha256(receipt)) fail('HUMAN_EFFORT_CONFLICT', 'human effort observation is immutable');
|
|
1352
|
+
return Object.freeze({ ...existing.payload, event_id: existing.event_id, replay: true });
|
|
1353
|
+
}
|
|
1354
|
+
if (receipt.phase === 'opened') {
|
|
1355
|
+
if (!['active', 'ready_to_merge'].includes(attempt.state)
|
|
1356
|
+
|| receipt.claim_id !== attempt.claim.claim_id || receipt.operator_id !== attempt.claim.operator_id) {
|
|
1357
|
+
fail('HUMAN_EFFORT_CLAIM_MISMATCH', 'new human effort session must bind the current attempt claim');
|
|
1358
|
+
}
|
|
1359
|
+
} else {
|
|
1360
|
+
const opened = entries.find((record) => record.payload.receipt.phase === 'opened')?.payload.receipt;
|
|
1361
|
+
if (!opened || receipt.opened_receipt_digest !== sha256(opened)
|
|
1362
|
+
|| ['operator_id', 'claim_id', 'account', 'project_binding'].some((key) => sha256(receipt[key]) !== sha256(opened[key]))) {
|
|
1363
|
+
fail('HUMAN_EFFORT_SESSION_MISMATCH', 'human effort result must bind its exact opened session');
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
const record = this.#append('human_effort_observed', { attempt_id, receipt });
|
|
1367
|
+
return Object.freeze({ ...record.payload, event_id: record.event_id, replay: false });
|
|
1368
|
+
});
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1371
|
+
humanEffortForAttempt(attempt_id) {
|
|
1372
|
+
return this.#exclusive(() => {
|
|
1373
|
+
const attempt = this.#attempts().get(attempt_id);
|
|
1374
|
+
if (!attempt) fail('RAIL_ATTEMPT_NOT_FOUND', 'human effort requires a canonical attempt');
|
|
1375
|
+
const entries = this.#records.filter((record) => record.type === 'human_effort_observed' && record.payload.attempt_id === attempt_id);
|
|
1376
|
+
const opened = entries.filter((record) => record.payload.receipt.phase === 'opened');
|
|
1377
|
+
if (!opened.length) fail('HUMAN_EFFORT_REQUIRED', 'human effort is unknown until the operator explicitly declares timed or zero effort');
|
|
1378
|
+
const sessions = opened.flatMap((record, index) => {
|
|
1379
|
+
const start = record.payload.receipt;
|
|
1380
|
+
const endings = entries.filter((entry) => entry.payload.receipt.session_id === start.session_id && entry.payload.receipt.phase !== 'opened');
|
|
1381
|
+
const isMostRecentOpened = index === opened.length - 1;
|
|
1382
|
+
if (endings.length !== 1 || endings[0].payload.receipt.phase !== 'finished') {
|
|
1383
|
+
// A session superseded by a later opened session is only excused when it
|
|
1384
|
+
// was explicitly marked interrupted (the operator's terminal dropped); an
|
|
1385
|
+
// ending-less session, or the latest session itself, never qualifies.
|
|
1386
|
+
const supersededInterruption = !isMostRecentOpened && endings.length === 1 && endings[0].payload.receipt.phase === 'interrupted';
|
|
1387
|
+
if (!supersededInterruption) {
|
|
1388
|
+
fail('HUMAN_EFFORT_INCOMPLETE', 'interrupted or unfinished human effort cannot be inferred from wall-clock timestamps');
|
|
1389
|
+
}
|
|
1390
|
+
return [];
|
|
1391
|
+
}
|
|
1392
|
+
const finish = endings[0].payload.receipt;
|
|
1393
|
+
const binding = { purpose: 'rail_human_effort', attempt_id, handoff_id: attempt.handoff_id, task_id: attempt.task_id };
|
|
1394
|
+
this.#verifyAuthority('human_effort', start, binding);
|
|
1395
|
+
this.#verifyAuthority('human_effort', finish, { ...binding, opened_receipt_digest: sha256(start),
|
|
1396
|
+
operator_id: start.operator_id, claim_id: start.claim_id, account: start.account, project_binding: start.project_binding });
|
|
1397
|
+
return [{ session_id: start.session_id, operator_id: start.operator_id, account: start.account,
|
|
1398
|
+
declaration: finish.declaration, value: finish.value,
|
|
1399
|
+
source_event_ids: [record.event_id, endings[0].event_id], receipt_digest: sha256(finish) }];
|
|
1400
|
+
});
|
|
1401
|
+
const value = sessions.reduce((total, session) => total + session.value, 0);
|
|
1402
|
+
if (!Number.isFinite(value)) fail('INVALID_HUMAN_EFFORT', 'human effort exceeds the finite measurement range');
|
|
1403
|
+
return Object.freeze(canonicalize({ schema_version: 1, evidence_class: 'operator-declared-timed-effort',
|
|
1404
|
+
attempt_id, unit: 'milliseconds', value, sessions }));
|
|
1405
|
+
});
|
|
1406
|
+
}
|
|
1407
|
+
|
|
1408
|
+
/** Records observation only, even after lease loss; grants no retry or recovery authority. */
|
|
1409
|
+
recordProviderProcessObservation({ attempt_id, terminal_authority_receipt, terminal_output } = {}) {
|
|
1410
|
+
return this.#exclusive(() => {
|
|
1411
|
+
assertString(attempt_id, 'INVALID_PROCESS_OBSERVATION', 'attempt_id');
|
|
1412
|
+
assertObject(terminal_authority_receipt, 'INVALID_PROCESS_OBSERVATION', 'terminal_authority_receipt');
|
|
1413
|
+
const dispatches = providerProcessDispatches(this.#records, attempt_id).filter((record) =>
|
|
1414
|
+
record.payload.invocation.launch_nonce === terminal_authority_receipt.launch_nonce);
|
|
1415
|
+
if (dispatches.length !== 1) fail('RAIL_PROCESS_DISPATCH_REQUIRED', 'process observation requires one exact canonical dispatch');
|
|
1416
|
+
const dispatch = dispatches[0];
|
|
1417
|
+
this.#assertCompletionNotRestored(dispatch);
|
|
1418
|
+
assertProviderProcessReceipt(dispatch, terminal_authority_receipt, (kind, receipt, expected) => this.#verifyAuthority(kind, receipt, expected));
|
|
1419
|
+
if (terminal_output !== undefined) assertProviderProcessOutput(terminal_output, terminal_authority_receipt);
|
|
1420
|
+
const existing = this.#records.find((record) => record.type === 'provider_process_observed'
|
|
1421
|
+
&& record.payload.attempt_id === attempt_id && record.payload.dispatch_event_id === dispatch.event_id);
|
|
1422
|
+
if (existing) {
|
|
1423
|
+
if (sha256(existing.payload.terminal_authority_receipt) !== sha256(terminal_authority_receipt)) {
|
|
1424
|
+
fail('RAIL_PROCESS_RECEIPT_CONFLICT', 'observed completion receipt is immutable');
|
|
1425
|
+
}
|
|
1426
|
+
if (terminal_output !== undefined && sha256(existing.payload.terminal_output ?? null) !== sha256(terminal_output)) {
|
|
1427
|
+
fail('RAIL_PROCESS_OUTPUT_CONFLICT', 'observed output is immutable; a legacy receipt-only record cannot be retroactively upgraded');
|
|
1428
|
+
}
|
|
1429
|
+
return Object.freeze({ ...existing.payload, event_id: existing.event_id, replay: true });
|
|
1430
|
+
}
|
|
1431
|
+
const payload = { attempt_id, dispatch_event_id: dispatch.event_id,
|
|
1432
|
+
invocation_id: dispatch.payload.invocation.invocation_id, launch_nonce: dispatch.payload.invocation.launch_nonce,
|
|
1433
|
+
terminal_authority_receipt, ...(terminal_output === undefined ? {} : { terminal_output }) };
|
|
1434
|
+
const record = this.#append('provider_process_observed', payload);
|
|
1435
|
+
return Object.freeze({ ...record.payload, event_id: record.event_id, replay: false });
|
|
1436
|
+
});
|
|
1437
|
+
}
|
|
1438
|
+
|
|
1439
|
+
providerProcessingForAttempt(attempt_id) {
|
|
1440
|
+
return this.#exclusive(() => {
|
|
1441
|
+
assertString(attempt_id, 'INVALID_PROCESS_OBSERVATION', 'attempt_id');
|
|
1442
|
+
if (!this.#attempts().has(attempt_id)) fail('RAIL_ATTEMPT_NOT_FOUND', 'processing evidence requires a canonical attempt');
|
|
1443
|
+
return collectProviderProcessing(this.#records, { attempt_id,
|
|
1444
|
+
verifyAuthority: (kind, receipt, expected, context) => this.#verifyAuthority(kind, receipt, expected, context) });
|
|
1445
|
+
});
|
|
1446
|
+
}
|
|
1447
|
+
|
|
1448
|
+
/** Observe only canonical interrupted dispatches, never caller-owned PIDs. */
|
|
1449
|
+
observeInterruptedExecution({ attempt_id, launch_nonce, git_dir, expected_attempt_event_id, termination_attestation } = {}) {
|
|
1450
|
+
return this.#exclusive(() => {
|
|
1451
|
+
this.#assertAttemptRevision(attempt_id, expected_attempt_event_id);
|
|
1452
|
+
for (const [field, value] of Object.entries({ attempt_id, launch_nonce, git_dir })) {
|
|
1453
|
+
assertString(value, 'INVALID_EXECUTION_RECONCILIATION', field);
|
|
1454
|
+
}
|
|
1455
|
+
const dispatch = this.#records.find((record) => record.type === 'execution_dispatched'
|
|
1456
|
+
&& record.payload.attempt_id === attempt_id && record.payload.invocation.launch_nonce === launch_nonce);
|
|
1457
|
+
if (!dispatch) fail('RECOVERY_EXECUTION_DISPATCH_REQUIRED', 'interruption requires an existing canonical execution dispatch');
|
|
1458
|
+
const { invocation, launch_receipt: launch } = dispatch.payload;
|
|
1459
|
+
this.#verifyAuthority('execution_launch', launch, {
|
|
1460
|
+
purpose: 'terminal_launch', authority_kind: 'rail_execution',
|
|
1461
|
+
invocation_id: invocation.invocation_id, launch_nonce, working_dir: resolve(git_dir),
|
|
1462
|
+
provider_id: invocation.provider_id, harness_id: invocation.harness_id,
|
|
1463
|
+
model_id: invocation.model_pin.model_id, reasoning_configuration: invocation.model_pin.reasoning_configuration,
|
|
1464
|
+
prompt_digest: invocation.prompt_digest,
|
|
1465
|
+
});
|
|
1466
|
+
if (this.#records.some((record) => (record.type === 'execution_recorded'
|
|
1467
|
+
&& record.payload?.execution_evidence?.receipt_nonce === launch_nonce)
|
|
1468
|
+
|| (record.type === 'provider_process_observed' && record.payload.dispatch_event_id === dispatch.event_id))) {
|
|
1469
|
+
fail('RECOVERY_EXECUTION_ALREADY_OBSERVED', 'canonical completion material must not be downgraded to an unknown interruption');
|
|
1470
|
+
}
|
|
1471
|
+
const existing = this.#records.find((record) => record.type === 'execution_interruption_observed'
|
|
1472
|
+
&& record.payload.dispatch_event_id === dispatch.event_id);
|
|
1473
|
+
if (existing) {
|
|
1474
|
+
assertExecutionInterruptionLifetime(dispatch, existing);
|
|
1475
|
+
return Object.freeze(canonicalize({ ...existing.payload, event_id: existing.event_id, replay: true }));
|
|
1476
|
+
}
|
|
1477
|
+
const attempt = this.#attempts().get(attempt_id);
|
|
1478
|
+
if (!attempt || attempt.state !== EXECUTABLE_ATTEMPT_STATE) fail('ATTEMPT_NOT_ACTIVE', 'interruption observation requires an active attempt');
|
|
1479
|
+
if (!isExpired(attempt.claim, timestamp(this.#clock))) fail('LEASE_STILL_ACTIVE', 'interruption observation cannot race a live lease');
|
|
1480
|
+
observeStoppedClaimProcess(attempt);
|
|
1481
|
+
const reservation = this.#records.find((record) => ['execution_launch_reserved', 'rework_launch_reserved'].includes(record.type)
|
|
1482
|
+
&& record.payload.attempt_id === attempt_id && record.payload.invocation.launch_nonce === launch_nonce);
|
|
1483
|
+
if (!reservation || reservation.payload.claim_event_id !== attempt.claim_event_id
|
|
1484
|
+
|| reservation.payload.harness_process_id !== attempt.claim.harness_process_id
|
|
1485
|
+
|| dispatch.payload.claim_id !== attempt.claim.claim_id
|
|
1486
|
+
|| dispatch.payload.operator_id !== attempt.claim.operator_id
|
|
1487
|
+
|| sha256(reservation.payload.invocation) !== sha256(invocation)) {
|
|
1488
|
+
fail('RECOVERY_ADMISSION_CLAIM_MISMATCH', 'interrupted dispatch must belong to the current stopped supervisor generation');
|
|
1489
|
+
}
|
|
1490
|
+
this.#verifyExecutionAdmission({ attempt, invocation, receipt: reservation.payload.admission_receipt,
|
|
1491
|
+
project_dir: git_dir, harness_process_id: attempt.claim.harness_process_id });
|
|
1492
|
+
this.#assertAdmissionDispatchBound(reservation, launch);
|
|
1493
|
+
const unresolved = [...this.#effects().values()].filter((effect) => effect.attempt_id === attempt_id && !effect.receipt_ref);
|
|
1494
|
+
if (unresolved.length) fail('RECOVERY_EFFECTS_UNRESOLVED', 'unreceipted effects require external reconciliation before interruption recovery', {
|
|
1495
|
+
idempotency_keys: unresolved.map((effect) => effect.idempotency_key),
|
|
1496
|
+
});
|
|
1497
|
+
let processObservation;
|
|
1498
|
+
let lifetimeBinding;
|
|
1499
|
+
if (launch.process_containment_kind === 'darwin_scoped_process_group_v1') {
|
|
1500
|
+
assertTerminalLaunchContainment(launch, 'RECOVERY_DARWIN_LIFETIME_UNPROVEN');
|
|
1501
|
+
try { processObservation = observeDarwinPreviousBoot(launch.darwin_launch_binding.host_boot_identity); }
|
|
1502
|
+
catch (error) {
|
|
1503
|
+
// A same-boot crash has no new-boot proof; an explicit human attestation
|
|
1504
|
+
// that no launch processes remain is only ever consulted here, after the
|
|
1505
|
+
// stronger reboot proof has already failed.
|
|
1506
|
+
if (termination_attestation === undefined) fail('RECOVERY_DARWIN_LIFETIME_UNPROVEN', error.message, { cause_code: error.code });
|
|
1507
|
+
if (!isWellFormedTerminationAttestation(termination_attestation)) {
|
|
1508
|
+
fail('RECOVERY_ATTESTATION_INVALID', 'termination attestation must declare exactly operator_id, attested_at and the no-launch-processes-remain statement');
|
|
1509
|
+
}
|
|
1510
|
+
if (termination_attestation.operator_id !== attempt.claim.operator_id) {
|
|
1511
|
+
fail('RECOVERY_ATTESTATION_OPERATOR_MISMATCH', 'termination attestation must be signed by the current claim operator');
|
|
1512
|
+
}
|
|
1513
|
+
processObservation = Object.freeze({ status: 'exited', reason: 'human-attested', signal_sent: false,
|
|
1514
|
+
host_boot_identity: captureDarwinHostBootIdentity(), attestation: termination_attestation });
|
|
1515
|
+
}
|
|
1516
|
+
lifetimeBinding = { darwin_host_boot_identity_digest: sha256(launch.darwin_launch_binding.host_boot_identity) };
|
|
1517
|
+
} else {
|
|
1518
|
+
if (termination_attestation !== undefined) {
|
|
1519
|
+
fail('RECOVERY_ATTESTATION_UNSUPPORTED', 'termination attestation is only supported for Darwin scoped process group launches');
|
|
1520
|
+
}
|
|
1521
|
+
if (!launch.process_namespace_identity) fail('RECOVERY_NAMESPACE_IDENTITY_REQUIRED', 'legacy dispatch has no authenticated namespace identity');
|
|
1522
|
+
assertLinuxNamespaceIdentity(launch.process_namespace_identity);
|
|
1523
|
+
// Authentication, claim/lease and effect checks precede any signal.
|
|
1524
|
+
// Linux matches a pidfd; Darwin's new-boot proof never signals a PID.
|
|
1525
|
+
processObservation = observeLinuxNamespace(launch.process_namespace_identity, { terminate: true, timeoutMs: 1000 });
|
|
1526
|
+
lifetimeBinding = { process_namespace_identity_digest: sha256(launch.process_namespace_identity) };
|
|
1527
|
+
}
|
|
1528
|
+
if (processObservation.status !== 'exited') fail('RECOVERY_NAMESPACE_STILL_RUNNING', 'namespace termination is not yet observed');
|
|
1529
|
+
const record = this.#append('execution_interruption_observed', {
|
|
1530
|
+
attempt_id, dispatch_event_id: dispatch.event_id, launch_nonce,
|
|
1531
|
+
launch_receipt_digest: sha256(launch), ...lifetimeBinding,
|
|
1532
|
+
status: 'interrupted', process: processObservation,
|
|
1533
|
+
exit_status: null, process_elapsed_ms: null, result_commit_ref: null, output_status: 'unavailable',
|
|
1534
|
+
// Process absence is not a checkpoint and must not consume the nonce.
|
|
1535
|
+
next_required_action: 'preserve_worktree_checkpoint',
|
|
1536
|
+
}, dispatch.event_id);
|
|
1537
|
+
return Object.freeze(canonicalize({ ...record.payload, event_id: record.event_id, replay: false }));
|
|
1538
|
+
});
|
|
1539
|
+
}
|
|
1540
|
+
|
|
1541
|
+
#interruptedExecutionContext({ attempt_id, launch_nonce, git_dir, expected_attempt_event_id }) {
|
|
1542
|
+
this.#assertAttemptRevision(attempt_id, expected_attempt_event_id);
|
|
1543
|
+
const attempt = this.#attempts().get(attempt_id);
|
|
1544
|
+
if (!attempt || attempt.state !== EXECUTABLE_ATTEMPT_STATE) fail('ATTEMPT_NOT_ACTIVE', 'checkpoint requires an active interrupted attempt');
|
|
1545
|
+
if (!isExpired(attempt.claim, timestamp(this.#clock))) fail('LEASE_STILL_ACTIVE', 'checkpoint cannot race a live lease');
|
|
1546
|
+
observeStoppedClaimProcess(attempt);
|
|
1547
|
+
const dispatch = this.#records.find((record) => record.type === 'execution_dispatched'
|
|
1548
|
+
&& record.payload.attempt_id === attempt_id && record.payload.invocation.launch_nonce === launch_nonce);
|
|
1549
|
+
const interruption = this.#records.find((record) => record.type === 'execution_interruption_observed'
|
|
1550
|
+
&& record.payload.dispatch_event_id === dispatch?.event_id);
|
|
1551
|
+
if (!dispatch || !interruption) fail('RECOVERY_INTERRUPTION_OBSERVATION_REQUIRED', 'checkpoint requires a canonical observed interruption');
|
|
1552
|
+
assertExecutionInterruptionLifetime(dispatch, interruption);
|
|
1553
|
+
if (this.#records.some((record) => (record.type === 'execution_recorded'
|
|
1554
|
+
&& record.payload?.execution_evidence?.receipt_nonce === launch_nonce)
|
|
1555
|
+
|| (record.type === 'provider_process_observed' && record.payload.dispatch_event_id === dispatch.event_id))) {
|
|
1556
|
+
fail('RECOVERY_EXECUTION_ALREADY_OBSERVED', 'observed completion cannot be replaced by restoration');
|
|
1557
|
+
}
|
|
1558
|
+
const reservation = this.#records.find((record) => ['execution_launch_reserved', 'rework_launch_reserved'].includes(record.type)
|
|
1559
|
+
&& record.payload.attempt_id === attempt_id && record.payload.invocation.launch_nonce === launch_nonce);
|
|
1560
|
+
if (!reservation || reservation.payload.claim_event_id !== attempt.claim_event_id
|
|
1561
|
+
|| reservation.payload.harness_process_id !== attempt.claim.harness_process_id
|
|
1562
|
+
|| dispatch.payload.claim_id !== attempt.claim.claim_id || dispatch.payload.operator_id !== attempt.claim.operator_id
|
|
1563
|
+
|| sha256(reservation.payload.invocation) !== sha256(dispatch.payload.invocation)) {
|
|
1564
|
+
fail('RECOVERY_ADMISSION_CLAIM_MISMATCH', 'checkpoint belongs to another claim generation');
|
|
1565
|
+
}
|
|
1566
|
+
this.#verifyExecutionAdmission({ attempt, invocation: dispatch.payload.invocation,
|
|
1567
|
+
receipt: reservation.payload.admission_receipt, project_dir: git_dir, harness_process_id: attempt.claim.harness_process_id });
|
|
1568
|
+
this.#assertAdmissionDispatchBound(reservation, dispatch.payload.launch_receipt);
|
|
1569
|
+
const unresolved = [...this.#effects().values()].filter((effect) => effect.attempt_id === attempt_id && !effect.receipt_ref);
|
|
1570
|
+
if (unresolved.length) fail('RECOVERY_EFFECTS_UNRESOLVED', 'checkpoint recovery requires external effect reconciliation');
|
|
1571
|
+
return { dispatch, interruption, binding: executionCheckpointBinding(dispatch, interruption) };
|
|
1572
|
+
}
|
|
1573
|
+
|
|
1574
|
+
/** Preserve observed partial work, without restoring Git or recovering a claim. */
|
|
1575
|
+
preserveInterruptedExecution({ attempt_id, launch_nonce, git_dir, expected_attempt_event_id } = {}) {
|
|
1576
|
+
return this.#exclusive(() => {
|
|
1577
|
+
const { dispatch, interruption, binding } = this.#interruptedExecutionContext({ attempt_id, launch_nonce, git_dir, expected_attempt_event_id });
|
|
1578
|
+
const existing = this.#records.find((record) => record.type === 'execution_checkpoint_preserved'
|
|
1579
|
+
&& record.payload.dispatch_event_id === dispatch.event_id);
|
|
1580
|
+
let receipt = existing?.payload.checkpoint_receipt;
|
|
1581
|
+
if (!receipt) {
|
|
1582
|
+
if (!this.#checkpointWriter) fail('RECOVERY_CHECKPOINT_WRITER_REQUIRED', 'checkpoint requires the configured protected host adapter');
|
|
1583
|
+
receipt = this.#checkpointWriter({ projectDir: git_dir, dispatch, interruption });
|
|
1584
|
+
}
|
|
1585
|
+
this.#verifyAuthority('execution_checkpoint', receipt, { schema_version: 1, purpose: 'rail_execution_checkpoint',
|
|
1586
|
+
project_dir: resolve(git_dir), binding });
|
|
1587
|
+
if (existing) return Object.freeze(canonicalize({ ...existing.payload, event_id: existing.event_id, replay: true }));
|
|
1588
|
+
const record = this.#append('execution_checkpoint_preserved', { attempt_id, dispatch_event_id: dispatch.event_id,
|
|
1589
|
+
launch_nonce, checkpoint_receipt: receipt, next_required_action: 'restore_checkpointed_execution_base' }, interruption.event_id);
|
|
1590
|
+
return Object.freeze(canonicalize({ ...record.payload, event_id: record.event_id, replay: false }));
|
|
1591
|
+
});
|
|
1592
|
+
}
|
|
1593
|
+
|
|
1594
|
+
/** Restore only a canonical checkpoint under the same journal exclusion. The
|
|
1595
|
+
* nonce remains pending until recoverAttempt verifies current storage. */
|
|
1596
|
+
restoreInterruptedExecution({ attempt_id, launch_nonce, git_dir, expected_attempt_event_id } = {}) {
|
|
1597
|
+
return this.#exclusive(() => {
|
|
1598
|
+
const { dispatch, binding } = this.#interruptedExecutionContext({ attempt_id, launch_nonce, git_dir, expected_attempt_event_id });
|
|
1599
|
+
const checkpoint = this.#records.find((record) => record.type === 'execution_checkpoint_preserved'
|
|
1600
|
+
&& record.payload.dispatch_event_id === dispatch.event_id);
|
|
1601
|
+
if (!checkpoint) fail('RECOVERY_CHECKPOINT_REQUIRED', 'restore requires a canonical preserved checkpoint');
|
|
1602
|
+
this.#verifyAuthority('execution_checkpoint', checkpoint.payload.checkpoint_receipt, {
|
|
1603
|
+
schema_version: 1, purpose: 'rail_execution_checkpoint', project_dir: resolve(git_dir), binding });
|
|
1604
|
+
const existing = this.#records.find((record) => record.type === 'execution_worktree_restored'
|
|
1605
|
+
&& record.payload.dispatch_event_id === dispatch.event_id);
|
|
1606
|
+
if (existing) {
|
|
1607
|
+
this.#restorationProof(dispatch, { current: true });
|
|
1608
|
+
return Object.freeze(canonicalize({ ...existing.payload, event_id: existing.event_id, replay: true }));
|
|
1609
|
+
}
|
|
1610
|
+
if (!this.#restorationWriter) fail('RECOVERY_RESTORATION_WRITER_REQUIRED', 'restore requires the configured protected host adapter');
|
|
1611
|
+
const receipt = this.#restorationWriter({ projectDir: git_dir, receipt: checkpoint.payload.checkpoint_receipt });
|
|
1612
|
+
this.#verifyAuthority('execution_restoration', receipt, { schema_version: 1, purpose: 'rail_execution_restoration',
|
|
1613
|
+
project_dir: resolve(git_dir), binding, checkpoint_receipt_digest: sha256(checkpoint.payload.checkpoint_receipt),
|
|
1614
|
+
restored_base_commit_ref: binding.base_commit_ref }, { checkpoint_receipt: checkpoint.payload.checkpoint_receipt, current: true });
|
|
1615
|
+
const record = this.#append('execution_worktree_restored', { attempt_id, dispatch_event_id: dispatch.event_id,
|
|
1616
|
+
launch_nonce, checkpoint_event_id: checkpoint.event_id, restoration_receipt: receipt,
|
|
1617
|
+
next_required_action: 'recover_claim_with_fresh_launch_nonce' }, checkpoint.event_id);
|
|
1618
|
+
return Object.freeze(canonicalize({ ...record.payload, event_id: record.event_id, replay: false }));
|
|
1619
|
+
});
|
|
1620
|
+
}
|
|
1621
|
+
|
|
1622
|
+
/** Import facts from one durable observation, never launch or recover a claim. */
|
|
1623
|
+
reconcileObservedExecution({ attempt_id, launch_nonce, git_dir, expected_attempt_event_id } = {}) {
|
|
1624
|
+
return this.#exclusive(() => {
|
|
1625
|
+
this.#assertAttemptRevision(attempt_id, expected_attempt_event_id);
|
|
1626
|
+
for (const [field, value] of Object.entries({ attempt_id, launch_nonce, git_dir })) {
|
|
1627
|
+
assertString(value, 'INVALID_EXECUTION_RECONCILIATION', field);
|
|
1628
|
+
}
|
|
1629
|
+
const existing = this.#records.find((record) => record.type === 'execution_recorded'
|
|
1630
|
+
&& record.payload.attempt_id === attempt_id && record.payload.execution_evidence.receipt_nonce === launch_nonce);
|
|
1631
|
+
if (existing) return Object.freeze(canonicalize({ ...existing.payload.execution_evidence, event_id: existing.event_id, replay: true }));
|
|
1632
|
+
const attempt = this.#attempts().get(attempt_id);
|
|
1633
|
+
if (!attempt || attempt.state !== EXECUTABLE_ATTEMPT_STATE) fail('ATTEMPT_NOT_ACTIVE', 'completion reconciliation requires an active attempt');
|
|
1634
|
+
if (!isExpired(attempt.claim, timestamp(this.#clock))) fail('LEASE_STILL_ACTIVE', 'completion reconciliation cannot race a live lease');
|
|
1635
|
+
observeStoppedClaimProcess(attempt);
|
|
1636
|
+
const dispatch = this.#records.find((record) => record.type === 'execution_dispatched'
|
|
1637
|
+
&& record.payload.attempt_id === attempt_id && record.payload.invocation.launch_nonce === launch_nonce);
|
|
1638
|
+
const observation = this.#records.find((record) => record.type === 'provider_process_observed'
|
|
1639
|
+
&& record.payload.attempt_id === attempt_id && record.payload.dispatch_event_id === dispatch?.event_id);
|
|
1640
|
+
if (!dispatch || !observation?.payload.terminal_output) {
|
|
1641
|
+
fail('RECOVERY_EXECUTION_COMPLETION_UNAVAILABLE', 'reconciliation requires a complete canonical observation, not caller-supplied output');
|
|
1642
|
+
}
|
|
1643
|
+
if (dispatch.payload.launch_receipt.working_dir !== resolve(git_dir)) {
|
|
1644
|
+
fail('EXECUTION_RECONCILIATION_PROJECT_MISMATCH', 'completion must be reconciled in its exact dispatched project');
|
|
1645
|
+
}
|
|
1646
|
+
const { terminal_authority_receipt: receipt, terminal_output: output } = observation.payload;
|
|
1647
|
+
assertProviderProcessReceipt(dispatch, receipt, (kind, value, expected) => this.#verifyAuthority(kind, value, expected));
|
|
1648
|
+
assertProviderProcessOutput(output, receipt);
|
|
1649
|
+
const unresolvedEffects = [...this.#effects().values()].filter((effect) => effect.attempt_id === attempt_id && !effect.receipt_ref);
|
|
1650
|
+
if (unresolvedEffects.length) fail('RECOVERY_EFFECTS_UNRESOLVED', 'unreceipted effects require actual external reconciliation', {
|
|
1651
|
+
idempotency_keys: unresolvedEffects.map((effect) => effect.idempotency_key),
|
|
1652
|
+
});
|
|
1653
|
+
const { invocation, launch_receipt: launch, run_kind, source_decision_ref } = dispatch.payload;
|
|
1654
|
+
const execution = { terminal_id: receipt.terminal_id, provider: invocation.harness_id,
|
|
1655
|
+
model: invocation.model_pin.model_id, reasoning_configuration: invocation.model_pin.reasoning_configuration,
|
|
1656
|
+
exit_status: receipt.exit_status, stdout: output.stdout, stderr: output.stderr,
|
|
1657
|
+
...(launch.started_at === undefined ? {} : { started_at: launch.started_at }),
|
|
1658
|
+
terminal_authority_receipt: receipt };
|
|
1659
|
+
return this.#recordExecutionEvidence({ attempt_id, claim_id: attempt.claim.claim_id,
|
|
1660
|
+
operator_id: attempt.claim.operator_id, invocation, execution,
|
|
1661
|
+
base_commit_ref: launch.execution_base_commit_ref,
|
|
1662
|
+
result_commit_ref: receipt.exit_status === 0 && receipt.execution_result_status !== 'invalid_output'
|
|
1663
|
+
? receipt.result_commit_ref : null,
|
|
1664
|
+
run_kind, source_decision_ref: source_decision_ref ?? null,
|
|
1665
|
+
receipt_nonce: launch_nonce, git_dir,
|
|
1666
|
+
}, observation.event_id);
|
|
1667
|
+
});
|
|
1668
|
+
}
|
|
1669
|
+
|
|
1670
|
+
recordExecutionEvidence(input = {}) {
|
|
1671
|
+
return this.#exclusive(() => this.#recordExecutionEvidence(input));
|
|
1672
|
+
}
|
|
1673
|
+
|
|
1674
|
+
#recordExecutionEvidence({
|
|
1157
1675
|
attempt_id, claim_id, operator_id, invocation, execution, base_commit_ref,
|
|
1158
1676
|
result_commit_ref = null, run_kind = 'initial', source_decision_ref = null,
|
|
1159
1677
|
receipt_nonce, git_dir,
|
|
1160
|
-
} = {}) {
|
|
1161
|
-
return this.#exclusive(() => {
|
|
1678
|
+
} = {}, observedCompletionEventId = null) {
|
|
1162
1679
|
for (const [field, value] of Object.entries({ attempt_id, claim_id, operator_id, receipt_nonce, git_dir })) {
|
|
1163
1680
|
assertString(value, 'INVALID_EXECUTION_EVIDENCE', field);
|
|
1164
1681
|
}
|
|
@@ -1179,7 +1696,7 @@ export class RailEngine {
|
|
|
1179
1696
|
if (attempt.claim.claim_id !== claim_id || attempt.claim.operator_id !== operator_id) {
|
|
1180
1697
|
fail('CLAIM_OWNER_MISMATCH', 'execution evidence must be receipted by the active claim owner', { attempt_id });
|
|
1181
1698
|
}
|
|
1182
|
-
this.#assertLiveLease(attempt, timestamp(this.#clock));
|
|
1699
|
+
if (observedCompletionEventId === null) this.#assertLiveLease(attempt, timestamp(this.#clock));
|
|
1183
1700
|
if (invocation.attempt_id !== attempt_id) fail('EXECUTION_INVOCATION_MISMATCH', 'execution invocation is not bound to the active attempt');
|
|
1184
1701
|
if (receipt_nonce !== invocation.launch_nonce) fail('EXECUTION_LAUNCH_NONCE_MISMATCH', 'execution evidence must echo the dispatched launch nonce');
|
|
1185
1702
|
const dispatch = this.#records.find((record) => record.type === 'execution_dispatched'
|
|
@@ -1190,6 +1707,7 @@ export class RailEngine {
|
|
|
1190
1707
|
if (!dispatch || sha256(dispatch.payload.invocation) !== sha256(invocation)) {
|
|
1191
1708
|
fail('EXECUTION_DISPATCH_REQUIRED', 'execution evidence requires an exact canonical dispatch record');
|
|
1192
1709
|
}
|
|
1710
|
+
this.#assertCompletionNotRestored(dispatch);
|
|
1193
1711
|
if (execution.terminal_id !== dispatch.payload.launch_receipt.terminal_id
|
|
1194
1712
|
|| execution.provider !== invocation.harness_id
|
|
1195
1713
|
|| execution.model !== invocation.model_pin?.model_id
|
|
@@ -1209,6 +1727,11 @@ export class RailEngine {
|
|
|
1209
1727
|
|| typeof execution.terminal_authority_receipt?.authority_signature !== 'string') {
|
|
1210
1728
|
fail('EXECUTION_TERMINAL_AUTHORITY_MISMATCH', 'completion receipt does not bind the canonical launch and terminal result');
|
|
1211
1729
|
}
|
|
1730
|
+
if (dispatch.payload.launch_receipt.process_namespace_identity
|
|
1731
|
+
&& execution.terminal_authority_receipt.process_namespace_identity_digest !== sha256(dispatch.payload.launch_receipt.process_namespace_identity)) {
|
|
1732
|
+
fail('EXECUTION_TERMINAL_AUTHORITY_MISMATCH', 'completion must bind the exact observed namespace identity');
|
|
1733
|
+
}
|
|
1734
|
+
assertTerminalCompletionContainment(execution.terminal_authority_receipt, dispatch.payload.launch_receipt, 'EXECUTION_TERMINAL_AUTHORITY_MISMATCH');
|
|
1212
1735
|
const resultStatus = execution.terminal_authority_receipt.execution_result_status;
|
|
1213
1736
|
const invalidOutput = execution.exit_status === 0 && resultStatus === 'invalid_output';
|
|
1214
1737
|
let executionFailureCode = null;
|
|
@@ -1254,7 +1777,7 @@ export class RailEngine {
|
|
|
1254
1777
|
fail('EXECUTION_LAUNCH_ALREADY_RECEIPTED', 'this canonical launch already has execution evidence');
|
|
1255
1778
|
}
|
|
1256
1779
|
const successful = this.#executionEvidence(attempt_id, { successfulOnly: true });
|
|
1257
|
-
const expectedBase = successful.at(-1)?.result_commit_ref ?? attempt
|
|
1780
|
+
const expectedBase = successful.at(-1)?.result_commit_ref ?? attemptExecutionBase(attempt);
|
|
1258
1781
|
if (base_commit_ref !== expectedBase) fail('EXECUTION_BASE_MISMATCH', 'execution base must equal the latest canonical successful result or sealed base');
|
|
1259
1782
|
if (run_kind === 'rework') {
|
|
1260
1783
|
assertString(source_decision_ref, 'INVALID_EXECUTION_EVIDENCE', 'source_decision_ref');
|
|
@@ -1315,9 +1838,8 @@ export class RailEngine {
|
|
|
1315
1838
|
&& record.payload?.execution_evidence?.evidence_id === evidence_id);
|
|
1316
1839
|
if (existing) return Object.freeze(canonicalize({ ...existing.payload.execution_evidence, event_id: existing.event_id, replay: true }));
|
|
1317
1840
|
const evidence = canonicalize({ ...core, evidence_id, digest: sha256(core) });
|
|
1318
|
-
const record = this.#append('execution_recorded', { attempt_id, execution_evidence: evidence });
|
|
1841
|
+
const record = this.#append('execution_recorded', { attempt_id, execution_evidence: evidence }, observedCompletionEventId);
|
|
1319
1842
|
return Object.freeze(canonicalize({ ...evidence, event_id: record.event_id, replay: false }));
|
|
1320
|
-
});
|
|
1321
1843
|
}
|
|
1322
1844
|
|
|
1323
1845
|
#currentClaimForTask(taskId) {
|
|
@@ -1328,10 +1850,37 @@ export class RailEngine {
|
|
|
1328
1850
|
return undefined;
|
|
1329
1851
|
}
|
|
1330
1852
|
|
|
1331
|
-
#
|
|
1853
|
+
#mergedAttemptsForHandoff(handoff) {
|
|
1332
1854
|
return [...this.#attempts().values()]
|
|
1333
|
-
.filter((attempt) => attempt.handoff_id ===
|
|
1334
|
-
|
|
1855
|
+
.filter((attempt) => attempt.handoff_id === handoff.handoff_id
|
|
1856
|
+
&& attempt.manifest_digest === handoff.seal.manifest_digest && attempt.state === 'merged');
|
|
1857
|
+
}
|
|
1858
|
+
|
|
1859
|
+
#executionBaseForClaim(handoff, mergedAttempts, git_dir) {
|
|
1860
|
+
const mergedIds = new Set(mergedAttempts.map((attempt) => attempt.attempt_id));
|
|
1861
|
+
const completions = this.#records.filter((record) => record.type === 'attempt_completed'
|
|
1862
|
+
&& record.payload.completion.state === 'merged'
|
|
1863
|
+
&& mergedIds.has(record.payload.completion.attempt_id));
|
|
1864
|
+
const base = completions.at(-1)?.payload.completion.git.result_ref ?? handoff.seal.git_commit_ref;
|
|
1865
|
+
// A live HEAD is not integration evidence. Only canonical merged results can
|
|
1866
|
+
// advance the base, and all previous merged results must be contained in it.
|
|
1867
|
+
for (const predecessor of [handoff.seal.git_commit_ref,
|
|
1868
|
+
...completions.map((record) => record.payload.completion.git.result_ref)]) {
|
|
1869
|
+
if (predecessor === base) verifyLocalGitCommit({ git_dir, git_commit_ref: base });
|
|
1870
|
+
else {
|
|
1871
|
+
try {
|
|
1872
|
+
verifyCommitDescendsFrom({ git_dir, base_commit_ref: predecessor, result_commit_ref: base });
|
|
1873
|
+
} catch (error) {
|
|
1874
|
+
fail('EXECUTION_BASE_NOT_INTEGRATED', 'canonical merged results do not form an integrated execution base', {
|
|
1875
|
+
predecessor, execution_base_commit_ref: base, cause_code: error.code,
|
|
1876
|
+
});
|
|
1877
|
+
}
|
|
1878
|
+
}
|
|
1879
|
+
}
|
|
1880
|
+
return {
|
|
1881
|
+
execution_base_commit_ref: base,
|
|
1882
|
+
execution_base_completion_refs: completions.map((record) => record.event_id),
|
|
1883
|
+
};
|
|
1335
1884
|
}
|
|
1336
1885
|
|
|
1337
1886
|
#assertAcceptedReviews(attemptId, reviews) {
|
|
@@ -1359,8 +1908,10 @@ export class RailEngine {
|
|
|
1359
1908
|
fail(code, 'task has an existing claim and cannot be taken over without recovery', { task_id, attempt_id: incumbent.attempt.attempt_id });
|
|
1360
1909
|
}
|
|
1361
1910
|
if (this.#attempts().has(attempt_id)) fail('ATTEMPT_ALREADY_EXISTS', 'attempt_id already exists in journal', { attempt_id });
|
|
1362
|
-
const
|
|
1363
|
-
const
|
|
1911
|
+
const mergedAttempts = this.#mergedAttemptsForHandoff(handoff);
|
|
1912
|
+
const completed_task_ids = mergedAttempts.map((attempt) => attempt.task_id);
|
|
1913
|
+
const binding = createAttemptBinding({ handoff, task_id, attempt_id, completed_task_ids, git_dir });
|
|
1914
|
+
const attempt = canonicalize({ ...binding, ...this.#executionBaseForClaim(handoff, mergedAttempts, git_dir) });
|
|
1364
1915
|
const acquired_at = timestamp(this.#clock);
|
|
1365
1916
|
const claim = canonicalize({
|
|
1366
1917
|
claim_id: `claim-${sha256({ attempt_id, acquired_at, operator_id, harness_process_id }).slice(0, 24)}`,
|
|
@@ -1370,6 +1921,7 @@ export class RailEngine {
|
|
|
1370
1921
|
operator_id,
|
|
1371
1922
|
harness_process_id,
|
|
1372
1923
|
hostname: hostname(),
|
|
1924
|
+
...(process.platform === 'darwin' ? { darwin_host_boot_identity: captureDarwinHostBootIdentity() } : {}),
|
|
1373
1925
|
lease: { acquired_at, expires_at: plusMs(acquired_at, lease_ttl_ms), heartbeat_at: acquired_at },
|
|
1374
1926
|
idempotency_scope: `task:${task_id}:attempt:${attempt_id}`,
|
|
1375
1927
|
});
|
|
@@ -1378,8 +1930,9 @@ export class RailEngine {
|
|
|
1378
1930
|
});
|
|
1379
1931
|
}
|
|
1380
1932
|
|
|
1381
|
-
heartbeat({ attempt_id, claim_id, operator_id, harness_process_id, lease_ttl_ms = 300000 } = {}) {
|
|
1933
|
+
heartbeat({ attempt_id, claim_id, operator_id, harness_process_id, lease_ttl_ms = 300000, expected_attempt_event_id } = {}) {
|
|
1382
1934
|
return this.#exclusive(() => {
|
|
1935
|
+
this.#assertAttemptRevision(attempt_id, expected_attempt_event_id);
|
|
1383
1936
|
assertString(attempt_id, 'INVALID_HEARTBEAT', 'attempt_id');
|
|
1384
1937
|
assertString(claim_id, 'INVALID_HEARTBEAT', 'claim_id');
|
|
1385
1938
|
assertString(operator_id, 'INVALID_HEARTBEAT', 'operator_id');
|
|
@@ -1481,7 +2034,8 @@ export class RailEngine {
|
|
|
1481
2034
|
&& record.payload?.attempt_id === attempt_id
|
|
1482
2035
|
&& record.payload?.run_kind === 'rework'
|
|
1483
2036
|
&& record.payload?.rework_reservation_id === existing.payload.reservation_id);
|
|
1484
|
-
const unresolvedLaunches = launches.filter((launch) => !
|
|
2037
|
+
const unresolvedLaunches = launches.filter((launch) => !this.#launchReconciled(launch.payload.invocation.launch_nonce)
|
|
2038
|
+
&& !dispatches.some((dispatch) =>
|
|
1485
2039
|
dispatch.payload?.invocation?.launch_nonce === launch.payload.invocation.launch_nonce));
|
|
1486
2040
|
if (unresolvedLaunches.length > 0) {
|
|
1487
2041
|
fail('RAIL_REWORK_LAUNCH_UNRESOLVED', 'the previous rework reservation has a launch intent without authenticated provider dispatch', {
|
|
@@ -1489,7 +2043,8 @@ export class RailEngine {
|
|
|
1489
2043
|
launch_nonces: unresolvedLaunches.map((launch) => launch.payload.invocation.launch_nonce),
|
|
1490
2044
|
});
|
|
1491
2045
|
}
|
|
1492
|
-
const unresolved = dispatches.filter((dispatch) => !this.#
|
|
2046
|
+
const unresolved = dispatches.filter((dispatch) => !this.#launchReconciled(dispatch.payload.invocation.launch_nonce)
|
|
2047
|
+
&& !this.#records.some((record) => record.type === 'execution_recorded'
|
|
1493
2048
|
&& record.payload?.execution_evidence?.receipt_nonce === dispatch.payload.invocation.launch_nonce));
|
|
1494
2049
|
if (unresolved.length > 0) {
|
|
1495
2050
|
fail('RAIL_REWORK_DISPATCH_UNRESOLVED', 'the previous rework reservation has a dispatch without canonical execution evidence', {
|
|
@@ -1503,7 +2058,9 @@ export class RailEngine {
|
|
|
1503
2058
|
this.#append('rework_reservation_released', canonicalize({
|
|
1504
2059
|
attempt_id, claim_id, operator_id, harness_process_id,
|
|
1505
2060
|
source_decision_ref, reservation_id: existing.payload.reservation_id,
|
|
1506
|
-
reason: dispatches.
|
|
2061
|
+
reason: dispatches.some((dispatch) => this.#launchReconciled(dispatch.payload.invocation.launch_nonce))
|
|
2062
|
+
? 'recovered_after_checkpoint_restoration' : dispatches.length > 0 ? 'recovered_after_canonical_execution'
|
|
2063
|
+
: launches.length > 0 ? 'recovered_after_not_started_admission' : 'recovered_before_launch_intent',
|
|
1507
2064
|
reconciled_claim_event_id: existing.payload.claim_event_id,
|
|
1508
2065
|
}), attempt.recovery_event_id);
|
|
1509
2066
|
}
|
|
@@ -1520,6 +2077,7 @@ export class RailEngine {
|
|
|
1520
2077
|
reserveReworkLaunch({
|
|
1521
2078
|
attempt_id, claim_id, operator_id, harness_process_id,
|
|
1522
2079
|
source_decision_ref, reservation_id, invocation,
|
|
2080
|
+
admission_receipt = null, project_dir = null,
|
|
1523
2081
|
} = {}) {
|
|
1524
2082
|
return this.#exclusive(() => {
|
|
1525
2083
|
for (const [field, value] of Object.entries({
|
|
@@ -1530,6 +2088,7 @@ export class RailEngine {
|
|
|
1530
2088
|
assertString(invocation.invocation_id, 'INVALID_REWORK_LAUNCH_RESERVATION', 'invocation.invocation_id');
|
|
1531
2089
|
assertString(invocation.launch_nonce, 'INVALID_REWORK_LAUNCH_RESERVATION', 'invocation.launch_nonce');
|
|
1532
2090
|
assertString(invocation.prompt_digest, 'INVALID_REWORK_LAUNCH_RESERVATION', 'invocation.prompt_digest');
|
|
2091
|
+
if (this.#launchReconciled(invocation.launch_nonce)) fail('EXECUTION_LAUNCH_ALREADY_RECONCILED', 'reconciled rework launch nonce may not be reused');
|
|
1533
2092
|
const attempt = this.#attempts().get(attempt_id);
|
|
1534
2093
|
if (!attempt || attempt.state !== EXECUTABLE_ATTEMPT_STATE) {
|
|
1535
2094
|
fail('ATTEMPT_NOT_ACTIVE', 'rework launch reservation requires an active attempt', { attempt_id });
|
|
@@ -1555,6 +2114,9 @@ export class RailEngine {
|
|
|
1555
2114
|
|| invocation.model_pin?.reasoning_configuration !== binding.reasoning_configuration) {
|
|
1556
2115
|
fail('EXECUTION_BINDING_MISMATCH', 'rework launch reservation does not match the sealed task binding');
|
|
1557
2116
|
}
|
|
2117
|
+
if (admission_receipt !== null) this.#verifyExecutionAdmission({
|
|
2118
|
+
attempt, invocation, receipt: admission_receipt, project_dir, harness_process_id,
|
|
2119
|
+
});
|
|
1558
2120
|
const reservation = [...this.#records].reverse().find((record) => record.type === 'rework_reserved'
|
|
1559
2121
|
&& record.payload?.attempt_id === attempt_id
|
|
1560
2122
|
&& record.payload?.source_decision_ref === source_decision_ref);
|
|
@@ -1572,7 +2134,8 @@ export class RailEngine {
|
|
|
1572
2134
|
if (existing) {
|
|
1573
2135
|
if (sha256(existing.payload.invocation) !== sha256(invocation)
|
|
1574
2136
|
|| existing.payload.reservation_id !== reservation_id
|
|
1575
|
-
|| existing.payload.claim_event_id !== attempt.claim_event_id
|
|
2137
|
+
|| existing.payload.claim_event_id !== attempt.claim_event_id
|
|
2138
|
+
|| sha256(existing.payload.admission_receipt ?? null) !== sha256(admission_receipt)) {
|
|
1576
2139
|
fail('EXECUTION_LAUNCH_NONCE_CONFLICT', 'launch nonce is already reserved for different rework material');
|
|
1577
2140
|
}
|
|
1578
2141
|
return Object.freeze(canonicalize({ ...existing.payload, event_id: existing.event_id, replay: true }));
|
|
@@ -1580,6 +2143,7 @@ export class RailEngine {
|
|
|
1580
2143
|
const unresolvedLaunch = this.#records.find((record) => record.type === 'rework_launch_reserved'
|
|
1581
2144
|
&& record.payload?.attempt_id === attempt_id
|
|
1582
2145
|
&& record.payload?.reservation_id === reservation_id
|
|
2146
|
+
&& !this.#launchReconciled(record.payload.invocation.launch_nonce)
|
|
1583
2147
|
&& !this.#records.some((candidate) => candidate.type === 'execution_dispatched'
|
|
1584
2148
|
&& candidate.payload?.invocation?.launch_nonce === record.payload.invocation.launch_nonce));
|
|
1585
2149
|
if (unresolvedLaunch) {
|
|
@@ -1589,6 +2153,7 @@ export class RailEngine {
|
|
|
1589
2153
|
&& record.payload?.attempt_id === attempt_id
|
|
1590
2154
|
&& record.payload?.run_kind === 'rework'
|
|
1591
2155
|
&& record.payload?.rework_reservation_id === reservation_id
|
|
2156
|
+
&& !this.#launchReconciled(record.payload.invocation.launch_nonce)
|
|
1592
2157
|
&& !this.#records.some((candidate) => candidate.type === 'execution_recorded'
|
|
1593
2158
|
&& candidate.payload?.execution_evidence?.receipt_nonce === record.payload.invocation.launch_nonce));
|
|
1594
2159
|
if (unresolvedDispatch) {
|
|
@@ -1598,6 +2163,7 @@ export class RailEngine {
|
|
|
1598
2163
|
attempt_id, claim_id, operator_id, harness_process_id,
|
|
1599
2164
|
claim_event_id: attempt.claim_event_id, source_decision_ref,
|
|
1600
2165
|
reservation_id, invocation, reserved_at: timestamp(this.#clock),
|
|
2166
|
+
...(admission_receipt ? { admission_receipt } : {}),
|
|
1601
2167
|
});
|
|
1602
2168
|
const record = this.#append('rework_launch_reserved', payload, reservation.event_id);
|
|
1603
2169
|
return Object.freeze(canonicalize({ ...payload, event_id: record.event_id, replay: false }));
|
|
@@ -1635,6 +2201,7 @@ export class RailEngine {
|
|
|
1635
2201
|
if (existing) return Object.freeze(canonicalize({ ...existing.payload, event_id: existing.event_id, replay: true }));
|
|
1636
2202
|
const unresolvedLaunch = this.#records.find((record) => record.type === 'rework_launch_reserved'
|
|
1637
2203
|
&& record.payload?.reservation_id === reservation_id
|
|
2204
|
+
&& !this.#launchReconciled(record.payload.invocation.launch_nonce)
|
|
1638
2205
|
&& !this.#records.some((candidate) => candidate.type === 'execution_dispatched'
|
|
1639
2206
|
&& candidate.payload?.invocation?.launch_nonce === record.payload.invocation.launch_nonce));
|
|
1640
2207
|
if (unresolvedLaunch) {
|
|
@@ -1643,6 +2210,7 @@ export class RailEngine {
|
|
|
1643
2210
|
const unresolvedDispatch = this.#records.find((record) => record.type === 'execution_dispatched'
|
|
1644
2211
|
&& record.payload?.run_kind === 'rework'
|
|
1645
2212
|
&& record.payload?.rework_reservation_id === reservation_id
|
|
2213
|
+
&& !this.#launchReconciled(record.payload.invocation.launch_nonce)
|
|
1646
2214
|
&& !this.#records.some((candidate) => candidate.type === 'execution_recorded'
|
|
1647
2215
|
&& candidate.payload?.execution_evidence?.receipt_nonce === record.payload.invocation.launch_nonce));
|
|
1648
2216
|
if (unresolvedDispatch) {
|
|
@@ -1708,12 +2276,28 @@ export class RailEngine {
|
|
|
1708
2276
|
fail('RAIL_ADJUDICATION_REQUIRED', 'a canonical AI adjudication decision is required before another review can be dispatched');
|
|
1709
2277
|
}
|
|
1710
2278
|
if (invocation.attempt_id !== attempt_id || invocation.action !== 'review') fail('REVIEW_INVOCATION_MISMATCH', 'review launch is not bound to this attempt');
|
|
1711
|
-
const selected =
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
2279
|
+
const selected = attempt.review_binding
|
|
2280
|
+
? {
|
|
2281
|
+
provider_id: attempt.review_binding.provider_id,
|
|
2282
|
+
harness_id: attempt.review_binding.harness_id,
|
|
2283
|
+
model_pin: {
|
|
2284
|
+
model_id: attempt.review_binding.model_id,
|
|
2285
|
+
catalog_snapshot_ref: snapshot.snapshot_id,
|
|
2286
|
+
reasoning_configuration: attempt.review_binding.reasoning_configuration,
|
|
2287
|
+
},
|
|
2288
|
+
}
|
|
2289
|
+
: selectCanonicalReviewer({
|
|
2290
|
+
installation, snapshot,
|
|
2291
|
+
builder_identity: {
|
|
2292
|
+
provider_id: attempt.execution_binding.provider_id,
|
|
2293
|
+
model_id: attempt.execution_binding.model_id,
|
|
2294
|
+
},
|
|
2295
|
+
clock: this.#clock,
|
|
2296
|
+
});
|
|
2297
|
+
assertEligibleInvocation({
|
|
2298
|
+
installation,
|
|
2299
|
+
snapshot,
|
|
2300
|
+
invocation: { ...selected, action: 'review' },
|
|
1717
2301
|
clock: this.#clock,
|
|
1718
2302
|
});
|
|
1719
2303
|
if (invocation.provider_id !== selected.provider_id
|
|
@@ -1721,7 +2305,9 @@ export class RailEngine {
|
|
|
1721
2305
|
|| invocation.model_pin?.model_id !== selected.model_pin.model_id
|
|
1722
2306
|
|| invocation.model_pin?.catalog_snapshot_ref !== selected.model_pin.catalog_snapshot_ref
|
|
1723
2307
|
|| invocation.model_pin?.reasoning_configuration !== selected.model_pin.reasoning_configuration) {
|
|
1724
|
-
fail('NONCANONICAL_REVIEWER',
|
|
2308
|
+
fail(attempt.review_binding ? 'REVIEW_BINDING_MISMATCH' : 'NONCANONICAL_REVIEWER', attempt.review_binding
|
|
2309
|
+
? 'review launch must use the sealed task review binding'
|
|
2310
|
+
: 'review launch must use the strongest canonical independent reviewer');
|
|
1725
2311
|
}
|
|
1726
2312
|
this.#verifyAuthority('review_launch', launch_receipt, {
|
|
1727
2313
|
purpose: 'terminal_launch', authority_kind: 'rail_review',
|
|
@@ -1733,11 +2319,11 @@ export class RailEngine {
|
|
|
1733
2319
|
working_dir: resolve(working_dir),
|
|
1734
2320
|
prompt_digest: invocation.prompt_digest,
|
|
1735
2321
|
});
|
|
2322
|
+
assertTerminalLaunchContainment(launch_receipt, 'REVIEW_LAUNCH_RECEIPT_MISMATCH');
|
|
1736
2323
|
if (launch_receipt.task_id !== attempt.task_id
|
|
1737
2324
|
|| launch_receipt.working_dir !== resolve(working_dir)
|
|
1738
2325
|
|| launch_receipt.prompt_digest !== invocation.prompt_digest
|
|
1739
2326
|
|| !/^[a-f0-9]{64}$/.test(launch_receipt.command_digest || '')
|
|
1740
|
-
|| launch_receipt.process_containment_kind !== 'bubblewrap_pid_namespace_v1'
|
|
1741
2327
|
|| typeof launch_receipt.process_containment_id !== 'string' || !launch_receipt.process_containment_id
|
|
1742
2328
|
|| !/^[a-f0-9]{64}$/.test(launch_receipt.process_containment_digest || '')) {
|
|
1743
2329
|
fail('REVIEW_LAUNCH_RECEIPT_MISMATCH', 'review launch receipt must bind the canonical prompt, task, working directory and command');
|
|
@@ -1796,11 +2382,11 @@ export class RailEngine {
|
|
|
1796
2382
|
working_dir: resolve(working_dir),
|
|
1797
2383
|
prompt_digest: invocation.prompt_digest,
|
|
1798
2384
|
});
|
|
2385
|
+
assertTerminalLaunchContainment(launch_receipt, 'ADJUDICATION_LAUNCH_RECEIPT_MISMATCH');
|
|
1799
2386
|
if (launch_receipt.task_id !== attempt.task_id
|
|
1800
2387
|
|| launch_receipt.working_dir !== resolve(working_dir)
|
|
1801
2388
|
|| launch_receipt.prompt_digest !== invocation.prompt_digest
|
|
1802
2389
|
|| !/^[a-f0-9]{64}$/.test(launch_receipt.command_digest || '')
|
|
1803
|
-
|| launch_receipt.process_containment_kind !== 'bubblewrap_pid_namespace_v1'
|
|
1804
2390
|
|| typeof launch_receipt.process_containment_id !== 'string' || !launch_receipt.process_containment_id
|
|
1805
2391
|
|| !/^[a-f0-9]{64}$/.test(launch_receipt.process_containment_digest || '')) {
|
|
1806
2392
|
fail('ADJUDICATION_LAUNCH_RECEIPT_MISMATCH', 'adjudication launch receipt must bind the canonical prompt, task, working directory and command');
|
|
@@ -1866,6 +2452,7 @@ export class RailEngine {
|
|
|
1866
2452
|
|| terminal_authority_receipt.stdout_digest !== sha256(adjudication_output)) {
|
|
1867
2453
|
fail('ADJUDICATION_TERMINAL_AUTHORITY_MISMATCH', 'adjudication observation does not bind its canonical launch and full stdout');
|
|
1868
2454
|
}
|
|
2455
|
+
assertTerminalCompletionContainment(terminal_authority_receipt, dispatch.payload.launch_receipt, 'ADJUDICATION_TERMINAL_AUTHORITY_MISMATCH');
|
|
1869
2456
|
const parsedDecision = parseAdjudicationOutput(adjudication_output);
|
|
1870
2457
|
if (sha256(parsedDecision) !== sha256(validateAdjudicationDecision(decision))) {
|
|
1871
2458
|
fail('ADJUDICATION_PAYLOAD_MISMATCH', 'adjudication observation differs from terminal-authorized stdout');
|
|
@@ -1931,6 +2518,7 @@ export class RailEngine {
|
|
|
1931
2518
|
|| terminal_authority_receipt.terminal_id !== dispatch.payload.launch_receipt.terminal_id) {
|
|
1932
2519
|
fail('REVIEW_TERMINAL_AUTHORITY_MISMATCH', 'review completion does not bind the canonical launch');
|
|
1933
2520
|
}
|
|
2521
|
+
assertTerminalCompletionContainment(terminal_authority_receipt, dispatch.payload.launch_receipt, 'REVIEW_TERMINAL_AUTHORITY_MISMATCH');
|
|
1934
2522
|
assertString(git_dir, 'INVALID_REVIEW_DECISION', 'git_dir');
|
|
1935
2523
|
assertString(result_commit_ref, 'INVALID_REVIEW_DECISION', 'result_commit_ref');
|
|
1936
2524
|
const validated = acceptReview({ review, invocation, installation, snapshot, clock: this.#clock });
|
|
@@ -1972,7 +2560,7 @@ export class RailEngine {
|
|
|
1972
2560
|
if (requiredReworkSource && executionEvidence.source_decision_ref !== requiredReworkSource) {
|
|
1973
2561
|
fail('RAIL_REWORK_RESULT_REQUIRED', 'a successful canonical rework for the latest decision is required before another review');
|
|
1974
2562
|
}
|
|
1975
|
-
verifyCommitDescendsFrom({ git_dir, base_commit_ref: attempt
|
|
2563
|
+
verifyCommitDescendsFrom({ git_dir, base_commit_ref: attemptExecutionBase(attempt), result_commit_ref });
|
|
1976
2564
|
const priorReviewEvidence = this.#reviewEvidence(attempt_id);
|
|
1977
2565
|
const expected = decideReviewProgress({
|
|
1978
2566
|
prior_review_evidence: priorReviewEvidence,
|
|
@@ -1985,7 +2573,7 @@ export class RailEngine {
|
|
|
1985
2573
|
if (sha256(expected) !== sha256(decision)) fail('REVIEW_DECISION_MISMATCH', 'journal decision does not match canonical Review Council history');
|
|
1986
2574
|
const record = this.#append('review_decided', {
|
|
1987
2575
|
attempt_id, task_id: attempt.task_id, review_ref,
|
|
1988
|
-
base_commit_ref: attempt
|
|
2576
|
+
base_commit_ref: attemptExecutionBase(attempt), result_commit_ref,
|
|
1989
2577
|
review: canonicalize(validated),
|
|
1990
2578
|
invocation: canonicalize(invocation),
|
|
1991
2579
|
terminal_authority_receipt: canonicalize(terminal_authority_receipt),
|
|
@@ -2047,6 +2635,7 @@ export class RailEngine {
|
|
|
2047
2635
|
|| entry.terminal_authority_receipt.stdout_digest !== sha256(entry.adjudication_output)) {
|
|
2048
2636
|
fail('ADJUDICATION_TERMINAL_AUTHORITY_MISMATCH', 'adjudication completion does not bind its canonical launch and full stdout');
|
|
2049
2637
|
}
|
|
2638
|
+
assertTerminalCompletionContainment(entry.terminal_authority_receipt, dispatch.payload.launch_receipt, 'ADJUDICATION_TERMINAL_AUTHORITY_MISMATCH');
|
|
2050
2639
|
const parsedDecision = parseAdjudicationOutput(entry.adjudication_output);
|
|
2051
2640
|
if (sha256(parsedDecision) !== sha256(validateAdjudicationDecision(entry.decision))) {
|
|
2052
2641
|
fail('ADJUDICATION_PAYLOAD_MISMATCH', 'adjudication decision differs from terminal-authorized stdout');
|
|
@@ -2120,8 +2709,9 @@ export class RailEngine {
|
|
|
2120
2709
|
});
|
|
2121
2710
|
}
|
|
2122
2711
|
|
|
2123
|
-
recoverAttempt({ attempt_id, operator_id, harness_process_id, git_dir, lease_ttl_ms = 300000 } = {}) {
|
|
2712
|
+
recoverAttempt({ attempt_id, operator_id, harness_process_id, git_dir, lease_ttl_ms = 300000, expected_attempt_event_id } = {}) {
|
|
2124
2713
|
return this.#exclusive(() => {
|
|
2714
|
+
this.#assertAttemptRevision(attempt_id, expected_attempt_event_id);
|
|
2125
2715
|
for (const [field, value] of Object.entries({ attempt_id, operator_id, harness_process_id })) assertString(value, 'INVALID_RECOVERY', field);
|
|
2126
2716
|
assertString(git_dir, 'INVALID_RECOVERY', 'git_dir');
|
|
2127
2717
|
const attempt = this.#attempts().get(attempt_id);
|
|
@@ -2136,23 +2726,60 @@ export class RailEngine {
|
|
|
2136
2726
|
? claimProcessPid(attempt)
|
|
2137
2727
|
: observeStoppedClaimProcess(attempt);
|
|
2138
2728
|
const successfulExecution = this.#executionEvidence(attempt_id, { successfulOnly: true }).at(-1);
|
|
2139
|
-
const expectedRecoveryCommit = successfulExecution?.result_commit_ref ?? attempt
|
|
2729
|
+
const expectedRecoveryCommit = successfulExecution?.result_commit_ref ?? attemptExecutionBase(attempt);
|
|
2140
2730
|
verifyLocalGitCommit({ git_dir, git_commit_ref: attempt.git_commit_ref });
|
|
2141
2731
|
verifyLocalGitCommit({ git_dir, git_commit_ref: expectedRecoveryCommit });
|
|
2732
|
+
const executionAdmissions = [];
|
|
2733
|
+
const reconcileAdmission = (record, legacyCode) => {
|
|
2734
|
+
if (!record.payload.admission_receipt) {
|
|
2735
|
+
fail(legacyCode, 'automatic recovery cannot prove whether a legacy reserved provider launch started before the supervisor stopped', {
|
|
2736
|
+
launch_nonces: [record.payload.invocation.launch_nonce],
|
|
2737
|
+
});
|
|
2738
|
+
}
|
|
2739
|
+
if (record.payload.claim_event_id !== attempt.claim_event_id
|
|
2740
|
+
|| record.payload.harness_process_id !== attempt.claim.harness_process_id) {
|
|
2741
|
+
fail('RECOVERY_ADMISSION_CLAIM_MISMATCH', 'execution admission belongs to another supervisor claim generation');
|
|
2742
|
+
}
|
|
2743
|
+
// A relinquished lease alone cannot prove an admission callback has
|
|
2744
|
+
// stopped. The old owner must actually be gone before its nonce closes.
|
|
2745
|
+
observeStoppedClaimProcess(attempt);
|
|
2746
|
+
this.#verifyExecutionAdmission({ attempt, invocation: record.payload.invocation,
|
|
2747
|
+
receipt: record.payload.admission_receipt, project_dir: git_dir,
|
|
2748
|
+
harness_process_id: record.payload.harness_process_id });
|
|
2749
|
+
executionAdmissions.push({ status: 'not_started', launch_nonce: record.payload.invocation.launch_nonce,
|
|
2750
|
+
reservation_event_id: record.event_id, admission_receipt_digest: sha256(record.payload.admission_receipt) });
|
|
2751
|
+
};
|
|
2142
2752
|
const unresolvedLaunchReservations = this.#records.filter((record) => record.type === 'execution_launch_reserved'
|
|
2143
2753
|
&& record.payload?.attempt_id === attempt_id
|
|
2754
|
+
&& !this.#launchReconciled(record.payload.invocation.launch_nonce)
|
|
2144
2755
|
&& !this.#records.some((candidate) => candidate.type === 'execution_dispatched'
|
|
2145
2756
|
&& candidate.payload?.invocation?.launch_nonce === record.payload.invocation.launch_nonce));
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2757
|
+
for (const record of unresolvedLaunchReservations) reconcileAdmission(record, 'RECOVERY_EXECUTION_LAUNCH_UNRESOLVED');
|
|
2758
|
+
const executionDispatches = [];
|
|
2759
|
+
const dispatchResolved = (dispatch) => {
|
|
2760
|
+
if (this.#launchReconciled(dispatch.payload.invocation.launch_nonce)) return true;
|
|
2761
|
+
if (this.#records.some((record) => record.type === 'execution_recorded'
|
|
2762
|
+
&& record.payload?.execution_evidence?.receipt_nonce === dispatch.payload.invocation.launch_nonce)) return true;
|
|
2763
|
+
const proof = this.#restorationProof(dispatch, { current: true });
|
|
2764
|
+
if (!proof) return false;
|
|
2765
|
+
const reservation = this.#records.find((record) => ['execution_launch_reserved', 'rework_launch_reserved'].includes(record.type)
|
|
2766
|
+
&& record.payload.attempt_id === attempt_id && record.payload.invocation.launch_nonce === proof.launch_nonce);
|
|
2767
|
+
if (!reservation || reservation.payload.claim_event_id !== attempt.claim_event_id
|
|
2768
|
+
|| reservation.payload.harness_process_id !== attempt.claim.harness_process_id
|
|
2769
|
+
|| dispatch.payload.claim_id !== attempt.claim.claim_id || dispatch.payload.operator_id !== attempt.claim.operator_id) {
|
|
2770
|
+
fail('RECOVERY_ADMISSION_CLAIM_MISMATCH', 'restoration belongs to another supervisor claim generation');
|
|
2771
|
+
}
|
|
2772
|
+
observeStoppedClaimProcess(attempt);
|
|
2773
|
+
this.#verifyExecutionAdmission({ attempt, invocation: dispatch.payload.invocation,
|
|
2774
|
+
receipt: reservation.payload.admission_receipt, project_dir: git_dir, harness_process_id: attempt.claim.harness_process_id });
|
|
2775
|
+
this.#assertAdmissionDispatchBound(reservation, dispatch.payload.launch_receipt);
|
|
2776
|
+
executionDispatches.push(proof);
|
|
2777
|
+
return true;
|
|
2778
|
+
};
|
|
2151
2779
|
const unresolvedInitialDispatches = this.#records.filter((record) => record.type === 'execution_dispatched'
|
|
2152
2780
|
&& record.payload?.attempt_id === attempt_id
|
|
2153
2781
|
&& record.payload?.run_kind === 'initial'
|
|
2154
|
-
&& !
|
|
2155
|
-
&& candidate.payload?.execution_evidence?.receipt_nonce === record.payload.invocation.launch_nonce));
|
|
2782
|
+
&& !dispatchResolved(record));
|
|
2156
2783
|
if (unresolvedInitialDispatches.length > 0) {
|
|
2157
2784
|
fail('RECOVERY_EXECUTION_DISPATCH_UNRESOLVED', 'automatic recovery cannot authorize another mutating provider after an interrupted initial dispatch', {
|
|
2158
2785
|
launch_nonces: unresolvedInitialDispatches.map((record) => record.payload.invocation.launch_nonce),
|
|
@@ -2160,18 +2787,14 @@ export class RailEngine {
|
|
|
2160
2787
|
}
|
|
2161
2788
|
const unresolvedReworkLaunches = this.#records.filter((record) => record.type === 'rework_launch_reserved'
|
|
2162
2789
|
&& record.payload?.attempt_id === attempt_id
|
|
2790
|
+
&& !this.#launchReconciled(record.payload.invocation.launch_nonce)
|
|
2163
2791
|
&& !this.#records.some((candidate) => candidate.type === 'execution_dispatched'
|
|
2164
2792
|
&& candidate.payload?.invocation?.launch_nonce === record.payload.invocation.launch_nonce));
|
|
2165
|
-
|
|
2166
|
-
fail('RECOVERY_REWORK_LAUNCH_UNRESOLVED', 'automatic recovery cannot prove whether a reserved rework launch started before the supervisor stopped', {
|
|
2167
|
-
launch_nonces: unresolvedReworkLaunches.map((record) => record.payload.invocation.launch_nonce),
|
|
2168
|
-
});
|
|
2169
|
-
}
|
|
2793
|
+
for (const record of unresolvedReworkLaunches) reconcileAdmission(record, 'RECOVERY_REWORK_LAUNCH_UNRESOLVED');
|
|
2170
2794
|
const unresolvedReworkDispatches = this.#records.filter((record) => record.type === 'execution_dispatched'
|
|
2171
2795
|
&& record.payload?.attempt_id === attempt_id
|
|
2172
2796
|
&& record.payload?.run_kind === 'rework'
|
|
2173
|
-
&& !
|
|
2174
|
-
&& candidate.payload?.execution_evidence?.receipt_nonce === record.payload.invocation.launch_nonce));
|
|
2797
|
+
&& !dispatchResolved(record));
|
|
2175
2798
|
if (unresolvedReworkDispatches.length > 0) {
|
|
2176
2799
|
fail('RECOVERY_REWORK_DISPATCH_UNRESOLVED', 'automatic recovery cannot authorize another mutating provider after an interrupted rework dispatch', {
|
|
2177
2800
|
launch_nonces: unresolvedReworkDispatches.map((record) => record.payload.invocation.launch_nonce),
|
|
@@ -2199,12 +2822,15 @@ export class RailEngine {
|
|
|
2199
2822
|
? { status: priorCompletion.payload.completion.ci.verdict, evidence_refs: priorCompletion.payload.completion.ci.evidence_refs }
|
|
2200
2823
|
: { status: 'not_started', evidence_refs: [`rail-observation://ci/${attempt_id}/not-started`] },
|
|
2201
2824
|
effects: effects.map((effect) => ({ idempotency_key: effect.idempotency_key, status: 'receipted', receipt_ref: effect.receipt_ref })),
|
|
2202
|
-
execution_dispatches:
|
|
2825
|
+
execution_dispatches: executionDispatches,
|
|
2826
|
+
...(executionAdmissions.length ? { execution_admissions: executionAdmissions } : {}),
|
|
2203
2827
|
});
|
|
2204
2828
|
const claim = canonicalize({
|
|
2205
2829
|
...attempt.claim,
|
|
2206
2830
|
operator_id,
|
|
2207
2831
|
harness_process_id,
|
|
2832
|
+
hostname: hostname(),
|
|
2833
|
+
...(process.platform === 'darwin' ? { darwin_host_boot_identity: captureDarwinHostBootIdentity() } : {}),
|
|
2208
2834
|
lease: { acquired_at: now, heartbeat_at: now, expires_at: plusMs(now, lease_ttl_ms) },
|
|
2209
2835
|
});
|
|
2210
2836
|
const resume_state = attempt.state;
|
|
@@ -2238,11 +2864,30 @@ export class RailEngine {
|
|
|
2238
2864
|
assertObject(metrics[field], 'INVALID_COMPLETION_METRICS', `metrics.${field}`);
|
|
2239
2865
|
assertString(metrics[field].unit, 'INVALID_COMPLETION_METRICS', `metrics.${field}.unit`);
|
|
2240
2866
|
assertString(metrics[field].evidence_ref, 'INVALID_COMPLETION_METRICS', `metrics.${field}.evidence_ref`);
|
|
2241
|
-
|
|
2867
|
+
const unknownProcessing = field === 'actual_ai_processing' && metrics[field].value === null
|
|
2868
|
+
&& metrics[field].coverage?.status === 'reconciled_incomplete';
|
|
2869
|
+
if (!unknownProcessing && (typeof metrics[field].value !== 'number' || !Number.isFinite(metrics[field].value) || metrics[field].value < 0)) fail('INVALID_COMPLETION_METRICS', `metrics.${field}.value must be a non-negative finite number or verified incomplete processing`);
|
|
2870
|
+
}
|
|
2871
|
+
if (metrics.actual_ai_processing.coverage !== undefined || this.#records.some((record) =>
|
|
2872
|
+
record.type === 'execution_worktree_restored' && record.payload.attempt_id === attempt_id)) {
|
|
2873
|
+
const processing = collectProviderProcessing(this.#records, { attempt_id,
|
|
2874
|
+
verifyAuthority: (kind, receipt, expected, context) => this.#verifyAuthority(kind, receipt, expected, context) });
|
|
2875
|
+
if (sha256(metrics.actual_ai_processing) !== sha256(providerProcessingMetric(processing, metrics.actual_ai_processing.evidence_ref))) {
|
|
2876
|
+
fail('COMPLETION_PROCESSING_COVERAGE_MISMATCH', 'processing values, subtotal and unknown invocation proofs must match the canonical journal');
|
|
2877
|
+
}
|
|
2242
2878
|
}
|
|
2243
2879
|
if (!Number.isInteger(metrics.rework_cycles) || metrics.rework_cycles < 0) fail('INVALID_COMPLETION_METRICS', 'metrics.rework_cycles must be a non-negative integer');
|
|
2244
2880
|
assertStringArray(metrics.rework_review_refs, 'INVALID_COMPLETION_METRICS', 'metrics.rework_review_refs');
|
|
2245
2881
|
if (metrics.rework_review_refs.length !== metrics.rework_cycles) fail('INVALID_COMPLETION_METRICS', 'each rework cycle requires one review reference');
|
|
2882
|
+
const metricRefs = [metrics.actual_human_effort.evidence_ref, metrics.actual_ai_processing.evidence_ref,
|
|
2883
|
+
completionMetricsReference({ attempt_id, commit_ref: git.result_ref, metrics })];
|
|
2884
|
+
if (!metricRefs.every((ref) => acceptance_evidence_refs.includes(ref))) {
|
|
2885
|
+
fail('COMPLETION_METRICS_BINDING_REQUIRED', 'certified acceptance must include metric sources and the exact attempt, commit, values, units and rework commitment');
|
|
2886
|
+
}
|
|
2887
|
+
if (metricRefs.slice(0, 2).some((ref) => ref.startsWith('rail-verification://sha256/'))
|
|
2888
|
+
&& metrics.source_journal_digest !== effortJournalDigest(this.#records, attempt_id)) {
|
|
2889
|
+
fail('COMPLETION_METRICS_STALE', 'local observed metrics must cover the current effort and provider journal inside the completion transaction');
|
|
2890
|
+
}
|
|
2246
2891
|
}
|
|
2247
2892
|
const existing = this.#records.find((record) => record.type === 'attempt_completed' && record.payload.completion.completion_id === completion_id);
|
|
2248
2893
|
if (existing) fail('COMPLETION_ALREADY_EXISTS', 'completion_id is immutable', { completion_id });
|
|
@@ -2264,7 +2909,7 @@ export class RailEngine {
|
|
|
2264
2909
|
commitRef: git.result_ref,
|
|
2265
2910
|
acceptanceEvidenceRefs: acceptance_evidence_refs,
|
|
2266
2911
|
delegationManifest: browserDispatch?.payload?.browser_manifest,
|
|
2267
|
-
verifyAuthority: (receipt, expected) => this.#verifyAuthority('browser_evidence', receipt, expected),
|
|
2912
|
+
verifyAuthority: (receipt, expected, context) => this.#verifyAuthority('browser_evidence', receipt, expected, context),
|
|
2268
2913
|
});
|
|
2269
2914
|
} else if (browser_evidence !== undefined) {
|
|
2270
2915
|
fail('UNEXPECTED_BROWSER_EVIDENCE', 'non-browser task completion must not claim real browser evidence');
|
|
@@ -2273,7 +2918,7 @@ export class RailEngine {
|
|
|
2273
2918
|
this.#assertLiveLease(attempt, timestamp(this.#clock));
|
|
2274
2919
|
assertString(git.base_ref, 'INVALID_COMPLETION', 'git.base_ref');
|
|
2275
2920
|
assertString(git.result_ref, 'INVALID_COMPLETION', 'git.result_ref');
|
|
2276
|
-
if (git.base_ref !== attempt
|
|
2921
|
+
if (git.base_ref !== attemptExecutionBase(attempt)) fail('GIT_BASE_MISMATCH', 'completion base_ref must equal the journal-bound attempt execution base', { attempt_id });
|
|
2277
2922
|
if (!git.commit_refs.includes(git.base_ref) || !git.commit_refs.includes(git.result_ref)) {
|
|
2278
2923
|
fail('GIT_COMMIT_MISMATCH', 'completion commit_refs must include both sealed base and reviewed result commits', { attempt_id });
|
|
2279
2924
|
}
|