onto-mcp 0.4.12 → 0.4.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.onto/authority/core-lexicon.yaml +1 -1
- package/.onto/authority/supported-models.yaml +137 -0
- package/.onto/domains/accounting-kr/competency_qs.md +153 -153
- package/.onto/processes/reconstruct/obligation-coverage-ledger.yaml +842 -0
- package/.onto/processes/reconstruct/obligation-coverage-recorded.yaml +225 -0
- package/.onto/processes/reconstruct/ontology-seeding-and-maturation-design.md +93 -54
- package/.onto/processes/reconstruct/reconstruct-contract-registry.yaml +169 -8
- package/.onto/processes/reconstruct/reconstruct-execution-ux-contract.md +72 -1
- package/.onto/processes/reconstruct/source-profiles/spreadsheet.md +35 -0
- package/.onto/processes/review/productized-live-path.md +9 -5
- package/.onto/processes/review/prompt-execution-runner-contract.md +32 -8
- package/.onto/processes/review/review-target-profile-contract.md +47 -4
- package/.onto/processes/shared/pipeline-execution-ledger-contract.md +11 -0
- package/.onto/processes/shared/target-material-kind-contract.md +1 -1
- package/README.md +64 -1
- package/dist/cli.js +11 -1
- package/dist/core-api/reconstruct-api.js +575 -24
- package/dist/core-api/review-api.js +70 -5
- package/dist/core-api/review-progress.js +7 -0
- package/dist/core-api/runtime-observation.js +7 -0
- package/dist/core-runtime/artifact-io.js +75 -3
- package/dist/core-runtime/cli/claude-code-review-unit-executor.js +19 -8
- package/dist/core-runtime/cli/materialize-review-prompt-packets.js +68 -14
- package/dist/core-runtime/cli/prepare-review-session.js +34 -1
- package/dist/core-runtime/cli/render-review-final-output.js +2 -1
- package/dist/core-runtime/cli/run-review-prompt-execution.js +758 -48
- package/dist/core-runtime/cli/run-seats.js +18 -0
- package/dist/core-runtime/cli/runtime-submit-context.js +10 -2
- package/dist/core-runtime/cli/unit-resubmit.js +269 -0
- package/dist/core-runtime/cli/worker-structured-output.js +80 -2
- package/dist/core-runtime/discovery/host-detection.js +71 -22
- package/dist/core-runtime/discovery/review-cert-assemble.js +208 -0
- package/dist/core-runtime/discovery/review-cert-record.js +442 -0
- package/dist/core-runtime/discovery/review-execution-units.js +22 -0
- package/dist/core-runtime/discovery/seat-inventory.js +167 -0
- package/dist/core-runtime/discovery/settings-chain.js +311 -52
- package/dist/core-runtime/discovery/supported-models.js +282 -29
- package/dist/core-runtime/discovery/synthesize-cert-assemble.js +197 -0
- package/dist/core-runtime/discovery/synthesize-cert-capsule.js +399 -0
- package/dist/core-runtime/discovery/synthesize-cert-judge.js +340 -0
- package/dist/core-runtime/discovery/synthesize-cert-loop.js +234 -0
- package/dist/core-runtime/discovery/synthesize-cert-mutation.js +117 -0
- package/dist/core-runtime/discovery/synthesize-cert-packet.js +182 -0
- package/dist/core-runtime/discovery/synthesize-cert-record.js +831 -0
- package/dist/core-runtime/discovery/synthesize-cert-sampler.js +336 -0
- package/dist/core-runtime/effort-calibration-ingest.js +135 -0
- package/dist/core-runtime/llm/dispatch-breaker.js +412 -0
- package/dist/core-runtime/llm/dispatch-fallback-adapter-capabilities.js +33 -0
- package/dist/core-runtime/llm/llm-caller.js +188 -47
- package/dist/core-runtime/llm/mock-llm-realization.js +35 -4
- package/dist/core-runtime/llm/model-switcher.js +1 -1
- package/dist/core-runtime/llm/openai-responses-incomplete-error.js +46 -0
- package/dist/core-runtime/llm/sealed-dispatch-capability.js +359 -0
- package/dist/core-runtime/llm/structured-dispatch-error.js +15 -0
- package/dist/core-runtime/onboard/claude-profile-scan.js +65 -0
- package/dist/core-runtime/onboard/cli-host.js +5 -2
- package/dist/core-runtime/onboard/configure-provider.js +33 -1
- package/dist/core-runtime/onboard/host-target.js +28 -3
- package/dist/core-runtime/onboard/register.js +54 -12
- package/dist/core-runtime/pipeline-execution-ledger.js +10 -0
- package/dist/core-runtime/reconstruct/artifact-types.js +35 -0
- package/dist/core-runtime/reconstruct/claim-projection-validation.js +17 -0
- package/dist/core-runtime/reconstruct/competency-projection-contract.js +63 -0
- package/dist/core-runtime/reconstruct/comprehension-artifact.js +321 -0
- package/dist/core-runtime/reconstruct/comprehension-reduce.js +367 -0
- package/dist/core-runtime/reconstruct/comprehension-semantic-map.js +771 -0
- package/dist/core-runtime/reconstruct/contract-registry.js +20 -0
- package/dist/core-runtime/reconstruct/dispatch-fallback-artifacts.js +640 -0
- package/dist/core-runtime/reconstruct/execution-telemetry.js +83 -9
- package/dist/core-runtime/reconstruct/final-output-sections.js +89 -0
- package/dist/core-runtime/reconstruct/governing-snapshot.js +36 -1
- package/dist/core-runtime/reconstruct/leaf-reader.js +305 -0
- package/dist/core-runtime/reconstruct/llm-dispatch-failure.js +270 -0
- package/dist/core-runtime/reconstruct/llm-touch-fingerprint.js +94 -0
- package/dist/core-runtime/reconstruct/material-admission-validation.js +18 -0
- package/dist/core-runtime/reconstruct/material-profile-validation.js +14 -0
- package/dist/core-runtime/reconstruct/materialize-preparation.js +304 -18
- package/dist/core-runtime/reconstruct/maturation-validation.js +1231 -53
- package/dist/core-runtime/reconstruct/mock-llm-realization.js +107 -3
- package/dist/core-runtime/reconstruct/obligation-assertion.js +18 -0
- package/dist/core-runtime/reconstruct/ontology-seed-validation.js +27 -0
- package/dist/core-runtime/reconstruct/output-budget.js +6 -0
- package/dist/core-runtime/reconstruct/pipeline-execution-ledger.js +87 -3
- package/dist/core-runtime/reconstruct/post-seed-validation.js +118 -5
- package/dist/core-runtime/reconstruct/purpose-authority-validation.js +117 -2
- package/dist/core-runtime/reconstruct/record.js +42 -0
- package/dist/core-runtime/reconstruct/registry-verification-validation.js +16 -0
- package/dist/core-runtime/reconstruct/run-control-validation.js +1011 -34
- package/dist/core-runtime/reconstruct/run.js +5198 -536
- package/dist/core-runtime/reconstruct/seed-authoring-readiness-validation.js +54 -3
- package/dist/core-runtime/reconstruct/semantic-quality-gate.js +105 -2
- package/dist/core-runtime/reconstruct/source-observation-delta-validation.js +41 -0
- package/dist/core-runtime/reconstruct/source-observations.js +166 -0
- package/dist/core-runtime/reconstruct/source-safety-validation.js +71 -183
- package/dist/core-runtime/reconstruct/source-scout-pack-validation.js +37 -1
- package/dist/core-runtime/reconstruct/terminal-validation.js +132 -2
- package/dist/core-runtime/review/continuation-plan.js +12 -5
- package/dist/core-runtime/review/issue-artifact-runtime.js +4 -1
- package/dist/core-runtime/review/materializers.js +100 -11
- package/dist/core-runtime/review/nesting-batch.js +6 -2
- package/dist/core-runtime/review/obligation-shardability.js +138 -0
- package/dist/core-runtime/review/pipeline-execution-ledger.js +42 -2
- package/dist/core-runtime/review/review-artifact-utils.js +217 -5
- package/dist/core-runtime/review/review-execution-steps.js +32 -8
- package/dist/core-runtime/review/review-progress-contract.js +8 -0
- package/dist/core-runtime/review/review-prompt-budget.js +97 -0
- package/dist/core-runtime/review/semantic-quality-gate.js +33 -7
- package/dist/core-runtime/review/spreadsheet-review-disposition.js +94 -0
- package/dist/core-runtime/route-identity.js +182 -0
- package/dist/core-runtime/spreadsheet-header-escalation.js +204 -0
- package/dist/core-runtime/spreadsheet-structure-observer.js +2689 -0
- package/dist/core-runtime/target-material-kind.js +58 -0
- package/dist/mcp/server.js +12 -5
- package/dist/tui/app.js +167 -0
- package/dist/tui/data/event-follower.js +116 -0
- package/dist/tui/data/node-detail.js +42 -0
- package/dist/tui/data/projection-poll.js +26 -0
- package/dist/tui/data/session-discovery.js +65 -0
- package/dist/tui/index.js +108 -0
- package/dist/tui/screens/log.js +25 -0
- package/dist/tui/screens/node-detail.js +17 -0
- package/dist/tui/screens/session-selector.js +26 -0
- package/dist/tui/screens/workflow-tree.js +90 -0
- package/dist/tui/view-model/reconstruct-adapter.js +162 -0
- package/dist/tui/view-model/review-adapter.js +270 -0
- package/dist/tui/view-model/tree-view-model.js +19 -0
- package/package.json +18 -3
- package/settings.example.json +77 -16
|
@@ -1,14 +1,63 @@
|
|
|
1
1
|
import crypto from "node:crypto";
|
|
2
|
+
import { constants as fsConstants } from "node:fs";
|
|
2
3
|
import fs from "node:fs/promises";
|
|
3
4
|
import path from "node:path";
|
|
5
|
+
import lockfile from "proper-lockfile";
|
|
4
6
|
import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
|
|
5
|
-
import {
|
|
7
|
+
import { z } from "zod";
|
|
8
|
+
import { assertArrayField, atomicWriteYamlDocument as writeYamlDocument, durableAtomicWriteYamlDocument as writeRunControlDocument, } from "../artifact-io.js";
|
|
9
|
+
import { assertObligation } from "./obligation-assertion.js";
|
|
10
|
+
import { assertDispatchFallbackRunControlHasNoLiveOwner, assertDispatchFallbackSessionAdmission, } from "./dispatch-fallback-artifacts.js";
|
|
11
|
+
import { RECONSTRUCT_LLM_DISPATCH_FAILURE_DIR, ReconstructLlmDispatchFailureArtifactSchema, assertReconstructLlmDispatchFailureDirectory, createReconstructLlmDispatchFailureArtifact, isReconstructLlmDispatchFailureRef, isReconstructLlmDispatchFailureTempRef, planReconstructLlmDispatchFailureWrite, publishReconstructLlmDispatchFailureTemp, readReconstructLlmDispatchFailureArtifact, readReconstructLlmDispatchFailureArtifactWithHash, reconstructLlmDispatchFailurePath, sha256ReconstructLlmDispatchFailureArtifact, writeReconstructLlmDispatchFailureTemp, } from "./llm-dispatch-failure.js";
|
|
6
12
|
function isoNow() {
|
|
7
13
|
return new Date().toISOString();
|
|
8
14
|
}
|
|
9
15
|
function sha256(value) {
|
|
10
16
|
return crypto.createHash("sha256").update(value).digest("hex");
|
|
11
17
|
}
|
|
18
|
+
const runControlMutationQueues = new Map();
|
|
19
|
+
async function withRunControlMutationLock(runControlPath, task) {
|
|
20
|
+
const key = path.resolve(runControlPath);
|
|
21
|
+
const previous = runControlMutationQueues.get(key) ?? Promise.resolve();
|
|
22
|
+
let releaseQueue = () => { };
|
|
23
|
+
const current = new Promise((resolve) => {
|
|
24
|
+
releaseQueue = resolve;
|
|
25
|
+
});
|
|
26
|
+
const queued = previous.then(() => current);
|
|
27
|
+
runControlMutationQueues.set(key, queued);
|
|
28
|
+
await previous;
|
|
29
|
+
const lockPath = `${key}.write-lock`;
|
|
30
|
+
let releaseFileLock = null;
|
|
31
|
+
try {
|
|
32
|
+
await fs.mkdir(path.dirname(key), { recursive: true });
|
|
33
|
+
releaseFileLock = await lockfile.lock(key, {
|
|
34
|
+
lockfilePath: lockPath,
|
|
35
|
+
realpath: false,
|
|
36
|
+
stale: 5_000,
|
|
37
|
+
update: 1_000,
|
|
38
|
+
retries: {
|
|
39
|
+
retries: 600,
|
|
40
|
+
factor: 1,
|
|
41
|
+
minTimeout: 50,
|
|
42
|
+
maxTimeout: 50,
|
|
43
|
+
randomize: false,
|
|
44
|
+
},
|
|
45
|
+
});
|
|
46
|
+
return await task();
|
|
47
|
+
}
|
|
48
|
+
finally {
|
|
49
|
+
try {
|
|
50
|
+
if (releaseFileLock)
|
|
51
|
+
await releaseFileLock();
|
|
52
|
+
}
|
|
53
|
+
finally {
|
|
54
|
+
releaseQueue();
|
|
55
|
+
if (runControlMutationQueues.get(key) === queued) {
|
|
56
|
+
runControlMutationQueues.delete(key);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
12
61
|
async function sha256File(filePath) {
|
|
13
62
|
try {
|
|
14
63
|
return crypto
|
|
@@ -25,9 +74,26 @@ async function sha256File(filePath) {
|
|
|
25
74
|
async function writeYamlDocumentAtomicCreate(filePath, value) {
|
|
26
75
|
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
27
76
|
const tempPath = path.join(path.dirname(filePath), `.${path.basename(filePath)}.${process.pid}.${crypto.randomUUID()}.tmp`);
|
|
28
|
-
await fs.
|
|
77
|
+
const handle = await fs.open(tempPath, fsConstants.O_CREAT |
|
|
78
|
+
fsConstants.O_EXCL |
|
|
79
|
+
fsConstants.O_WRONLY |
|
|
80
|
+
fsConstants.O_NOFOLLOW, 0o600);
|
|
81
|
+
try {
|
|
82
|
+
await handle.writeFile(stringifyYaml(value), "utf8");
|
|
83
|
+
await handle.sync();
|
|
84
|
+
}
|
|
85
|
+
finally {
|
|
86
|
+
await handle.close();
|
|
87
|
+
}
|
|
29
88
|
try {
|
|
30
89
|
await fs.link(tempPath, filePath);
|
|
90
|
+
const directoryHandle = await fs.open(path.dirname(filePath), fsConstants.O_RDONLY);
|
|
91
|
+
try {
|
|
92
|
+
await directoryHandle.sync();
|
|
93
|
+
}
|
|
94
|
+
finally {
|
|
95
|
+
await directoryHandle.close();
|
|
96
|
+
}
|
|
31
97
|
return true;
|
|
32
98
|
}
|
|
33
99
|
catch (error) {
|
|
@@ -136,7 +202,10 @@ function violation(args) {
|
|
|
136
202
|
function requiresTerminalValidationTrust(runControl) {
|
|
137
203
|
assertArrayField(runControl.attempt_rows, "run-control", "attempt_rows");
|
|
138
204
|
assertArrayField(runControl.resume_rows, "run-control", "resume_rows");
|
|
139
|
-
|
|
205
|
+
// A graceful-terminal `halted` attempt (design §16.6) is a terminal outcome that produced its
|
|
206
|
+
// own terminal manifest validation, so it demands the same terminal-validation trust as a
|
|
207
|
+
// completed attempt — it must not slip past the fail-closed authority check.
|
|
208
|
+
return runControl.attempt_rows.some((row) => row.attempt_status === "completed" || row.attempt_status === "halted") || runControl.resume_rows.some((row) => row.resume_decision === "resume_allowed");
|
|
140
209
|
}
|
|
141
210
|
function inferTerminalValidationRef(args) {
|
|
142
211
|
assertArrayField(args.runControl.write_transactions, "run-control", "write_transactions");
|
|
@@ -161,6 +230,28 @@ export function validateReconstructRunControl(args) {
|
|
|
161
230
|
assertArrayField(args.runControl.write_transactions, "run-control", "write_transactions");
|
|
162
231
|
assertArrayField(args.runControl.resume_rows, "run-control", "resume_rows");
|
|
163
232
|
const violations = [];
|
|
233
|
+
// G(a) slice 10: record the one obligation whose enforcement matches the authoritative contract
|
|
234
|
+
// (ontology-seeding-and-maturation-design.md §"reconstruct-run-control-validation.yaml must prove")
|
|
235
|
+
// — committed write transactions carry artifact_refs and hashes — stamped before the per-transaction
|
|
236
|
+
// loop so it fires on a zero-transaction artifact. The block below trips invalid_transaction (missing
|
|
237
|
+
// artifact_ref/owner_attempt_id) and transaction_hash_missing (committed without committed_hash).
|
|
238
|
+
// PARKED (not recorded), each name broader than this validator's actual check — these are honest
|
|
239
|
+
// declared≠wired surfacings, NOT laundered (see obligation-coverage-ledger.yaml notes):
|
|
240
|
+
// - preserve_post_write_hash_observation_without_claiming_atomic_commit_when_writer_did_not_prove_atomic_rename:
|
|
241
|
+
// the validator never reads commit_method; the atomic_rename-vs-observed_file_hash truthfulness
|
|
242
|
+
// is the writer's responsibility, not validated here.
|
|
243
|
+
// - reject_conflicting_request_fingerprints_before_semantic_artifacts_are_consumed: the validator
|
|
244
|
+
// rejects rows already marked duplicate_conflict/rejected_conflict (conflicting_request) but never
|
|
245
|
+
// compares request_fingerprint values — fingerprint conflict detection is in the writer.
|
|
246
|
+
// - validate_current_attempt_and_session_root_lock_ownership: presence of an active attempt
|
|
247
|
+
// (active_attempt_missing) + a session_root lock (session_lock_missing) + rejection of recorded
|
|
248
|
+
// conflicting locks (conflicting_lock) are checked, but the lock's owner_attempt_id is never
|
|
249
|
+
// cross-checked against the current attempt — the named ownership linkage is unverified.
|
|
250
|
+
// - validate_session_root_request_fingerprint_target_signature_runtime_version_and_idempotency_are_replayable:
|
|
251
|
+
// of the five named replayable quantities only session_root is validated; request_fingerprint,
|
|
252
|
+
// target signature, runtime version, and idempotency key are never inspected.
|
|
253
|
+
const assertedObligationIds = [];
|
|
254
|
+
assertObligation(assertedObligationIds, "validate_committed_write_transactions_have_artifact_refs_and_hashes");
|
|
164
255
|
if (args.runControl.schema_version !== "1") {
|
|
165
256
|
violations.push(violation({
|
|
166
257
|
code: "schema_shape_invalid",
|
|
@@ -208,15 +299,89 @@ export function validateReconstructRunControl(args) {
|
|
|
208
299
|
message: "run-control must record at least one attempt row",
|
|
209
300
|
}));
|
|
210
301
|
}
|
|
211
|
-
const
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
row
|
|
215
|
-
|
|
302
|
+
const duplicateIds = (rows, field) => {
|
|
303
|
+
const seen = new Set();
|
|
304
|
+
const duplicates = new Set();
|
|
305
|
+
for (const row of rows) {
|
|
306
|
+
const value = row[field];
|
|
307
|
+
if (typeof value !== "string")
|
|
308
|
+
continue;
|
|
309
|
+
if (seen.has(value))
|
|
310
|
+
duplicates.add(value);
|
|
311
|
+
seen.add(value);
|
|
312
|
+
}
|
|
313
|
+
return [...duplicates];
|
|
314
|
+
};
|
|
315
|
+
for (const [label, rows, field] of [
|
|
316
|
+
["request", args.runControl.request_rows, "request_id"],
|
|
317
|
+
["attempt", args.runControl.attempt_rows, "attempt_id"],
|
|
318
|
+
["lock", args.runControl.lock_rows, "lock_id"],
|
|
319
|
+
["transaction", args.runControl.write_transactions, "transaction_id"],
|
|
320
|
+
["resume", args.runControl.resume_rows, "resume_id"],
|
|
321
|
+
]) {
|
|
322
|
+
for (const duplicateId of duplicateIds(rows, field)) {
|
|
323
|
+
violations.push(violation({
|
|
324
|
+
code: "schema_shape_invalid",
|
|
325
|
+
message: `run-control ${label} ids must be unique`,
|
|
326
|
+
subjectId: duplicateId,
|
|
327
|
+
}));
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
const latestAttempt = args.runControl.attempt_rows.at(-1) ?? null;
|
|
331
|
+
let trustedFailedAttempt = false;
|
|
332
|
+
if (latestAttempt?.attempt_status === "failed") {
|
|
333
|
+
const failureTransactions = args.runControl.write_transactions.filter((row) => row.owner_attempt_id === latestAttempt.attempt_id &&
|
|
334
|
+
isReconstructLlmDispatchFailureRef(args.runControl.session_root, row.artifact_ref));
|
|
335
|
+
const transaction = failureTransactions.length === 1
|
|
336
|
+
? failureTransactions[0]
|
|
337
|
+
: undefined;
|
|
338
|
+
const parsedArtifact = ReconstructLlmDispatchFailureArtifactSchema.safeParse(args.failedTerminalArtifact);
|
|
339
|
+
const ownerSessionLocks = args.runControl.lock_rows.filter((row) => row.lock_scope === "session_root" &&
|
|
340
|
+
row.owner_attempt_id === latestAttempt.attempt_id);
|
|
341
|
+
if (!transaction || args.failedTerminalArtifactRef === undefined) {
|
|
342
|
+
violations.push(violation({
|
|
343
|
+
code: "failed_terminal_missing",
|
|
344
|
+
message: "latest failed attempt requires an owner-linked LLM dispatch failure transaction",
|
|
345
|
+
subjectId: latestAttempt.attempt_id,
|
|
346
|
+
}));
|
|
347
|
+
}
|
|
348
|
+
else if (transaction.transaction_status !== "committed" ||
|
|
349
|
+
!transaction.committed_hash ||
|
|
350
|
+
transaction.prepared_content_hash !== transaction.committed_hash ||
|
|
351
|
+
path.resolve(transaction.artifact_ref) !==
|
|
352
|
+
path.resolve(args.failedTerminalArtifactRef ?? "") ||
|
|
353
|
+
transaction.committed_hash !== args.failedTerminalArtifactSha256 ||
|
|
354
|
+
!parsedArtifact.success ||
|
|
355
|
+
parsedArtifact.data.session_id !== args.runControl.session_id ||
|
|
356
|
+
(args.expectedSessionId !== undefined &&
|
|
357
|
+
parsedArtifact.data.session_id !== args.expectedSessionId) ||
|
|
358
|
+
parsedArtifact.data.owner_attempt_id !== latestAttempt.attempt_id ||
|
|
359
|
+
path.resolve(transaction.artifact_ref) !== path.resolve(reconstructLlmDispatchFailurePath(args.runControl.session_root, parsedArtifact.data.failure_id)) ||
|
|
360
|
+
ownerSessionLocks.length !== 1 ||
|
|
361
|
+
ownerSessionLocks[0]?.lock_status !== "released" ||
|
|
362
|
+
args.runControl.lock_rows.some((row) => row.lock_scope === "session_root" && row.lock_status === "held")) {
|
|
363
|
+
violations.push(violation({
|
|
364
|
+
code: "failed_terminal_invalid",
|
|
365
|
+
message: "latest failed attempt failure artifact must match owner, ref, schema, prepared hash, and committed hash",
|
|
366
|
+
subjectId: transaction.transaction_id,
|
|
367
|
+
}));
|
|
368
|
+
}
|
|
369
|
+
else {
|
|
370
|
+
trustedFailedAttempt = true;
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
const currentAttempt = trustedFailedAttempt
|
|
374
|
+
? latestAttempt
|
|
375
|
+
: [...args.runControl.attempt_rows]
|
|
376
|
+
.reverse()
|
|
377
|
+
.find((row) => row.attempt_status === "running" ||
|
|
378
|
+
row.attempt_status === "completed" ||
|
|
379
|
+
row.attempt_status === "recovered" ||
|
|
380
|
+
row.attempt_status === "halted") ?? null;
|
|
216
381
|
if (!currentAttempt) {
|
|
217
382
|
violations.push(violation({
|
|
218
383
|
code: "active_attempt_missing",
|
|
219
|
-
message: "run-control must have a running, completed, or
|
|
384
|
+
message: "run-control must have a running, completed, recovered, halted, or trusted failed attempt",
|
|
220
385
|
}));
|
|
221
386
|
}
|
|
222
387
|
const activeLocks = args.runControl.lock_rows.filter((row) => row.lock_scope === "session_root" &&
|
|
@@ -227,6 +392,17 @@ export function validateReconstructRunControl(args) {
|
|
|
227
392
|
message: "run-control must record a session_root lock",
|
|
228
393
|
}));
|
|
229
394
|
}
|
|
395
|
+
if (currentAttempt?.attempt_status === "running") {
|
|
396
|
+
const heldSessionLocks = args.runControl.lock_rows.filter((row) => row.lock_scope === "session_root" && row.lock_status === "held");
|
|
397
|
+
if (heldSessionLocks.length !== 1 ||
|
|
398
|
+
heldSessionLocks[0]?.owner_attempt_id !== currentAttempt.attempt_id) {
|
|
399
|
+
violations.push(violation({
|
|
400
|
+
code: "conflicting_lock",
|
|
401
|
+
message: "the current running attempt must be the unique held session_root lock owner",
|
|
402
|
+
subjectId: currentAttempt.attempt_id,
|
|
403
|
+
}));
|
|
404
|
+
}
|
|
405
|
+
}
|
|
230
406
|
if (args.runControl.lock_rows.some((row) => row.lock_status === "conflict_blocked" ||
|
|
231
407
|
row.lock_status === "stolen_invalid")) {
|
|
232
408
|
violations.push(violation({
|
|
@@ -249,6 +425,18 @@ export function validateReconstructRunControl(args) {
|
|
|
249
425
|
subjectId: row.transaction_id,
|
|
250
426
|
}));
|
|
251
427
|
}
|
|
428
|
+
if (isReconstructLlmDispatchFailureRef(args.runControl.session_root, row.artifact_ref)) {
|
|
429
|
+
if (!row.prepared_content_hash ||
|
|
430
|
+
(row.transaction_status === "prepared" && !row.temp_ref) ||
|
|
431
|
+
(row.transaction_status === "committed" &&
|
|
432
|
+
row.prepared_content_hash !== row.committed_hash)) {
|
|
433
|
+
violations.push(violation({
|
|
434
|
+
code: "invalid_transaction",
|
|
435
|
+
message: "LLM dispatch failure transaction must preserve prepared content hash and temp/commit state",
|
|
436
|
+
subjectId: row.transaction_id,
|
|
437
|
+
}));
|
|
438
|
+
}
|
|
439
|
+
}
|
|
252
440
|
}
|
|
253
441
|
const attemptIds = new Set(args.runControl.attempt_rows.map((row) => row.attempt_id));
|
|
254
442
|
for (const row of args.runControl.resume_rows) {
|
|
@@ -315,7 +503,7 @@ export function validateReconstructRunControl(args) {
|
|
|
315
503
|
if (!terminalValidationRef) {
|
|
316
504
|
violations.push(violation({
|
|
317
505
|
code: "terminal_validation_missing",
|
|
318
|
-
message: "completed attempts and resume_allowed rows require
|
|
506
|
+
message: "completed/halted attempts and resume_allowed rows require terminal run-manifest validation authority",
|
|
319
507
|
}));
|
|
320
508
|
}
|
|
321
509
|
else {
|
|
@@ -324,14 +512,14 @@ export function validateReconstructRunControl(args) {
|
|
|
324
512
|
code: args.terminalValidationStatus === null
|
|
325
513
|
? "terminal_validation_missing"
|
|
326
514
|
: "terminal_validation_invalid",
|
|
327
|
-
message: "
|
|
515
|
+
message: "terminal run-manifest validation must exist and have validation_status=valid before completion, halt, or resume_allowed",
|
|
328
516
|
subjectId: terminalValidationRef,
|
|
329
517
|
}));
|
|
330
518
|
}
|
|
331
519
|
if (!committedRefs.has(path.resolve(terminalValidationRef))) {
|
|
332
520
|
violations.push(violation({
|
|
333
521
|
code: "expected_transaction_missing",
|
|
334
|
-
message: "run-control validation is missing a committed hash transaction for the
|
|
522
|
+
message: "run-control validation is missing a committed hash transaction for the terminal run-manifest validation artifact",
|
|
335
523
|
subjectId: terminalValidationRef,
|
|
336
524
|
}));
|
|
337
525
|
}
|
|
@@ -360,21 +548,42 @@ export function validateReconstructRunControl(args) {
|
|
|
360
548
|
validation_results: violations.length === 0
|
|
361
549
|
? ["reconstruct_run_control_valid"]
|
|
362
550
|
: ["reconstruct_run_control_invalid"],
|
|
551
|
+
asserted_obligation_ids: assertedObligationIds,
|
|
363
552
|
violations,
|
|
364
553
|
};
|
|
365
554
|
}
|
|
366
|
-
|
|
367
|
-
const runControl = await readYamlDocument(args.runControlPath);
|
|
555
|
+
async function buildReconstructRunControlValidationArtifactFromRunControl(args) {
|
|
368
556
|
const terminalValidationRef = inferTerminalValidationRef({
|
|
369
|
-
runControl,
|
|
557
|
+
runControl: args.runControl,
|
|
370
558
|
runControlPath: args.runControlPath,
|
|
371
559
|
...(args.terminalValidationRef !== undefined
|
|
372
560
|
? { explicitRef: args.terminalValidationRef }
|
|
373
561
|
: {}),
|
|
374
562
|
});
|
|
375
563
|
const terminalValidationStatus = await readValidationStatusIfPresent(terminalValidationRef);
|
|
376
|
-
const
|
|
377
|
-
|
|
564
|
+
const latestAttempt = args.runControl.attempt_rows.at(-1) ?? null;
|
|
565
|
+
const failedTransaction = latestAttempt?.attempt_status === "failed"
|
|
566
|
+
? args.runControl.write_transactions.find((row) => row.owner_attempt_id === latestAttempt.attempt_id &&
|
|
567
|
+
isReconstructLlmDispatchFailureRef(args.runControl.session_root, row.artifact_ref))
|
|
568
|
+
: undefined;
|
|
569
|
+
let failedTerminalArtifact = null;
|
|
570
|
+
let failedTerminalArtifactSha256 = null;
|
|
571
|
+
if (failedTransaction) {
|
|
572
|
+
try {
|
|
573
|
+
const failedRead = await readReconstructLlmDispatchFailureArtifactWithHash({
|
|
574
|
+
sessionRoot: args.runControl.session_root,
|
|
575
|
+
artifactRef: failedTransaction.artifact_ref,
|
|
576
|
+
});
|
|
577
|
+
failedTerminalArtifact = failedRead.artifact;
|
|
578
|
+
failedTerminalArtifactSha256 = failedRead.sha256;
|
|
579
|
+
}
|
|
580
|
+
catch {
|
|
581
|
+
failedTerminalArtifact = null;
|
|
582
|
+
failedTerminalArtifactSha256 = null;
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
return validateReconstructRunControl({
|
|
586
|
+
runControl: args.runControl,
|
|
378
587
|
runControlRef: args.runControlPath,
|
|
379
588
|
...(args.expectedSessionId !== undefined
|
|
380
589
|
? { expectedSessionId: args.expectedSessionId }
|
|
@@ -387,10 +596,81 @@ export async function writeReconstructRunControlValidationArtifact(args) {
|
|
|
387
596
|
: {}),
|
|
388
597
|
terminalValidationRef,
|
|
389
598
|
terminalValidationStatus,
|
|
599
|
+
...(failedTransaction
|
|
600
|
+
? {
|
|
601
|
+
failedTerminalArtifactRef: failedTransaction.artifact_ref,
|
|
602
|
+
failedTerminalArtifact,
|
|
603
|
+
failedTerminalArtifactSha256,
|
|
604
|
+
}
|
|
605
|
+
: {}),
|
|
390
606
|
});
|
|
607
|
+
}
|
|
608
|
+
async function buildReconstructRunControlValidationArtifact(args) {
|
|
609
|
+
const runControl = await readYamlDocument(args.runControlPath);
|
|
610
|
+
return buildReconstructRunControlValidationArtifactFromRunControl({
|
|
611
|
+
...args,
|
|
612
|
+
runControl,
|
|
613
|
+
});
|
|
614
|
+
}
|
|
615
|
+
export async function writeReconstructRunControlValidationArtifact(args) {
|
|
616
|
+
const validation = await buildReconstructRunControlValidationArtifact(args);
|
|
391
617
|
await writeYamlDocument(args.outputPath, validation);
|
|
392
618
|
return validation;
|
|
393
619
|
}
|
|
620
|
+
function comparableRunControlValidation(validation, options) {
|
|
621
|
+
if (validation === null || typeof validation !== "object" || Array.isArray(validation)) {
|
|
622
|
+
return null;
|
|
623
|
+
}
|
|
624
|
+
const persisted = { ...validation };
|
|
625
|
+
if (options.stripInMemoryFields) {
|
|
626
|
+
delete persisted.asserted_obligation_ids;
|
|
627
|
+
}
|
|
628
|
+
const parsed = PersistedRunControlValidationSchema.safeParse(persisted);
|
|
629
|
+
if (!parsed.success)
|
|
630
|
+
return null;
|
|
631
|
+
try {
|
|
632
|
+
return JSON.stringify({ ...parsed.data, created_at: null });
|
|
633
|
+
}
|
|
634
|
+
catch {
|
|
635
|
+
return null;
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
const PersistedRunControlValidationSchema = z.object({
|
|
639
|
+
schema_version: z.literal("1"),
|
|
640
|
+
session_id: z.string().min(1),
|
|
641
|
+
created_at: z.string().datetime({ offset: true }),
|
|
642
|
+
reconstruct_run_control_ref: z.string().min(1).nullable(),
|
|
643
|
+
validation_status: z.enum(["valid", "invalid"]),
|
|
644
|
+
request_count: z.number().int().nonnegative(),
|
|
645
|
+
attempt_count: z.number().int().nonnegative(),
|
|
646
|
+
active_lock_count: z.number().int().nonnegative(),
|
|
647
|
+
transaction_count: z.number().int().nonnegative(),
|
|
648
|
+
current_attempt_id: z.string().min(1).nullable(),
|
|
649
|
+
validation_results: z.array(z.string()),
|
|
650
|
+
violations: z.array(z.object({
|
|
651
|
+
code: z.enum([
|
|
652
|
+
"schema_shape_invalid",
|
|
653
|
+
"session_id_mismatch",
|
|
654
|
+
"session_root_missing",
|
|
655
|
+
"request_row_missing",
|
|
656
|
+
"attempt_row_missing",
|
|
657
|
+
"active_attempt_missing",
|
|
658
|
+
"session_lock_missing",
|
|
659
|
+
"conflicting_request",
|
|
660
|
+
"conflicting_lock",
|
|
661
|
+
"invalid_transaction",
|
|
662
|
+
"transaction_hash_missing",
|
|
663
|
+
"terminal_validation_missing",
|
|
664
|
+
"terminal_validation_invalid",
|
|
665
|
+
"expected_transaction_missing",
|
|
666
|
+
"failed_terminal_missing",
|
|
667
|
+
"failed_terminal_invalid",
|
|
668
|
+
"invalid_resume",
|
|
669
|
+
]),
|
|
670
|
+
message: z.string(),
|
|
671
|
+
subject_id: z.string().nullable(),
|
|
672
|
+
}).strict()),
|
|
673
|
+
}).strict();
|
|
394
674
|
export async function writeReconstructRunBootstrapDiagnostic(args) {
|
|
395
675
|
const artifact = {
|
|
396
676
|
schema_version: "1",
|
|
@@ -407,7 +687,11 @@ export async function writeReconstructRunBootstrapDiagnostic(args) {
|
|
|
407
687
|
await writeYamlDocument(args.outputPath, artifact);
|
|
408
688
|
return artifact;
|
|
409
689
|
}
|
|
410
|
-
|
|
690
|
+
async function initializeReconstructRunControlUnlocked(args) {
|
|
691
|
+
await assertDispatchFallbackSessionAdmission({
|
|
692
|
+
sessionRoot: args.sessionRoot,
|
|
693
|
+
enabled: args.dispatchFallbackEnabled === true,
|
|
694
|
+
});
|
|
411
695
|
const requestFingerprint = reconstructRequestFingerprint({
|
|
412
696
|
projectRoot: args.projectRoot,
|
|
413
697
|
sessionRoot: args.sessionRoot,
|
|
@@ -422,6 +706,10 @@ export async function initializeReconstructRunControl(args) {
|
|
|
422
706
|
const idempotencyKeyHash = sha256(`reconstruct:${requestFingerprint}`);
|
|
423
707
|
const existing = await readYamlDocumentIfPresent(args.outputPath);
|
|
424
708
|
if (existing) {
|
|
709
|
+
assertDispatchFallbackRunControlHasNoLiveOwner({
|
|
710
|
+
runControl: existing,
|
|
711
|
+
enabled: args.dispatchFallbackEnabled === true,
|
|
712
|
+
});
|
|
425
713
|
const conflict = existing.request_rows.some((row) => row.request_fingerprint !== requestFingerprint);
|
|
426
714
|
if (conflict) {
|
|
427
715
|
await writeReconstructRunBootstrapDiagnostic({
|
|
@@ -440,11 +728,43 @@ export async function initializeReconstructRunControl(args) {
|
|
|
440
728
|
if (!completedAttempt) {
|
|
441
729
|
const now = isoNow();
|
|
442
730
|
const sourceAttempt = [...existing.attempt_rows].reverse()[0] ?? null;
|
|
443
|
-
const
|
|
731
|
+
const heldSessionLock = existing.lock_rows.find((row) => row.lock_scope === "session_root" && row.lock_status === "held");
|
|
732
|
+
if (sourceAttempt?.attempt_status === "running" ||
|
|
733
|
+
heldSessionLock !== undefined) {
|
|
734
|
+
throw new Error("cannot resume a live reconstruct attempt; lease expiry is not takeover authority");
|
|
735
|
+
}
|
|
736
|
+
if (existing.resume_rows.some((row) => row.source_attempt_id === sourceAttempt?.attempt_id &&
|
|
737
|
+
row.resume_decision === "blocked_partial_write")) {
|
|
738
|
+
throw new Error("cannot resume reconstruct session with blocked_partial_write authority");
|
|
739
|
+
}
|
|
740
|
+
let failureProvenanceRefs = [];
|
|
741
|
+
const sourceFailureTransactions = sourceAttempt?.attempt_status === "failed"
|
|
742
|
+
? existing.write_transactions.filter((row) => row.owner_attempt_id === sourceAttempt.attempt_id &&
|
|
743
|
+
isReconstructLlmDispatchFailureRef(args.sessionRoot, row.artifact_ref))
|
|
744
|
+
: [];
|
|
745
|
+
if (sourceFailureTransactions.length > 0) {
|
|
746
|
+
const failedValidation = await writeReconstructRunControlValidationArtifact({
|
|
747
|
+
runControlPath: args.outputPath,
|
|
748
|
+
outputPath: args.validationOutputPath,
|
|
749
|
+
expectedSessionId: args.sessionId,
|
|
750
|
+
expectedSessionRoot: args.sessionRoot,
|
|
751
|
+
});
|
|
752
|
+
if (failedValidation.validation_status !== "valid") {
|
|
753
|
+
throw new Error(`cannot resume from an untrusted failed terminal: ${failedValidation.violations.map((item) => item.code).join(",")}`);
|
|
754
|
+
}
|
|
755
|
+
failureProvenanceRefs = sourceFailureTransactions
|
|
756
|
+
.filter((row) => row.transaction_status === "committed")
|
|
757
|
+
.map((row) => row.artifact_ref);
|
|
758
|
+
if (failureProvenanceRefs.length !== 1) {
|
|
759
|
+
throw new Error("cannot resume without exactly one trusted failure provenance ref");
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
const resumeId = idFor("resume", `${requestFingerprint}:${now}:${crypto.randomUUID()}`);
|
|
444
763
|
const attemptId = idFor("attempt", `${resumeId}:attempt`);
|
|
445
764
|
const trustedArtifactRefs = existing.write_transactions
|
|
446
765
|
.filter((row) => row.transaction_status === "committed" &&
|
|
447
|
-
row.committed_hash !== null
|
|
766
|
+
row.committed_hash !== null &&
|
|
767
|
+
!isReconstructLlmDispatchFailureRef(args.sessionRoot, row.artifact_ref))
|
|
448
768
|
.map((row) => row.artifact_ref)
|
|
449
769
|
.sort();
|
|
450
770
|
const provenanceMatchRefs = await collectResumeProvenanceMatchRefs(args.sessionRoot);
|
|
@@ -453,6 +773,7 @@ export async function initializeReconstructRunControl(args) {
|
|
|
453
773
|
args.validationOutputPath,
|
|
454
774
|
...provenanceMatchRefs,
|
|
455
775
|
...trustedArtifactRefs,
|
|
776
|
+
...failureProvenanceRefs,
|
|
456
777
|
];
|
|
457
778
|
existing.updated_at = now;
|
|
458
779
|
existing.resume_rows.push({
|
|
@@ -468,6 +789,7 @@ export async function initializeReconstructRunControl(args) {
|
|
|
468
789
|
args.validationOutputPath,
|
|
469
790
|
...provenanceMatchRefs,
|
|
470
791
|
...trustedArtifactRefs,
|
|
792
|
+
...failureProvenanceRefs,
|
|
471
793
|
].sort(),
|
|
472
794
|
resume_decision: "resume_pending_provenance",
|
|
473
795
|
});
|
|
@@ -493,6 +815,7 @@ export async function initializeReconstructRunControl(args) {
|
|
|
493
815
|
args.outputPath,
|
|
494
816
|
args.validationOutputPath,
|
|
495
817
|
...trustedArtifactRefs,
|
|
818
|
+
...failureProvenanceRefs,
|
|
496
819
|
],
|
|
497
820
|
});
|
|
498
821
|
existing.lock_rows = existing.lock_rows.map((row) => row.lock_scope === "session_root" && row.lock_status === "held"
|
|
@@ -508,7 +831,7 @@ export async function initializeReconstructRunControl(args) {
|
|
|
508
831
|
conflict_policy: "recover_expired_lease",
|
|
509
832
|
lock_status: "held",
|
|
510
833
|
});
|
|
511
|
-
await
|
|
834
|
+
await writeRunControlDocument(args.outputPath, existing);
|
|
512
835
|
const validation = await writeReconstructRunControlValidationArtifact({
|
|
513
836
|
runControlPath: args.outputPath,
|
|
514
837
|
outputPath: args.validationOutputPath,
|
|
@@ -583,7 +906,7 @@ export async function initializeReconstructRunControl(args) {
|
|
|
583
906
|
};
|
|
584
907
|
const created = await writeYamlDocumentAtomicCreate(args.outputPath, runControl);
|
|
585
908
|
if (!created) {
|
|
586
|
-
return
|
|
909
|
+
return initializeReconstructRunControlUnlocked(args);
|
|
587
910
|
}
|
|
588
911
|
const validation = await writeReconstructRunControlValidationArtifact({
|
|
589
912
|
runControlPath: args.outputPath,
|
|
@@ -593,7 +916,10 @@ export async function initializeReconstructRunControl(args) {
|
|
|
593
916
|
});
|
|
594
917
|
return { runControl, validation, requestFingerprint, attemptId };
|
|
595
918
|
}
|
|
596
|
-
export async function
|
|
919
|
+
export async function initializeReconstructRunControl(args) {
|
|
920
|
+
return withRunControlMutationLock(args.outputPath, () => initializeReconstructRunControlUnlocked(args));
|
|
921
|
+
}
|
|
922
|
+
async function markReconstructRunControlAttemptFailedUnlocked(args) {
|
|
597
923
|
const runControl = await readYamlDocument(args.runControlPath);
|
|
598
924
|
const failedAt = isoNow();
|
|
599
925
|
let changed = false;
|
|
@@ -613,7 +939,7 @@ export async function markReconstructRunControlAttemptFailed(args) {
|
|
|
613
939
|
runControl.lock_rows = runControl.lock_rows.map((row) => row.owner_attempt_id === args.attemptId && row.lock_status === "held"
|
|
614
940
|
? { ...row, lock_status: "released" }
|
|
615
941
|
: row);
|
|
616
|
-
await
|
|
942
|
+
await writeRunControlDocument(args.runControlPath, runControl);
|
|
617
943
|
}
|
|
618
944
|
const validation = await writeReconstructRunControlValidationArtifact({
|
|
619
945
|
runControlPath: args.runControlPath,
|
|
@@ -623,6 +949,617 @@ export async function markReconstructRunControlAttemptFailed(args) {
|
|
|
623
949
|
});
|
|
624
950
|
return { runControl, validation };
|
|
625
951
|
}
|
|
952
|
+
export async function markReconstructRunControlAttemptFailed(args) {
|
|
953
|
+
return withRunControlMutationLock(args.runControlPath, () => markReconstructRunControlAttemptFailedUnlocked(args));
|
|
954
|
+
}
|
|
955
|
+
async function persistReconstructLlmDispatchFailureUnlocked(args) {
|
|
956
|
+
let runControl = await readYamlDocument(args.runControlPath);
|
|
957
|
+
const latestAttempt = runControl.attempt_rows.at(-1) ?? null;
|
|
958
|
+
const ownerLocks = runControl.lock_rows.filter((row) => row.owner_attempt_id === args.attemptId &&
|
|
959
|
+
row.lock_scope === "session_root" &&
|
|
960
|
+
row.lock_status === "held");
|
|
961
|
+
if (latestAttempt?.attempt_id !== args.attemptId ||
|
|
962
|
+
latestAttempt.attempt_status !== "running" ||
|
|
963
|
+
ownerLocks.length !== 1) {
|
|
964
|
+
throw new Error(`cannot persist LLM dispatch failure for non-owning attempt ${args.attemptId}`);
|
|
965
|
+
}
|
|
966
|
+
const artifact = createReconstructLlmDispatchFailureArtifact({
|
|
967
|
+
sessionId: args.sessionId,
|
|
968
|
+
attemptId: args.attemptId,
|
|
969
|
+
error: args.error,
|
|
970
|
+
});
|
|
971
|
+
const plan = await planReconstructLlmDispatchFailureWrite({
|
|
972
|
+
sessionRoot: args.sessionRoot,
|
|
973
|
+
artifact,
|
|
974
|
+
});
|
|
975
|
+
const temp = await writeReconstructLlmDispatchFailureTemp({
|
|
976
|
+
sessionRoot: args.sessionRoot,
|
|
977
|
+
artifact,
|
|
978
|
+
plan,
|
|
979
|
+
});
|
|
980
|
+
await args.faultInjector?.("after_temp_write");
|
|
981
|
+
const transactionId = idFor("write", `${args.attemptId}:${plan.finalRef}`);
|
|
982
|
+
const existingTransaction = runControl.write_transactions.find((row) => row.transaction_id === transactionId);
|
|
983
|
+
const attempt = runControl.attempt_rows.find((row) => row.attempt_id === args.attemptId);
|
|
984
|
+
const heldLock = runControl.lock_rows.some((row) => row.owner_attempt_id === args.attemptId &&
|
|
985
|
+
row.lock_scope === "session_root" &&
|
|
986
|
+
row.lock_status === "held");
|
|
987
|
+
if ((!attempt || attempt.attempt_status !== "running" || !heldLock) &&
|
|
988
|
+
existingTransaction?.transaction_status !== "committed") {
|
|
989
|
+
throw new Error(`cannot prepare LLM dispatch failure for non-owning attempt ${args.attemptId}`);
|
|
990
|
+
}
|
|
991
|
+
if (existingTransaction) {
|
|
992
|
+
if (existingTransaction.owner_attempt_id !== args.attemptId ||
|
|
993
|
+
path.resolve(existingTransaction.artifact_ref) !== path.resolve(plan.finalRef) ||
|
|
994
|
+
existingTransaction.prepared_content_hash !== plan.contentSha256 ||
|
|
995
|
+
(existingTransaction.transaction_status !== "prepared" &&
|
|
996
|
+
existingTransaction.transaction_status !== "committed")) {
|
|
997
|
+
throw new Error(`LLM dispatch failure transaction conflicts: ${transactionId}`);
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
else {
|
|
1001
|
+
runControl.updated_at = isoNow();
|
|
1002
|
+
runControl.write_transactions.push({
|
|
1003
|
+
transaction_id: transactionId,
|
|
1004
|
+
owner_attempt_id: args.attemptId,
|
|
1005
|
+
artifact_ref: plan.finalRef,
|
|
1006
|
+
temp_ref: plan.tempRef,
|
|
1007
|
+
expected_prior_hash: null,
|
|
1008
|
+
prepared_content_hash: plan.contentSha256,
|
|
1009
|
+
committed_hash: null,
|
|
1010
|
+
commit_method: "append_only",
|
|
1011
|
+
transaction_status: "prepared",
|
|
1012
|
+
recovery_ref: null,
|
|
1013
|
+
});
|
|
1014
|
+
await writeRunControlDocument(args.runControlPath, runControl);
|
|
1015
|
+
}
|
|
1016
|
+
await args.faultInjector?.("after_prepare");
|
|
1017
|
+
await publishReconstructLlmDispatchFailureTemp({
|
|
1018
|
+
sessionRoot: args.sessionRoot,
|
|
1019
|
+
tempRef: temp.tempRef,
|
|
1020
|
+
finalRef: temp.finalRef,
|
|
1021
|
+
});
|
|
1022
|
+
await args.faultInjector?.("after_publish");
|
|
1023
|
+
const publishedRead = await readReconstructLlmDispatchFailureArtifactWithHash({
|
|
1024
|
+
sessionRoot: args.sessionRoot,
|
|
1025
|
+
artifactRef: temp.finalRef,
|
|
1026
|
+
});
|
|
1027
|
+
const published = publishedRead.artifact;
|
|
1028
|
+
const publishedHash = publishedRead.sha256;
|
|
1029
|
+
if (published.failure_id !== artifact.failure_id ||
|
|
1030
|
+
published.owner_attempt_id !== args.attemptId ||
|
|
1031
|
+
publishedHash !== temp.contentSha256) {
|
|
1032
|
+
throw new Error("published LLM dispatch failure artifact failed integrity verification");
|
|
1033
|
+
}
|
|
1034
|
+
runControl = await readYamlDocument(args.runControlPath);
|
|
1035
|
+
const alreadyCommitted = runControl.write_transactions.find((row) => row.transaction_id === transactionId &&
|
|
1036
|
+
row.transaction_status === "committed" &&
|
|
1037
|
+
row.owner_attempt_id === args.attemptId &&
|
|
1038
|
+
row.committed_hash === publishedHash);
|
|
1039
|
+
if (alreadyCommitted) {
|
|
1040
|
+
const validation = await writeReconstructRunControlValidationArtifact({
|
|
1041
|
+
runControlPath: args.runControlPath,
|
|
1042
|
+
outputPath: args.validationOutputPath,
|
|
1043
|
+
expectedSessionId: args.sessionId,
|
|
1044
|
+
expectedSessionRoot: args.sessionRoot,
|
|
1045
|
+
});
|
|
1046
|
+
if (validation.validation_status !== "valid") {
|
|
1047
|
+
throw new Error(`reconciled LLM dispatch failure did not produce a valid failed terminal: ${validation.violations.map((item) => item.code).join(",")}`);
|
|
1048
|
+
}
|
|
1049
|
+
return {
|
|
1050
|
+
artifact: published,
|
|
1051
|
+
artifactRef: temp.finalRef,
|
|
1052
|
+
runControl,
|
|
1053
|
+
validation,
|
|
1054
|
+
};
|
|
1055
|
+
}
|
|
1056
|
+
const preparedTransaction = runControl.write_transactions.find((row) => row.transaction_id === transactionId &&
|
|
1057
|
+
row.transaction_status === "prepared" &&
|
|
1058
|
+
row.owner_attempt_id === args.attemptId &&
|
|
1059
|
+
row.prepared_content_hash === publishedHash);
|
|
1060
|
+
const runningAttempt = runControl.attempt_rows.some((row) => row.attempt_id === args.attemptId && row.attempt_status === "running");
|
|
1061
|
+
const owningLock = runControl.lock_rows.some((row) => row.owner_attempt_id === args.attemptId &&
|
|
1062
|
+
row.lock_scope === "session_root" &&
|
|
1063
|
+
row.lock_status === "held");
|
|
1064
|
+
if (!preparedTransaction || !runningAttempt || !owningLock) {
|
|
1065
|
+
throw new Error(`cannot commit LLM dispatch failure without prepared transaction ownership for ${args.attemptId}`);
|
|
1066
|
+
}
|
|
1067
|
+
const committedAt = isoNow();
|
|
1068
|
+
runControl.updated_at = committedAt;
|
|
1069
|
+
runControl.write_transactions = runControl.write_transactions.map((row) => row.transaction_id === transactionId && row.transaction_status === "prepared"
|
|
1070
|
+
? {
|
|
1071
|
+
...row,
|
|
1072
|
+
temp_ref: null,
|
|
1073
|
+
committed_hash: publishedHash,
|
|
1074
|
+
transaction_status: "committed",
|
|
1075
|
+
}
|
|
1076
|
+
: row);
|
|
1077
|
+
runControl.attempt_rows = runControl.attempt_rows.map((row) => row.attempt_id === args.attemptId && row.attempt_status === "running"
|
|
1078
|
+
? { ...row, completed_at: committedAt, attempt_status: "failed" }
|
|
1079
|
+
: row);
|
|
1080
|
+
runControl.lock_rows = runControl.lock_rows.map((row) => row.owner_attempt_id === args.attemptId && row.lock_status === "held"
|
|
1081
|
+
? { ...row, lock_status: "released" }
|
|
1082
|
+
: row);
|
|
1083
|
+
await writeRunControlDocument(args.runControlPath, runControl);
|
|
1084
|
+
await args.faultInjector?.("after_commit");
|
|
1085
|
+
const validation = await writeReconstructRunControlValidationArtifact({
|
|
1086
|
+
runControlPath: args.runControlPath,
|
|
1087
|
+
outputPath: args.validationOutputPath,
|
|
1088
|
+
expectedSessionId: args.sessionId,
|
|
1089
|
+
expectedSessionRoot: args.sessionRoot,
|
|
1090
|
+
});
|
|
1091
|
+
if (validation.validation_status !== "valid") {
|
|
1092
|
+
throw new Error(`persisted LLM dispatch failure did not produce a valid failed terminal: ${validation.violations.map((item) => item.code).join(",")}`);
|
|
1093
|
+
}
|
|
1094
|
+
return { artifact, artifactRef: temp.finalRef, runControl, validation };
|
|
1095
|
+
}
|
|
1096
|
+
export async function persistReconstructLlmDispatchFailure(args) {
|
|
1097
|
+
return withRunControlMutationLock(args.runControlPath, () => persistReconstructLlmDispatchFailureUnlocked(args));
|
|
1098
|
+
}
|
|
1099
|
+
function blockedPartialWriteResumeRow(args) {
|
|
1100
|
+
const resumeId = idFor("resume", `${args.attemptId}:blocked_partial_write:${args.staleRefs.join(":")}`);
|
|
1101
|
+
return {
|
|
1102
|
+
resume_id: resumeId,
|
|
1103
|
+
resume_token_hash: sha256(resumeId),
|
|
1104
|
+
source_attempt_id: args.attemptId,
|
|
1105
|
+
checkpoint_refs: [args.runControlPath],
|
|
1106
|
+
trusted_artifact_refs: [],
|
|
1107
|
+
stale_artifact_refs: args.staleRefs,
|
|
1108
|
+
required_revalidation_refs: [args.runControlPath],
|
|
1109
|
+
resume_decision: "blocked_partial_write",
|
|
1110
|
+
};
|
|
1111
|
+
}
|
|
1112
|
+
function abandonCurrentAttemptForPartialWrite(args) {
|
|
1113
|
+
const current = args.runControl.attempt_rows.at(-1) ?? null;
|
|
1114
|
+
if (current?.attempt_status !== "running")
|
|
1115
|
+
return false;
|
|
1116
|
+
const abandonedAt = isoNow();
|
|
1117
|
+
args.runControl.attempt_rows = args.runControl.attempt_rows.map((row) => row.attempt_id === current.attempt_id && row.attempt_status === "running"
|
|
1118
|
+
? { ...row, completed_at: abandonedAt, attempt_status: "abandoned" }
|
|
1119
|
+
: row);
|
|
1120
|
+
args.runControl.lock_rows = args.runControl.lock_rows.map((row) => row.owner_attempt_id === current.attempt_id && row.lock_status === "held"
|
|
1121
|
+
? { ...row, lock_status: "released" }
|
|
1122
|
+
: row);
|
|
1123
|
+
if (!args.runControl.resume_rows.some((row) => row.source_attempt_id === current.attempt_id &&
|
|
1124
|
+
row.resume_decision === "blocked_partial_write")) {
|
|
1125
|
+
args.runControl.resume_rows.push(blockedPartialWriteResumeRow({
|
|
1126
|
+
runControlPath: args.runControlPath,
|
|
1127
|
+
attemptId: current.attempt_id,
|
|
1128
|
+
staleRefs: args.staleRefs,
|
|
1129
|
+
}));
|
|
1130
|
+
}
|
|
1131
|
+
return true;
|
|
1132
|
+
}
|
|
1133
|
+
async function pathExists(filePath) {
|
|
1134
|
+
if (!filePath)
|
|
1135
|
+
return false;
|
|
1136
|
+
try {
|
|
1137
|
+
await fs.access(filePath);
|
|
1138
|
+
return true;
|
|
1139
|
+
}
|
|
1140
|
+
catch (error) {
|
|
1141
|
+
if (error.code === "ENOENT")
|
|
1142
|
+
return false;
|
|
1143
|
+
throw error;
|
|
1144
|
+
}
|
|
1145
|
+
}
|
|
1146
|
+
async function removeFailureTempFile(sessionRoot, tempRef) {
|
|
1147
|
+
const directoryPath = await assertReconstructLlmDispatchFailureDirectory(sessionRoot);
|
|
1148
|
+
if (!isReconstructLlmDispatchFailureTempRef(sessionRoot, tempRef) ||
|
|
1149
|
+
path.dirname(path.resolve(tempRef)) !== directoryPath) {
|
|
1150
|
+
throw new Error(`cannot remove failure temp outside session: ${tempRef}`);
|
|
1151
|
+
}
|
|
1152
|
+
const stat = await fs.lstat(tempRef).catch((error) => {
|
|
1153
|
+
if (error.code === "ENOENT")
|
|
1154
|
+
return null;
|
|
1155
|
+
throw error;
|
|
1156
|
+
});
|
|
1157
|
+
if (!stat)
|
|
1158
|
+
return;
|
|
1159
|
+
if (stat.isSymbolicLink() || !stat.isFile()) {
|
|
1160
|
+
throw new Error(`cannot remove non-regular failure temp: ${tempRef}`);
|
|
1161
|
+
}
|
|
1162
|
+
await fs.rm(tempRef);
|
|
1163
|
+
}
|
|
1164
|
+
async function validatePreparedFailureArtifact(args) {
|
|
1165
|
+
const read = await readReconstructLlmDispatchFailureArtifactWithHash({
|
|
1166
|
+
sessionRoot: args.sessionRoot,
|
|
1167
|
+
artifactRef: args.artifactRef,
|
|
1168
|
+
allowTemp: isReconstructLlmDispatchFailureTempRef(args.sessionRoot, args.artifactRef),
|
|
1169
|
+
});
|
|
1170
|
+
const artifact = read.artifact;
|
|
1171
|
+
const hash = read.sha256;
|
|
1172
|
+
if (artifact.session_id !== args.sessionId ||
|
|
1173
|
+
artifact.owner_attempt_id !== args.ownerAttemptId ||
|
|
1174
|
+
!args.preparedContentHash ||
|
|
1175
|
+
hash !== args.preparedContentHash) {
|
|
1176
|
+
throw new Error("prepared failure transaction artifact does not match owner/hash");
|
|
1177
|
+
}
|
|
1178
|
+
if (!isReconstructLlmDispatchFailureTempRef(args.sessionRoot, args.artifactRef) &&
|
|
1179
|
+
path.resolve(args.artifactRef) !== path.resolve(reconstructLlmDispatchFailurePath(args.sessionRoot, artifact.failure_id))) {
|
|
1180
|
+
throw new Error("prepared failure transaction artifact path does not match failure id");
|
|
1181
|
+
}
|
|
1182
|
+
return { artifact, hash };
|
|
1183
|
+
}
|
|
1184
|
+
async function reconcileReconstructLlmDispatchFailuresUnlocked(args) {
|
|
1185
|
+
const sessionRoot = path.resolve(args.sessionRoot);
|
|
1186
|
+
const runControlPath = path.resolve(args.runControlPath ?? path.join(sessionRoot, "reconstruct-run-control.yaml"));
|
|
1187
|
+
const validationOutputPath = path.resolve(args.validationOutputPath ??
|
|
1188
|
+
path.join(sessionRoot, "reconstruct-run-control-validation.yaml"));
|
|
1189
|
+
let runControl;
|
|
1190
|
+
try {
|
|
1191
|
+
runControl = await readYamlDocument(runControlPath);
|
|
1192
|
+
}
|
|
1193
|
+
catch (error) {
|
|
1194
|
+
if (error.code === "ENOENT")
|
|
1195
|
+
return null;
|
|
1196
|
+
throw error;
|
|
1197
|
+
}
|
|
1198
|
+
if (path.resolve(runControl.session_root) !== sessionRoot ||
|
|
1199
|
+
runControl.session_id !== path.basename(sessionRoot) ||
|
|
1200
|
+
runControlPath !== path.join(sessionRoot, "reconstruct-run-control.yaml") ||
|
|
1201
|
+
validationOutputPath !==
|
|
1202
|
+
path.join(sessionRoot, "reconstruct-run-control-validation.yaml")) {
|
|
1203
|
+
throw new Error("reconstruct failure reconciliation session identity mismatch");
|
|
1204
|
+
}
|
|
1205
|
+
let changed = false;
|
|
1206
|
+
const failureDirectory = await assertReconstructLlmDispatchFailureDirectory(sessionRoot);
|
|
1207
|
+
let failureEntries = [];
|
|
1208
|
+
try {
|
|
1209
|
+
failureEntries = await fs.readdir(failureDirectory, { withFileTypes: true });
|
|
1210
|
+
}
|
|
1211
|
+
catch (error) {
|
|
1212
|
+
if (error.code !== "ENOENT")
|
|
1213
|
+
throw error;
|
|
1214
|
+
}
|
|
1215
|
+
const trackedTempRefs = new Set(runControl.write_transactions
|
|
1216
|
+
.map((row) => row.temp_ref)
|
|
1217
|
+
.filter((ref) => Boolean(ref))
|
|
1218
|
+
.map((ref) => path.resolve(ref)));
|
|
1219
|
+
const trackedFinalRefs = new Set(runControl.write_transactions
|
|
1220
|
+
.filter((row) => isReconstructLlmDispatchFailureRef(sessionRoot, row.artifact_ref))
|
|
1221
|
+
.map((row) => path.resolve(row.artifact_ref)));
|
|
1222
|
+
const untrackedPartialRefs = failureEntries
|
|
1223
|
+
.filter((entry) => {
|
|
1224
|
+
const entryRef = path.join(failureDirectory, entry.name);
|
|
1225
|
+
return entry.name.startsWith(".scratch-") ||
|
|
1226
|
+
(entry.name.startsWith(".pending-") && !trackedTempRefs.has(entryRef));
|
|
1227
|
+
})
|
|
1228
|
+
.map((entry) => path.join(failureDirectory, entry.name));
|
|
1229
|
+
const ambiguousPartialWrite = untrackedPartialRefs.length > 1;
|
|
1230
|
+
if (ambiguousPartialWrite) {
|
|
1231
|
+
changed = abandonCurrentAttemptForPartialWrite({
|
|
1232
|
+
runControl,
|
|
1233
|
+
runControlPath,
|
|
1234
|
+
staleRefs: untrackedPartialRefs,
|
|
1235
|
+
}) || changed;
|
|
1236
|
+
}
|
|
1237
|
+
for (const entry of failureEntries) {
|
|
1238
|
+
const entryRef = path.join(failureDirectory, entry.name);
|
|
1239
|
+
if (entry.name.startsWith(".scratch-")) {
|
|
1240
|
+
if (!ambiguousPartialWrite) {
|
|
1241
|
+
changed = abandonCurrentAttemptForPartialWrite({
|
|
1242
|
+
runControl,
|
|
1243
|
+
runControlPath,
|
|
1244
|
+
staleRefs: [entryRef],
|
|
1245
|
+
}) || changed;
|
|
1246
|
+
}
|
|
1247
|
+
continue;
|
|
1248
|
+
}
|
|
1249
|
+
if (entry.name.startsWith(".pending-") && !trackedTempRefs.has(entryRef)) {
|
|
1250
|
+
if (ambiguousPartialWrite)
|
|
1251
|
+
continue;
|
|
1252
|
+
try {
|
|
1253
|
+
const read = await readReconstructLlmDispatchFailureArtifactWithHash({
|
|
1254
|
+
sessionRoot,
|
|
1255
|
+
artifactRef: entryRef,
|
|
1256
|
+
allowTemp: true,
|
|
1257
|
+
});
|
|
1258
|
+
const latestAttempt = runControl.attempt_rows.at(-1) ?? null;
|
|
1259
|
+
const ownerLocks = runControl.lock_rows.filter((row) => row.lock_scope === "session_root" &&
|
|
1260
|
+
row.owner_attempt_id === read.artifact.owner_attempt_id);
|
|
1261
|
+
if (entry.name.startsWith(`.pending-${read.sha256.slice(0, 16)}-`) &&
|
|
1262
|
+
read.artifact.session_id === runControl.session_id &&
|
|
1263
|
+
latestAttempt?.attempt_id === read.artifact.owner_attempt_id &&
|
|
1264
|
+
latestAttempt.attempt_status === "running" &&
|
|
1265
|
+
ownerLocks.length === 1 &&
|
|
1266
|
+
ownerLocks[0]?.lock_status === "held") {
|
|
1267
|
+
const finalRef = reconstructLlmDispatchFailurePath(sessionRoot, read.artifact.failure_id);
|
|
1268
|
+
const transactionId = idFor("write", `${read.artifact.owner_attempt_id}:${finalRef}`);
|
|
1269
|
+
if (!runControl.write_transactions.some((row) => row.transaction_id === transactionId)) {
|
|
1270
|
+
runControl.write_transactions.push({
|
|
1271
|
+
transaction_id: transactionId,
|
|
1272
|
+
owner_attempt_id: read.artifact.owner_attempt_id,
|
|
1273
|
+
artifact_ref: finalRef,
|
|
1274
|
+
temp_ref: entryRef,
|
|
1275
|
+
expected_prior_hash: null,
|
|
1276
|
+
prepared_content_hash: read.sha256,
|
|
1277
|
+
committed_hash: null,
|
|
1278
|
+
commit_method: "append_only",
|
|
1279
|
+
transaction_status: "prepared",
|
|
1280
|
+
recovery_ref: null,
|
|
1281
|
+
});
|
|
1282
|
+
trackedFinalRefs.add(path.resolve(finalRef));
|
|
1283
|
+
changed = true;
|
|
1284
|
+
runControl.updated_at = isoNow();
|
|
1285
|
+
await writeRunControlDocument(runControlPath, runControl);
|
|
1286
|
+
}
|
|
1287
|
+
}
|
|
1288
|
+
else {
|
|
1289
|
+
changed = abandonCurrentAttemptForPartialWrite({
|
|
1290
|
+
runControl,
|
|
1291
|
+
runControlPath,
|
|
1292
|
+
staleRefs: [entryRef],
|
|
1293
|
+
}) || changed;
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
catch {
|
|
1297
|
+
changed = abandonCurrentAttemptForPartialWrite({
|
|
1298
|
+
runControl,
|
|
1299
|
+
runControlPath,
|
|
1300
|
+
staleRefs: [entryRef],
|
|
1301
|
+
}) || changed;
|
|
1302
|
+
}
|
|
1303
|
+
continue;
|
|
1304
|
+
}
|
|
1305
|
+
if (!entry.name.startsWith("failure-") ||
|
|
1306
|
+
path.extname(entry.name) !== ".yaml" ||
|
|
1307
|
+
trackedFinalRefs.has(entryRef)) {
|
|
1308
|
+
continue;
|
|
1309
|
+
}
|
|
1310
|
+
let artifact = null;
|
|
1311
|
+
let artifactHash = null;
|
|
1312
|
+
try {
|
|
1313
|
+
const stat = await fs.lstat(entryRef);
|
|
1314
|
+
if (stat.isFile() && !stat.isSymbolicLink()) {
|
|
1315
|
+
artifactHash = await sha256ReconstructLlmDispatchFailureArtifact(entryRef);
|
|
1316
|
+
}
|
|
1317
|
+
}
|
|
1318
|
+
catch {
|
|
1319
|
+
artifactHash = null;
|
|
1320
|
+
}
|
|
1321
|
+
try {
|
|
1322
|
+
artifact = await readReconstructLlmDispatchFailureArtifact(entryRef);
|
|
1323
|
+
}
|
|
1324
|
+
catch {
|
|
1325
|
+
artifact = null;
|
|
1326
|
+
}
|
|
1327
|
+
const ownerAttemptId = artifact &&
|
|
1328
|
+
artifact.session_id === runControl.session_id &&
|
|
1329
|
+
runControl.attempt_rows.some((row) => row.attempt_id === artifact.owner_attempt_id)
|
|
1330
|
+
? artifact.owner_attempt_id
|
|
1331
|
+
: [...runControl.attempt_rows].reverse().find((row) => row.attempt_status === "running")?.attempt_id ?? null;
|
|
1332
|
+
if (!ownerAttemptId || !artifactHash)
|
|
1333
|
+
continue;
|
|
1334
|
+
const transactionId = idFor("write", `${ownerAttemptId}:${entryRef}:orphan`);
|
|
1335
|
+
if (!runControl.write_transactions.some((row) => row.transaction_id === transactionId)) {
|
|
1336
|
+
runControl.write_transactions.push({
|
|
1337
|
+
transaction_id: transactionId,
|
|
1338
|
+
owner_attempt_id: ownerAttemptId,
|
|
1339
|
+
artifact_ref: entryRef,
|
|
1340
|
+
temp_ref: null,
|
|
1341
|
+
expected_prior_hash: null,
|
|
1342
|
+
prepared_content_hash: artifactHash,
|
|
1343
|
+
committed_hash: null,
|
|
1344
|
+
commit_method: "append_only",
|
|
1345
|
+
transaction_status: "quarantined",
|
|
1346
|
+
recovery_ref: "blocked_partial_write",
|
|
1347
|
+
});
|
|
1348
|
+
}
|
|
1349
|
+
const abandonedAt = isoNow();
|
|
1350
|
+
runControl.attempt_rows = runControl.attempt_rows.map((row) => row.attempt_id === ownerAttemptId && row.attempt_status === "running"
|
|
1351
|
+
? { ...row, completed_at: abandonedAt, attempt_status: "abandoned" }
|
|
1352
|
+
: row);
|
|
1353
|
+
runControl.lock_rows = runControl.lock_rows.map((row) => row.owner_attempt_id === ownerAttemptId && row.lock_status === "held"
|
|
1354
|
+
? { ...row, lock_status: "released" }
|
|
1355
|
+
: row);
|
|
1356
|
+
if (!runControl.resume_rows.some((row) => row.source_attempt_id === ownerAttemptId &&
|
|
1357
|
+
row.resume_decision === "blocked_partial_write")) {
|
|
1358
|
+
runControl.resume_rows.push(blockedPartialWriteResumeRow({
|
|
1359
|
+
runControlPath,
|
|
1360
|
+
attemptId: ownerAttemptId,
|
|
1361
|
+
staleRefs: [entryRef],
|
|
1362
|
+
}));
|
|
1363
|
+
}
|
|
1364
|
+
changed = true;
|
|
1365
|
+
}
|
|
1366
|
+
for (const transaction of runControl.write_transactions) {
|
|
1367
|
+
if (transaction.transaction_status !== "prepared" ||
|
|
1368
|
+
!isReconstructLlmDispatchFailureRef(sessionRoot, transaction.artifact_ref)) {
|
|
1369
|
+
continue;
|
|
1370
|
+
}
|
|
1371
|
+
try {
|
|
1372
|
+
const latestAttempt = runControl.attempt_rows.at(-1) ?? null;
|
|
1373
|
+
const ownerAttempt = runControl.attempt_rows.find((row) => row.attempt_id === transaction.owner_attempt_id);
|
|
1374
|
+
const ownerLocks = runControl.lock_rows.filter((row) => row.lock_scope === "session_root" &&
|
|
1375
|
+
row.owner_attempt_id === transaction.owner_attempt_id);
|
|
1376
|
+
if (latestAttempt?.attempt_id !== transaction.owner_attempt_id ||
|
|
1377
|
+
ownerAttempt?.attempt_status !== "running" ||
|
|
1378
|
+
ownerLocks.length !== 1 ||
|
|
1379
|
+
ownerLocks[0]?.lock_status !== "held") {
|
|
1380
|
+
throw new Error("prepared failure transaction is not owned by the latest running attempt and its unique held lock");
|
|
1381
|
+
}
|
|
1382
|
+
if (transaction.temp_ref &&
|
|
1383
|
+
!isReconstructLlmDispatchFailureTempRef(sessionRoot, transaction.temp_ref)) {
|
|
1384
|
+
throw new Error("prepared failure transaction temp ref escapes failure directory");
|
|
1385
|
+
}
|
|
1386
|
+
const finalExists = await pathExists(transaction.artifact_ref);
|
|
1387
|
+
const tempExists = await pathExists(transaction.temp_ref);
|
|
1388
|
+
if (!finalExists) {
|
|
1389
|
+
if (!tempExists) {
|
|
1390
|
+
if (Date.parse(ownerLocks[0].lease_expires_at) > Date.now()) {
|
|
1391
|
+
continue;
|
|
1392
|
+
}
|
|
1393
|
+
throw new Error("prepared failure transaction has neither temp nor final artifact");
|
|
1394
|
+
}
|
|
1395
|
+
await validatePreparedFailureArtifact({
|
|
1396
|
+
sessionRoot,
|
|
1397
|
+
artifactRef: transaction.temp_ref,
|
|
1398
|
+
sessionId: runControl.session_id,
|
|
1399
|
+
ownerAttemptId: transaction.owner_attempt_id,
|
|
1400
|
+
preparedContentHash: transaction.prepared_content_hash,
|
|
1401
|
+
});
|
|
1402
|
+
await publishReconstructLlmDispatchFailureTemp({
|
|
1403
|
+
sessionRoot,
|
|
1404
|
+
tempRef: transaction.temp_ref,
|
|
1405
|
+
finalRef: transaction.artifact_ref,
|
|
1406
|
+
});
|
|
1407
|
+
}
|
|
1408
|
+
const validated = await validatePreparedFailureArtifact({
|
|
1409
|
+
sessionRoot,
|
|
1410
|
+
artifactRef: transaction.artifact_ref,
|
|
1411
|
+
sessionId: runControl.session_id,
|
|
1412
|
+
ownerAttemptId: transaction.owner_attempt_id,
|
|
1413
|
+
preparedContentHash: transaction.prepared_content_hash,
|
|
1414
|
+
});
|
|
1415
|
+
if (await pathExists(transaction.temp_ref)) {
|
|
1416
|
+
await removeFailureTempFile(sessionRoot, transaction.temp_ref);
|
|
1417
|
+
}
|
|
1418
|
+
transaction.temp_ref = null;
|
|
1419
|
+
transaction.committed_hash = validated.hash;
|
|
1420
|
+
transaction.transaction_status = "committed";
|
|
1421
|
+
const completedAt = isoNow();
|
|
1422
|
+
runControl.attempt_rows = runControl.attempt_rows.map((row) => row.attempt_id === transaction.owner_attempt_id &&
|
|
1423
|
+
row.attempt_status === "running"
|
|
1424
|
+
? { ...row, completed_at: completedAt, attempt_status: "failed" }
|
|
1425
|
+
: row);
|
|
1426
|
+
runControl.lock_rows = runControl.lock_rows.map((row) => row.owner_attempt_id === transaction.owner_attempt_id &&
|
|
1427
|
+
row.lock_status === "held"
|
|
1428
|
+
? { ...row, lock_status: "released" }
|
|
1429
|
+
: row);
|
|
1430
|
+
changed = true;
|
|
1431
|
+
}
|
|
1432
|
+
catch {
|
|
1433
|
+
const staleRefs = [transaction.artifact_ref, transaction.temp_ref]
|
|
1434
|
+
.filter((ref) => Boolean(ref));
|
|
1435
|
+
transaction.transaction_status = "quarantined";
|
|
1436
|
+
transaction.recovery_ref = "blocked_partial_write";
|
|
1437
|
+
const abandonedAt = isoNow();
|
|
1438
|
+
runControl.attempt_rows = runControl.attempt_rows.map((row) => row.attempt_id === transaction.owner_attempt_id &&
|
|
1439
|
+
row.attempt_status === "running"
|
|
1440
|
+
? { ...row, completed_at: abandonedAt, attempt_status: "abandoned" }
|
|
1441
|
+
: row);
|
|
1442
|
+
runControl.lock_rows = runControl.lock_rows.map((row) => row.owner_attempt_id === transaction.owner_attempt_id &&
|
|
1443
|
+
row.lock_status === "held"
|
|
1444
|
+
? { ...row, lock_status: "released" }
|
|
1445
|
+
: row);
|
|
1446
|
+
if (!runControl.resume_rows.some((row) => row.source_attempt_id === transaction.owner_attempt_id &&
|
|
1447
|
+
row.resume_decision === "blocked_partial_write")) {
|
|
1448
|
+
runControl.resume_rows.push(blockedPartialWriteResumeRow({
|
|
1449
|
+
runControlPath,
|
|
1450
|
+
attemptId: transaction.owner_attempt_id,
|
|
1451
|
+
staleRefs,
|
|
1452
|
+
}));
|
|
1453
|
+
}
|
|
1454
|
+
changed = true;
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
if (changed) {
|
|
1458
|
+
runControl.updated_at = isoNow();
|
|
1459
|
+
await writeRunControlDocument(runControlPath, runControl);
|
|
1460
|
+
}
|
|
1461
|
+
const latestAttempt = runControl.attempt_rows.at(-1) ?? null;
|
|
1462
|
+
const validation = changed || latestAttempt?.attempt_status === "failed"
|
|
1463
|
+
? await writeReconstructRunControlValidationArtifact({
|
|
1464
|
+
runControlPath,
|
|
1465
|
+
outputPath: validationOutputPath,
|
|
1466
|
+
expectedSessionId: runControl.session_id,
|
|
1467
|
+
expectedSessionRoot: sessionRoot,
|
|
1468
|
+
})
|
|
1469
|
+
: null;
|
|
1470
|
+
return { runControl, validation };
|
|
1471
|
+
}
|
|
1472
|
+
export async function reconcileReconstructLlmDispatchFailures(args) {
|
|
1473
|
+
const sessionRoot = path.resolve(args.sessionRoot);
|
|
1474
|
+
const runControlPath = path.resolve(args.runControlPath ?? path.join(sessionRoot, "reconstruct-run-control.yaml"));
|
|
1475
|
+
const validationOutputPath = path.resolve(args.validationOutputPath ??
|
|
1476
|
+
path.join(sessionRoot, "reconstruct-run-control-validation.yaml"));
|
|
1477
|
+
let snapshot;
|
|
1478
|
+
try {
|
|
1479
|
+
snapshot = await readYamlDocument(runControlPath);
|
|
1480
|
+
}
|
|
1481
|
+
catch (error) {
|
|
1482
|
+
if (error.code === "ENOENT")
|
|
1483
|
+
return null;
|
|
1484
|
+
throw error;
|
|
1485
|
+
}
|
|
1486
|
+
if (path.resolve(snapshot.session_root) !== sessionRoot ||
|
|
1487
|
+
snapshot.session_id !== path.basename(sessionRoot) ||
|
|
1488
|
+
runControlPath !== path.join(sessionRoot, "reconstruct-run-control.yaml") ||
|
|
1489
|
+
validationOutputPath !==
|
|
1490
|
+
path.join(sessionRoot, "reconstruct-run-control-validation.yaml")) {
|
|
1491
|
+
throw new Error("reconstruct failure reconciliation session identity mismatch");
|
|
1492
|
+
}
|
|
1493
|
+
const failureDirectory = path.join(sessionRoot, RECONSTRUCT_LLM_DISPATCH_FAILURE_DIR);
|
|
1494
|
+
const failureDirectoryStat = await fs.lstat(failureDirectory)
|
|
1495
|
+
.catch((error) => {
|
|
1496
|
+
if (error.code === "ENOENT")
|
|
1497
|
+
return null;
|
|
1498
|
+
throw error;
|
|
1499
|
+
});
|
|
1500
|
+
if (failureDirectoryStat &&
|
|
1501
|
+
(!failureDirectoryStat.isDirectory() || failureDirectoryStat.isSymbolicLink())) {
|
|
1502
|
+
throw new Error(`LLM dispatch failure directory is not a real directory: ${failureDirectory}`);
|
|
1503
|
+
}
|
|
1504
|
+
if (failureDirectoryStat) {
|
|
1505
|
+
const [realSessionRoot, realFailureDirectory] = await Promise.all([
|
|
1506
|
+
fs.realpath(sessionRoot),
|
|
1507
|
+
fs.realpath(failureDirectory),
|
|
1508
|
+
]);
|
|
1509
|
+
if (path.dirname(realFailureDirectory) !== realSessionRoot) {
|
|
1510
|
+
throw new Error(`LLM dispatch failure directory escapes session root: ${realFailureDirectory}`);
|
|
1511
|
+
}
|
|
1512
|
+
}
|
|
1513
|
+
const failureEntries = failureDirectoryStat
|
|
1514
|
+
? await fs.readdir(failureDirectory, { withFileTypes: true })
|
|
1515
|
+
: [];
|
|
1516
|
+
const trackedFinalRefs = new Set(snapshot.write_transactions
|
|
1517
|
+
.filter((row) => isReconstructLlmDispatchFailureRef(sessionRoot, row.artifact_ref))
|
|
1518
|
+
.map((row) => path.resolve(row.artifact_ref)));
|
|
1519
|
+
const needsMutation = snapshot.write_transactions.some((row) => row.transaction_status === "prepared" &&
|
|
1520
|
+
isReconstructLlmDispatchFailureRef(sessionRoot, row.artifact_ref)) || failureEntries.some((entry) => {
|
|
1521
|
+
const entryRef = path.join(failureDirectory, entry.name);
|
|
1522
|
+
return entry.name.startsWith(".scratch-") ||
|
|
1523
|
+
entry.name.startsWith(".pending-") ||
|
|
1524
|
+
(entry.name.startsWith("failure-") &&
|
|
1525
|
+
path.extname(entry.name) === ".yaml" &&
|
|
1526
|
+
!trackedFinalRefs.has(path.resolve(entryRef)));
|
|
1527
|
+
});
|
|
1528
|
+
const latestAttempt = snapshot.attempt_rows.at(-1) ?? null;
|
|
1529
|
+
let persistedValidation = null;
|
|
1530
|
+
if (latestAttempt?.attempt_status === "failed") {
|
|
1531
|
+
try {
|
|
1532
|
+
const raw = await readYamlDocumentIfPresent(validationOutputPath);
|
|
1533
|
+
if (raw !== null &&
|
|
1534
|
+
comparableRunControlValidation(raw, { stripInMemoryFields: false }) !== null) {
|
|
1535
|
+
persistedValidation = raw;
|
|
1536
|
+
}
|
|
1537
|
+
}
|
|
1538
|
+
catch {
|
|
1539
|
+
persistedValidation = null;
|
|
1540
|
+
}
|
|
1541
|
+
}
|
|
1542
|
+
const expectedValidation = latestAttempt?.attempt_status === "failed"
|
|
1543
|
+
? await buildReconstructRunControlValidationArtifactFromRunControl({
|
|
1544
|
+
runControl: snapshot,
|
|
1545
|
+
runControlPath,
|
|
1546
|
+
expectedSessionId: snapshot.session_id,
|
|
1547
|
+
expectedSessionRoot: sessionRoot,
|
|
1548
|
+
})
|
|
1549
|
+
: null;
|
|
1550
|
+
const persistedComparable = comparableRunControlValidation(persistedValidation, { stripInMemoryFields: false });
|
|
1551
|
+
const expectedComparable = comparableRunControlValidation(expectedValidation, { stripInMemoryFields: true });
|
|
1552
|
+
const validationIsCurrent = persistedComparable !== null &&
|
|
1553
|
+
expectedComparable !== null &&
|
|
1554
|
+
persistedComparable === expectedComparable;
|
|
1555
|
+
if (!needsMutation && latestAttempt?.attempt_status !== "failed") {
|
|
1556
|
+
return { runControl: snapshot, validation: null };
|
|
1557
|
+
}
|
|
1558
|
+
if (!needsMutation && validationIsCurrent) {
|
|
1559
|
+
return { runControl: snapshot, validation: persistedValidation };
|
|
1560
|
+
}
|
|
1561
|
+
return withRunControlMutationLock(runControlPath, () => reconcileReconstructLlmDispatchFailuresUnlocked(args));
|
|
1562
|
+
}
|
|
626
1563
|
function artifactRefsForTransactions(artifactRefs, extraRefs) {
|
|
627
1564
|
return [...new Set([
|
|
628
1565
|
...Object.values(artifactRefs),
|
|
@@ -653,7 +1590,7 @@ async function appendWriteTransactions(args) {
|
|
|
653
1590
|
existingTransactionIds.add(transactionId);
|
|
654
1591
|
}
|
|
655
1592
|
}
|
|
656
|
-
|
|
1593
|
+
async function recordReconstructRunControlTransactionsUnlocked(args) {
|
|
657
1594
|
const runControl = await readYamlDocument(args.runControlPath);
|
|
658
1595
|
const attempt = runControl.attempt_rows.find((row) => row.attempt_id === args.attemptId);
|
|
659
1596
|
if (!attempt || attempt.attempt_status !== "running") {
|
|
@@ -671,7 +1608,7 @@ export async function recordReconstructRunControlTransactions(args) {
|
|
|
671
1608
|
attemptId: args.attemptId,
|
|
672
1609
|
refs: [...new Set(args.artifactRefs)].sort(),
|
|
673
1610
|
});
|
|
674
|
-
await
|
|
1611
|
+
await writeRunControlDocument(args.runControlPath, runControl);
|
|
675
1612
|
const validation = await writeReconstructRunControlValidationArtifact({
|
|
676
1613
|
runControlPath: args.runControlPath,
|
|
677
1614
|
outputPath: args.validationOutputPath,
|
|
@@ -683,20 +1620,57 @@ export async function recordReconstructRunControlTransactions(args) {
|
|
|
683
1620
|
});
|
|
684
1621
|
return { runControl, validation };
|
|
685
1622
|
}
|
|
686
|
-
export async function
|
|
687
|
-
|
|
1623
|
+
export async function recordReconstructRunControlTransactions(args) {
|
|
1624
|
+
return withRunControlMutationLock(args.runControlPath, () => recordReconstructRunControlTransactionsUnlocked(args));
|
|
1625
|
+
}
|
|
1626
|
+
async function finalizeReconstructRunControlUnlocked(args) {
|
|
1627
|
+
const attemptStatus = args.attemptStatus ?? "completed";
|
|
1628
|
+
const terminalValidationStatus = await readValidationStatusIfPresent(args.terminalRunManifestValidationPath);
|
|
688
1629
|
if (terminalValidationStatus !== "valid") {
|
|
689
1630
|
throw new Error([
|
|
690
|
-
"reconstruct run-control cannot finalize without valid
|
|
691
|
-
`terminal_validation_ref=${args.
|
|
1631
|
+
"reconstruct run-control cannot finalize without valid terminal run-manifest validation.",
|
|
1632
|
+
`terminal_validation_ref=${args.terminalRunManifestValidationPath}`,
|
|
1633
|
+
`attempt_status=${attemptStatus}`,
|
|
692
1634
|
`validation_status=${terminalValidationStatus ?? "missing"}`,
|
|
693
1635
|
].join(" "));
|
|
694
1636
|
}
|
|
695
1637
|
const runControl = await readYamlDocument(args.runControlPath);
|
|
1638
|
+
if (runControl.resume_rows.some((row) => row.source_attempt_id === args.attemptId &&
|
|
1639
|
+
row.resume_decision === "blocked_partial_write")) {
|
|
1640
|
+
throw new Error("reconstruct run-control cannot finalize a blocked_partial_write attempt");
|
|
1641
|
+
}
|
|
1642
|
+
const activeAttempt = runControl.attempt_rows.find((row) => row.attempt_id === args.attemptId);
|
|
1643
|
+
const activeOwnerLocks = runControl.lock_rows.filter((row) => row.lock_scope === "session_root" &&
|
|
1644
|
+
row.owner_attempt_id === args.attemptId &&
|
|
1645
|
+
row.lock_status === "held");
|
|
1646
|
+
if (activeAttempt?.attempt_status !== "running" || activeOwnerLocks.length !== 1) {
|
|
1647
|
+
throw new Error(`reconstruct run-control finalize requires the exact running owner: ${args.attemptId}`);
|
|
1648
|
+
}
|
|
1649
|
+
if (activeAttempt.attempt_kind === "resume" && activeAttempt.trigger_ref) {
|
|
1650
|
+
const resume = runControl.resume_rows.find((row) => row.resume_id === activeAttempt.trigger_ref);
|
|
1651
|
+
if (!resume || resume.resume_decision !== "resume_pending_provenance") {
|
|
1652
|
+
throw new Error("reconstruct resume is not pending trusted provenance");
|
|
1653
|
+
}
|
|
1654
|
+
for (const failureRef of resume.required_revalidation_refs.filter((ref) => isReconstructLlmDispatchFailureRef(runControl.session_root, ref))) {
|
|
1655
|
+
const sourceTransaction = runControl.write_transactions.find((row) => row.owner_attempt_id === resume.source_attempt_id &&
|
|
1656
|
+
path.resolve(row.artifact_ref) === path.resolve(failureRef) &&
|
|
1657
|
+
row.transaction_status === "committed" &&
|
|
1658
|
+
row.committed_hash !== null);
|
|
1659
|
+
const failureRead = sourceTransaction
|
|
1660
|
+
? await readReconstructLlmDispatchFailureArtifactWithHash({
|
|
1661
|
+
sessionRoot: runControl.session_root,
|
|
1662
|
+
artifactRef: failureRef,
|
|
1663
|
+
})
|
|
1664
|
+
: null;
|
|
1665
|
+
if (!sourceTransaction || failureRead?.sha256 !== sourceTransaction.committed_hash) {
|
|
1666
|
+
throw new Error(`reconstruct resume failure provenance changed before finalize: ${failureRef}`);
|
|
1667
|
+
}
|
|
1668
|
+
}
|
|
1669
|
+
}
|
|
696
1670
|
const completedAt = isoNow();
|
|
697
1671
|
runControl.updated_at = completedAt;
|
|
698
1672
|
runControl.attempt_rows = runControl.attempt_rows.map((row) => row.attempt_id === args.attemptId
|
|
699
|
-
? { ...row, completed_at: completedAt, attempt_status:
|
|
1673
|
+
? { ...row, completed_at: completedAt, attempt_status: attemptStatus }
|
|
700
1674
|
: row);
|
|
701
1675
|
const completedAttempt = runControl.attempt_rows.find((row) => row.attempt_id === args.attemptId);
|
|
702
1676
|
if (completedAttempt?.attempt_kind === "resume" && completedAttempt.trigger_ref) {
|
|
@@ -710,20 +1684,23 @@ export async function finalizeReconstructRunControl(args) {
|
|
|
710
1684
|
: row);
|
|
711
1685
|
const refs = artifactRefsForTransactions(args.artifactRefs, [
|
|
712
1686
|
...(args.extraArtifactRefs ?? []),
|
|
713
|
-
args.
|
|
1687
|
+
args.terminalRunManifestValidationPath,
|
|
714
1688
|
]);
|
|
715
1689
|
await appendWriteTransactions({
|
|
716
1690
|
runControl,
|
|
717
1691
|
attemptId: args.attemptId,
|
|
718
1692
|
refs,
|
|
719
1693
|
});
|
|
720
|
-
await
|
|
1694
|
+
await writeRunControlDocument(args.runControlPath, runControl);
|
|
721
1695
|
const validation = await writeReconstructRunControlValidationArtifact({
|
|
722
1696
|
runControlPath: args.runControlPath,
|
|
723
1697
|
outputPath: args.validationOutputPath,
|
|
724
1698
|
expectedSessionId: args.expectedSessionId,
|
|
725
1699
|
expectedSessionRoot: args.expectedSessionRoot,
|
|
726
|
-
terminalValidationRef: args.
|
|
1700
|
+
terminalValidationRef: args.terminalRunManifestValidationPath,
|
|
727
1701
|
});
|
|
728
1702
|
return { runControl, validation };
|
|
729
1703
|
}
|
|
1704
|
+
export async function finalizeReconstructRunControl(args) {
|
|
1705
|
+
return withRunControlMutationLock(args.runControlPath, () => finalizeReconstructRunControlUnlocked(args));
|
|
1706
|
+
}
|