onto-mcp 0.4.11 → 0.4.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.onto/authority/core-lexicon.yaml +1 -1
- package/.onto/authority/supported-models.yaml +153 -0
- package/.onto/domains/accounting-kr/competency_qs.md +153 -153
- package/.onto/processes/reconstruct/obligation-coverage-ledger.yaml +842 -0
- package/.onto/processes/reconstruct/obligation-coverage-recorded.yaml +225 -0
- package/.onto/processes/reconstruct/ontology-seeding-and-maturation-design.md +93 -54
- package/.onto/processes/reconstruct/reconstruct-contract-registry.yaml +202 -49
- package/.onto/processes/reconstruct/reconstruct-execution-ux-contract.md +72 -1
- package/.onto/processes/reconstruct/source-profiles/spreadsheet.md +35 -0
- package/.onto/processes/review/nesting-batch-worker-contract.md +1 -1
- package/.onto/processes/review/productized-live-path.md +9 -5
- package/.onto/processes/review/prompt-execution-runner-contract.md +32 -8
- package/.onto/processes/review/review-target-profile-contract.md +47 -4
- package/.onto/processes/shared/pipeline-execution-ledger-contract.md +11 -0
- package/.onto/processes/shared/target-material-kind-contract.md +1 -1
- package/README.md +64 -1
- package/dist/cli.js +11 -1
- package/dist/core-api/reconstruct-api.js +679 -18
- package/dist/core-api/review-api.js +70 -5
- package/dist/core-api/review-progress.js +7 -0
- package/dist/core-api/runtime-observation.js +7 -0
- package/dist/core-runtime/artifact-io.js +131 -0
- package/dist/core-runtime/cli/claude-code-review-unit-executor.js +21 -9
- package/dist/core-runtime/cli/claude-nesting-batch-worker.js +4 -2
- package/dist/core-runtime/cli/materialize-review-prompt-packets.js +68 -14
- package/dist/core-runtime/cli/prepare-review-session.js +34 -1
- package/dist/core-runtime/cli/render-review-final-output.js +2 -1
- package/dist/core-runtime/cli/run-review-prompt-execution.js +758 -48
- package/dist/core-runtime/cli/run-seats.js +18 -0
- package/dist/core-runtime/cli/runtime-submit-context.js +10 -2
- package/dist/core-runtime/cli/unit-resubmit.js +269 -0
- package/dist/core-runtime/cli/worker-structured-output.js +80 -2
- package/dist/core-runtime/discovery/host-detection.js +71 -22
- package/dist/core-runtime/discovery/review-cert-assemble.js +208 -0
- package/dist/core-runtime/discovery/review-cert-record.js +442 -0
- package/dist/core-runtime/discovery/review-execution-units.js +22 -0
- package/dist/core-runtime/discovery/seat-inventory.js +167 -0
- package/dist/core-runtime/discovery/settings-chain.js +311 -52
- package/dist/core-runtime/discovery/supported-models.js +289 -25
- package/dist/core-runtime/discovery/synthesize-cert-assemble.js +197 -0
- package/dist/core-runtime/discovery/synthesize-cert-capsule.js +399 -0
- package/dist/core-runtime/discovery/synthesize-cert-judge.js +340 -0
- package/dist/core-runtime/discovery/synthesize-cert-loop.js +234 -0
- package/dist/core-runtime/discovery/synthesize-cert-mutation.js +117 -0
- package/dist/core-runtime/discovery/synthesize-cert-packet.js +182 -0
- package/dist/core-runtime/discovery/synthesize-cert-record.js +831 -0
- package/dist/core-runtime/discovery/synthesize-cert-sampler.js +336 -0
- package/dist/core-runtime/effort-calibration-ingest.js +326 -0
- package/dist/core-runtime/effort-calibration-reconstruct.js +47 -0
- package/dist/core-runtime/effort-calibration-review.js +22 -0
- package/dist/core-runtime/effort-calibration-sweep.js +110 -0
- package/dist/core-runtime/effort-frontier.js +134 -0
- package/dist/core-runtime/llm/claude-bin.js +77 -0
- package/dist/core-runtime/llm/dispatch-breaker.js +412 -0
- package/dist/core-runtime/llm/dispatch-fallback-adapter-capabilities.js +33 -0
- package/dist/core-runtime/llm/llm-caller.js +316 -35
- package/dist/core-runtime/llm/mock-llm-realization.js +35 -4
- package/dist/core-runtime/llm/model-switcher.js +1 -1
- package/dist/core-runtime/llm/openai-responses-incomplete-error.js +46 -0
- package/dist/core-runtime/llm/sealed-dispatch-capability.js +359 -0
- package/dist/core-runtime/llm/structured-dispatch-error.js +15 -0
- package/dist/core-runtime/onboard/claude-profile-scan.js +65 -0
- package/dist/core-runtime/onboard/cli-host.js +5 -2
- package/dist/core-runtime/onboard/configure-provider.js +33 -1
- package/dist/core-runtime/onboard/host-target.js +28 -3
- package/dist/core-runtime/onboard/register.js +54 -12
- package/dist/core-runtime/pipeline-execution-ledger.js +10 -0
- package/dist/core-runtime/reconstruct/artifact-types.js +37 -0
- package/dist/core-runtime/reconstruct/benchmark-evidence.js +16 -0
- package/dist/core-runtime/reconstruct/claim-projection-validation.js +19 -5
- package/dist/core-runtime/reconstruct/competency-projection-contract.js +63 -0
- package/dist/core-runtime/reconstruct/comprehension-artifact.js +321 -0
- package/dist/core-runtime/reconstruct/comprehension-reduce.js +367 -0
- package/dist/core-runtime/reconstruct/comprehension-semantic-map.js +771 -0
- package/dist/core-runtime/reconstruct/contract-registry.js +20 -0
- package/dist/core-runtime/reconstruct/directive-validation.js +4 -3
- package/dist/core-runtime/reconstruct/dispatch-fallback-artifacts.js +640 -0
- package/dist/core-runtime/reconstruct/execution-telemetry.js +84 -9
- package/dist/core-runtime/reconstruct/final-output-sections.js +89 -0
- package/dist/core-runtime/reconstruct/governing-snapshot.js +36 -1
- package/dist/core-runtime/reconstruct/leaf-reader.js +305 -0
- package/dist/core-runtime/reconstruct/llm-dispatch-failure.js +270 -0
- package/dist/core-runtime/reconstruct/llm-touch-fingerprint.js +94 -0
- package/dist/core-runtime/reconstruct/material-admission-validation.js +20 -5
- package/dist/core-runtime/reconstruct/material-profile-validation.js +24 -6
- package/dist/core-runtime/reconstruct/materialize-preparation.js +380 -23
- package/dist/core-runtime/reconstruct/maturation-validation.js +1522 -67
- package/dist/core-runtime/reconstruct/mock-llm-realization.js +120 -1
- package/dist/core-runtime/reconstruct/obligation-assertion.js +18 -0
- package/dist/core-runtime/reconstruct/ontology-seed-validation.js +39 -7
- package/dist/core-runtime/reconstruct/output-budget.js +6 -0
- package/dist/core-runtime/reconstruct/pipeline-execution-ledger.js +112 -5
- package/dist/core-runtime/reconstruct/post-seed-validation.js +122 -11
- package/dist/core-runtime/reconstruct/proof-authority-validation.js +2 -6
- package/dist/core-runtime/reconstruct/purpose-authority-validation.js +119 -7
- package/dist/core-runtime/reconstruct/record.js +49 -3
- package/dist/core-runtime/reconstruct/registry-verification-validation.js +41 -5
- package/dist/core-runtime/reconstruct/run-control-validation.js +1019 -37
- package/dist/core-runtime/reconstruct/run.js +5409 -517
- package/dist/core-runtime/reconstruct/seed-authoring-readiness-validation.js +56 -8
- package/dist/core-runtime/reconstruct/semantic-quality-gate.js +112 -7
- package/dist/core-runtime/reconstruct/source-observation-delta-validation.js +53 -5
- package/dist/core-runtime/reconstruct/source-observations.js +166 -0
- package/dist/core-runtime/reconstruct/source-safety-validation.js +77 -188
- package/dist/core-runtime/reconstruct/source-scout-pack-validation.js +50 -6
- package/dist/core-runtime/reconstruct/terminal-validation.js +145 -13
- package/dist/core-runtime/review/continuation-plan.js +12 -5
- package/dist/core-runtime/review/issue-artifact-runtime.js +4 -1
- package/dist/core-runtime/review/materializers.js +100 -11
- package/dist/core-runtime/review/nesting-batch.js +6 -2
- package/dist/core-runtime/review/obligation-shardability.js +138 -0
- package/dist/core-runtime/review/pipeline-execution-ledger.js +42 -2
- package/dist/core-runtime/review/review-artifact-utils.js +219 -7
- package/dist/core-runtime/review/review-execution-steps.js +32 -8
- package/dist/core-runtime/review/review-progress-contract.js +8 -0
- package/dist/core-runtime/review/review-prompt-budget.js +97 -0
- package/dist/core-runtime/review/semantic-quality-gate.js +33 -7
- package/dist/core-runtime/review/spreadsheet-review-disposition.js +94 -0
- package/dist/core-runtime/route-identity.js +182 -0
- package/dist/core-runtime/spreadsheet-header-escalation.js +204 -0
- package/dist/core-runtime/spreadsheet-structure-observer.js +2689 -0
- package/dist/core-runtime/target-material-kind.js +58 -0
- package/dist/mcp/server.js +27 -5
- package/dist/mcp/tool-schemas.js +8 -0
- package/dist/tui/app.js +167 -0
- package/dist/tui/data/event-follower.js +116 -0
- package/dist/tui/data/node-detail.js +42 -0
- package/dist/tui/data/projection-poll.js +26 -0
- package/dist/tui/data/session-discovery.js +65 -0
- package/dist/tui/index.js +108 -0
- package/dist/tui/screens/log.js +25 -0
- package/dist/tui/screens/node-detail.js +17 -0
- package/dist/tui/screens/session-selector.js +26 -0
- package/dist/tui/screens/workflow-tree.js +90 -0
- package/dist/tui/view-model/reconstruct-adapter.js +162 -0
- package/dist/tui/view-model/review-adapter.js +270 -0
- package/dist/tui/view-model/tree-view-model.js +19 -0
- package/package.json +20 -3
- package/settings.example.json +77 -16
|
@@ -1,13 +1,63 @@
|
|
|
1
1
|
import crypto from "node:crypto";
|
|
2
|
+
import { constants as fsConstants } from "node:fs";
|
|
2
3
|
import fs from "node:fs/promises";
|
|
3
4
|
import path from "node:path";
|
|
5
|
+
import lockfile from "proper-lockfile";
|
|
4
6
|
import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
|
|
7
|
+
import { z } from "zod";
|
|
8
|
+
import { assertArrayField, atomicWriteYamlDocument as writeYamlDocument, durableAtomicWriteYamlDocument as writeRunControlDocument, } from "../artifact-io.js";
|
|
9
|
+
import { assertObligation } from "./obligation-assertion.js";
|
|
10
|
+
import { assertDispatchFallbackRunControlHasNoLiveOwner, assertDispatchFallbackSessionAdmission, } from "./dispatch-fallback-artifacts.js";
|
|
11
|
+
import { RECONSTRUCT_LLM_DISPATCH_FAILURE_DIR, ReconstructLlmDispatchFailureArtifactSchema, assertReconstructLlmDispatchFailureDirectory, createReconstructLlmDispatchFailureArtifact, isReconstructLlmDispatchFailureRef, isReconstructLlmDispatchFailureTempRef, planReconstructLlmDispatchFailureWrite, publishReconstructLlmDispatchFailureTemp, readReconstructLlmDispatchFailureArtifact, readReconstructLlmDispatchFailureArtifactWithHash, reconstructLlmDispatchFailurePath, sha256ReconstructLlmDispatchFailureArtifact, writeReconstructLlmDispatchFailureTemp, } from "./llm-dispatch-failure.js";
|
|
5
12
|
function isoNow() {
|
|
6
13
|
return new Date().toISOString();
|
|
7
14
|
}
|
|
8
15
|
function sha256(value) {
|
|
9
16
|
return crypto.createHash("sha256").update(value).digest("hex");
|
|
10
17
|
}
|
|
18
|
+
const runControlMutationQueues = new Map();
|
|
19
|
+
async function withRunControlMutationLock(runControlPath, task) {
|
|
20
|
+
const key = path.resolve(runControlPath);
|
|
21
|
+
const previous = runControlMutationQueues.get(key) ?? Promise.resolve();
|
|
22
|
+
let releaseQueue = () => { };
|
|
23
|
+
const current = new Promise((resolve) => {
|
|
24
|
+
releaseQueue = resolve;
|
|
25
|
+
});
|
|
26
|
+
const queued = previous.then(() => current);
|
|
27
|
+
runControlMutationQueues.set(key, queued);
|
|
28
|
+
await previous;
|
|
29
|
+
const lockPath = `${key}.write-lock`;
|
|
30
|
+
let releaseFileLock = null;
|
|
31
|
+
try {
|
|
32
|
+
await fs.mkdir(path.dirname(key), { recursive: true });
|
|
33
|
+
releaseFileLock = await lockfile.lock(key, {
|
|
34
|
+
lockfilePath: lockPath,
|
|
35
|
+
realpath: false,
|
|
36
|
+
stale: 5_000,
|
|
37
|
+
update: 1_000,
|
|
38
|
+
retries: {
|
|
39
|
+
retries: 600,
|
|
40
|
+
factor: 1,
|
|
41
|
+
minTimeout: 50,
|
|
42
|
+
maxTimeout: 50,
|
|
43
|
+
randomize: false,
|
|
44
|
+
},
|
|
45
|
+
});
|
|
46
|
+
return await task();
|
|
47
|
+
}
|
|
48
|
+
finally {
|
|
49
|
+
try {
|
|
50
|
+
if (releaseFileLock)
|
|
51
|
+
await releaseFileLock();
|
|
52
|
+
}
|
|
53
|
+
finally {
|
|
54
|
+
releaseQueue();
|
|
55
|
+
if (runControlMutationQueues.get(key) === queued) {
|
|
56
|
+
runControlMutationQueues.delete(key);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
11
61
|
async function sha256File(filePath) {
|
|
12
62
|
try {
|
|
13
63
|
return crypto
|
|
@@ -21,16 +71,29 @@ async function sha256File(filePath) {
|
|
|
21
71
|
throw error;
|
|
22
72
|
}
|
|
23
73
|
}
|
|
24
|
-
async function writeYamlDocument(filePath, value) {
|
|
25
|
-
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
26
|
-
await fs.writeFile(filePath, stringifyYaml(value), "utf8");
|
|
27
|
-
}
|
|
28
74
|
async function writeYamlDocumentAtomicCreate(filePath, value) {
|
|
29
75
|
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
30
76
|
const tempPath = path.join(path.dirname(filePath), `.${path.basename(filePath)}.${process.pid}.${crypto.randomUUID()}.tmp`);
|
|
31
|
-
await fs.
|
|
77
|
+
const handle = await fs.open(tempPath, fsConstants.O_CREAT |
|
|
78
|
+
fsConstants.O_EXCL |
|
|
79
|
+
fsConstants.O_WRONLY |
|
|
80
|
+
fsConstants.O_NOFOLLOW, 0o600);
|
|
81
|
+
try {
|
|
82
|
+
await handle.writeFile(stringifyYaml(value), "utf8");
|
|
83
|
+
await handle.sync();
|
|
84
|
+
}
|
|
85
|
+
finally {
|
|
86
|
+
await handle.close();
|
|
87
|
+
}
|
|
32
88
|
try {
|
|
33
89
|
await fs.link(tempPath, filePath);
|
|
90
|
+
const directoryHandle = await fs.open(path.dirname(filePath), fsConstants.O_RDONLY);
|
|
91
|
+
try {
|
|
92
|
+
await directoryHandle.sync();
|
|
93
|
+
}
|
|
94
|
+
finally {
|
|
95
|
+
await directoryHandle.close();
|
|
96
|
+
}
|
|
34
97
|
return true;
|
|
35
98
|
}
|
|
36
99
|
catch (error) {
|
|
@@ -137,9 +200,15 @@ function violation(args) {
|
|
|
137
200
|
};
|
|
138
201
|
}
|
|
139
202
|
function requiresTerminalValidationTrust(runControl) {
|
|
140
|
-
|
|
203
|
+
assertArrayField(runControl.attempt_rows, "run-control", "attempt_rows");
|
|
204
|
+
assertArrayField(runControl.resume_rows, "run-control", "resume_rows");
|
|
205
|
+
// A graceful-terminal `halted` attempt (design §16.6) is a terminal outcome that produced its
|
|
206
|
+
// own terminal manifest validation, so it demands the same terminal-validation trust as a
|
|
207
|
+
// completed attempt — it must not slip past the fail-closed authority check.
|
|
208
|
+
return runControl.attempt_rows.some((row) => row.attempt_status === "completed" || row.attempt_status === "halted") || runControl.resume_rows.some((row) => row.resume_decision === "resume_allowed");
|
|
141
209
|
}
|
|
142
210
|
function inferTerminalValidationRef(args) {
|
|
211
|
+
assertArrayField(args.runControl.write_transactions, "run-control", "write_transactions");
|
|
143
212
|
if (args.explicitRef !== undefined) {
|
|
144
213
|
return args.explicitRef ? path.resolve(args.explicitRef) : null;
|
|
145
214
|
}
|
|
@@ -155,7 +224,34 @@ function inferTerminalValidationRef(args) {
|
|
|
155
224
|
return path.resolve(path.dirname(args.runControlPath), TERMINAL_VALIDATION_FILENAME);
|
|
156
225
|
}
|
|
157
226
|
export function validateReconstructRunControl(args) {
|
|
227
|
+
assertArrayField(args.runControl.request_rows, "run-control", "request_rows");
|
|
228
|
+
assertArrayField(args.runControl.attempt_rows, "run-control", "attempt_rows");
|
|
229
|
+
assertArrayField(args.runControl.lock_rows, "run-control", "lock_rows");
|
|
230
|
+
assertArrayField(args.runControl.write_transactions, "run-control", "write_transactions");
|
|
231
|
+
assertArrayField(args.runControl.resume_rows, "run-control", "resume_rows");
|
|
158
232
|
const violations = [];
|
|
233
|
+
// G(a) slice 10: record the one obligation whose enforcement matches the authoritative contract
|
|
234
|
+
// (ontology-seeding-and-maturation-design.md §"reconstruct-run-control-validation.yaml must prove")
|
|
235
|
+
// — committed write transactions carry artifact_refs and hashes — stamped before the per-transaction
|
|
236
|
+
// loop so it fires on a zero-transaction artifact. The block below trips invalid_transaction (missing
|
|
237
|
+
// artifact_ref/owner_attempt_id) and transaction_hash_missing (committed without committed_hash).
|
|
238
|
+
// PARKED (not recorded), each name broader than this validator's actual check — these are honest
|
|
239
|
+
// declared≠wired surfacings, NOT laundered (see obligation-coverage-ledger.yaml notes):
|
|
240
|
+
// - preserve_post_write_hash_observation_without_claiming_atomic_commit_when_writer_did_not_prove_atomic_rename:
|
|
241
|
+
// the validator never reads commit_method; the atomic_rename-vs-observed_file_hash truthfulness
|
|
242
|
+
// is the writer's responsibility, not validated here.
|
|
243
|
+
// - reject_conflicting_request_fingerprints_before_semantic_artifacts_are_consumed: the validator
|
|
244
|
+
// rejects rows already marked duplicate_conflict/rejected_conflict (conflicting_request) but never
|
|
245
|
+
// compares request_fingerprint values — fingerprint conflict detection is in the writer.
|
|
246
|
+
// - validate_current_attempt_and_session_root_lock_ownership: presence of an active attempt
|
|
247
|
+
// (active_attempt_missing) + a session_root lock (session_lock_missing) + rejection of recorded
|
|
248
|
+
// conflicting locks (conflicting_lock) are checked, but the lock's owner_attempt_id is never
|
|
249
|
+
// cross-checked against the current attempt — the named ownership linkage is unverified.
|
|
250
|
+
// - validate_session_root_request_fingerprint_target_signature_runtime_version_and_idempotency_are_replayable:
|
|
251
|
+
// of the five named replayable quantities only session_root is validated; request_fingerprint,
|
|
252
|
+
// target signature, runtime version, and idempotency key are never inspected.
|
|
253
|
+
const assertedObligationIds = [];
|
|
254
|
+
assertObligation(assertedObligationIds, "validate_committed_write_transactions_have_artifact_refs_and_hashes");
|
|
159
255
|
if (args.runControl.schema_version !== "1") {
|
|
160
256
|
violations.push(violation({
|
|
161
257
|
code: "schema_shape_invalid",
|
|
@@ -203,15 +299,89 @@ export function validateReconstructRunControl(args) {
|
|
|
203
299
|
message: "run-control must record at least one attempt row",
|
|
204
300
|
}));
|
|
205
301
|
}
|
|
206
|
-
const
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
row
|
|
210
|
-
|
|
302
|
+
const duplicateIds = (rows, field) => {
|
|
303
|
+
const seen = new Set();
|
|
304
|
+
const duplicates = new Set();
|
|
305
|
+
for (const row of rows) {
|
|
306
|
+
const value = row[field];
|
|
307
|
+
if (typeof value !== "string")
|
|
308
|
+
continue;
|
|
309
|
+
if (seen.has(value))
|
|
310
|
+
duplicates.add(value);
|
|
311
|
+
seen.add(value);
|
|
312
|
+
}
|
|
313
|
+
return [...duplicates];
|
|
314
|
+
};
|
|
315
|
+
for (const [label, rows, field] of [
|
|
316
|
+
["request", args.runControl.request_rows, "request_id"],
|
|
317
|
+
["attempt", args.runControl.attempt_rows, "attempt_id"],
|
|
318
|
+
["lock", args.runControl.lock_rows, "lock_id"],
|
|
319
|
+
["transaction", args.runControl.write_transactions, "transaction_id"],
|
|
320
|
+
["resume", args.runControl.resume_rows, "resume_id"],
|
|
321
|
+
]) {
|
|
322
|
+
for (const duplicateId of duplicateIds(rows, field)) {
|
|
323
|
+
violations.push(violation({
|
|
324
|
+
code: "schema_shape_invalid",
|
|
325
|
+
message: `run-control ${label} ids must be unique`,
|
|
326
|
+
subjectId: duplicateId,
|
|
327
|
+
}));
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
const latestAttempt = args.runControl.attempt_rows.at(-1) ?? null;
|
|
331
|
+
let trustedFailedAttempt = false;
|
|
332
|
+
if (latestAttempt?.attempt_status === "failed") {
|
|
333
|
+
const failureTransactions = args.runControl.write_transactions.filter((row) => row.owner_attempt_id === latestAttempt.attempt_id &&
|
|
334
|
+
isReconstructLlmDispatchFailureRef(args.runControl.session_root, row.artifact_ref));
|
|
335
|
+
const transaction = failureTransactions.length === 1
|
|
336
|
+
? failureTransactions[0]
|
|
337
|
+
: undefined;
|
|
338
|
+
const parsedArtifact = ReconstructLlmDispatchFailureArtifactSchema.safeParse(args.failedTerminalArtifact);
|
|
339
|
+
const ownerSessionLocks = args.runControl.lock_rows.filter((row) => row.lock_scope === "session_root" &&
|
|
340
|
+
row.owner_attempt_id === latestAttempt.attempt_id);
|
|
341
|
+
if (!transaction || args.failedTerminalArtifactRef === undefined) {
|
|
342
|
+
violations.push(violation({
|
|
343
|
+
code: "failed_terminal_missing",
|
|
344
|
+
message: "latest failed attempt requires an owner-linked LLM dispatch failure transaction",
|
|
345
|
+
subjectId: latestAttempt.attempt_id,
|
|
346
|
+
}));
|
|
347
|
+
}
|
|
348
|
+
else if (transaction.transaction_status !== "committed" ||
|
|
349
|
+
!transaction.committed_hash ||
|
|
350
|
+
transaction.prepared_content_hash !== transaction.committed_hash ||
|
|
351
|
+
path.resolve(transaction.artifact_ref) !==
|
|
352
|
+
path.resolve(args.failedTerminalArtifactRef ?? "") ||
|
|
353
|
+
transaction.committed_hash !== args.failedTerminalArtifactSha256 ||
|
|
354
|
+
!parsedArtifact.success ||
|
|
355
|
+
parsedArtifact.data.session_id !== args.runControl.session_id ||
|
|
356
|
+
(args.expectedSessionId !== undefined &&
|
|
357
|
+
parsedArtifact.data.session_id !== args.expectedSessionId) ||
|
|
358
|
+
parsedArtifact.data.owner_attempt_id !== latestAttempt.attempt_id ||
|
|
359
|
+
path.resolve(transaction.artifact_ref) !== path.resolve(reconstructLlmDispatchFailurePath(args.runControl.session_root, parsedArtifact.data.failure_id)) ||
|
|
360
|
+
ownerSessionLocks.length !== 1 ||
|
|
361
|
+
ownerSessionLocks[0]?.lock_status !== "released" ||
|
|
362
|
+
args.runControl.lock_rows.some((row) => row.lock_scope === "session_root" && row.lock_status === "held")) {
|
|
363
|
+
violations.push(violation({
|
|
364
|
+
code: "failed_terminal_invalid",
|
|
365
|
+
message: "latest failed attempt failure artifact must match owner, ref, schema, prepared hash, and committed hash",
|
|
366
|
+
subjectId: transaction.transaction_id,
|
|
367
|
+
}));
|
|
368
|
+
}
|
|
369
|
+
else {
|
|
370
|
+
trustedFailedAttempt = true;
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
const currentAttempt = trustedFailedAttempt
|
|
374
|
+
? latestAttempt
|
|
375
|
+
: [...args.runControl.attempt_rows]
|
|
376
|
+
.reverse()
|
|
377
|
+
.find((row) => row.attempt_status === "running" ||
|
|
378
|
+
row.attempt_status === "completed" ||
|
|
379
|
+
row.attempt_status === "recovered" ||
|
|
380
|
+
row.attempt_status === "halted") ?? null;
|
|
211
381
|
if (!currentAttempt) {
|
|
212
382
|
violations.push(violation({
|
|
213
383
|
code: "active_attempt_missing",
|
|
214
|
-
message: "run-control must have a running, completed, or
|
|
384
|
+
message: "run-control must have a running, completed, recovered, halted, or trusted failed attempt",
|
|
215
385
|
}));
|
|
216
386
|
}
|
|
217
387
|
const activeLocks = args.runControl.lock_rows.filter((row) => row.lock_scope === "session_root" &&
|
|
@@ -222,6 +392,17 @@ export function validateReconstructRunControl(args) {
|
|
|
222
392
|
message: "run-control must record a session_root lock",
|
|
223
393
|
}));
|
|
224
394
|
}
|
|
395
|
+
if (currentAttempt?.attempt_status === "running") {
|
|
396
|
+
const heldSessionLocks = args.runControl.lock_rows.filter((row) => row.lock_scope === "session_root" && row.lock_status === "held");
|
|
397
|
+
if (heldSessionLocks.length !== 1 ||
|
|
398
|
+
heldSessionLocks[0]?.owner_attempt_id !== currentAttempt.attempt_id) {
|
|
399
|
+
violations.push(violation({
|
|
400
|
+
code: "conflicting_lock",
|
|
401
|
+
message: "the current running attempt must be the unique held session_root lock owner",
|
|
402
|
+
subjectId: currentAttempt.attempt_id,
|
|
403
|
+
}));
|
|
404
|
+
}
|
|
405
|
+
}
|
|
225
406
|
if (args.runControl.lock_rows.some((row) => row.lock_status === "conflict_blocked" ||
|
|
226
407
|
row.lock_status === "stolen_invalid")) {
|
|
227
408
|
violations.push(violation({
|
|
@@ -244,6 +425,18 @@ export function validateReconstructRunControl(args) {
|
|
|
244
425
|
subjectId: row.transaction_id,
|
|
245
426
|
}));
|
|
246
427
|
}
|
|
428
|
+
if (isReconstructLlmDispatchFailureRef(args.runControl.session_root, row.artifact_ref)) {
|
|
429
|
+
if (!row.prepared_content_hash ||
|
|
430
|
+
(row.transaction_status === "prepared" && !row.temp_ref) ||
|
|
431
|
+
(row.transaction_status === "committed" &&
|
|
432
|
+
row.prepared_content_hash !== row.committed_hash)) {
|
|
433
|
+
violations.push(violation({
|
|
434
|
+
code: "invalid_transaction",
|
|
435
|
+
message: "LLM dispatch failure transaction must preserve prepared content hash and temp/commit state",
|
|
436
|
+
subjectId: row.transaction_id,
|
|
437
|
+
}));
|
|
438
|
+
}
|
|
439
|
+
}
|
|
247
440
|
}
|
|
248
441
|
const attemptIds = new Set(args.runControl.attempt_rows.map((row) => row.attempt_id));
|
|
249
442
|
for (const row of args.runControl.resume_rows) {
|
|
@@ -310,7 +503,7 @@ export function validateReconstructRunControl(args) {
|
|
|
310
503
|
if (!terminalValidationRef) {
|
|
311
504
|
violations.push(violation({
|
|
312
505
|
code: "terminal_validation_missing",
|
|
313
|
-
message: "completed attempts and resume_allowed rows require
|
|
506
|
+
message: "completed/halted attempts and resume_allowed rows require terminal run-manifest validation authority",
|
|
314
507
|
}));
|
|
315
508
|
}
|
|
316
509
|
else {
|
|
@@ -319,14 +512,14 @@ export function validateReconstructRunControl(args) {
|
|
|
319
512
|
code: args.terminalValidationStatus === null
|
|
320
513
|
? "terminal_validation_missing"
|
|
321
514
|
: "terminal_validation_invalid",
|
|
322
|
-
message: "
|
|
515
|
+
message: "terminal run-manifest validation must exist and have validation_status=valid before completion, halt, or resume_allowed",
|
|
323
516
|
subjectId: terminalValidationRef,
|
|
324
517
|
}));
|
|
325
518
|
}
|
|
326
519
|
if (!committedRefs.has(path.resolve(terminalValidationRef))) {
|
|
327
520
|
violations.push(violation({
|
|
328
521
|
code: "expected_transaction_missing",
|
|
329
|
-
message: "run-control validation is missing a committed hash transaction for the
|
|
522
|
+
message: "run-control validation is missing a committed hash transaction for the terminal run-manifest validation artifact",
|
|
330
523
|
subjectId: terminalValidationRef,
|
|
331
524
|
}));
|
|
332
525
|
}
|
|
@@ -355,21 +548,42 @@ export function validateReconstructRunControl(args) {
|
|
|
355
548
|
validation_results: violations.length === 0
|
|
356
549
|
? ["reconstruct_run_control_valid"]
|
|
357
550
|
: ["reconstruct_run_control_invalid"],
|
|
551
|
+
asserted_obligation_ids: assertedObligationIds,
|
|
358
552
|
violations,
|
|
359
553
|
};
|
|
360
554
|
}
|
|
361
|
-
|
|
362
|
-
const runControl = await readYamlDocument(args.runControlPath);
|
|
555
|
+
async function buildReconstructRunControlValidationArtifactFromRunControl(args) {
|
|
363
556
|
const terminalValidationRef = inferTerminalValidationRef({
|
|
364
|
-
runControl,
|
|
557
|
+
runControl: args.runControl,
|
|
365
558
|
runControlPath: args.runControlPath,
|
|
366
559
|
...(args.terminalValidationRef !== undefined
|
|
367
560
|
? { explicitRef: args.terminalValidationRef }
|
|
368
561
|
: {}),
|
|
369
562
|
});
|
|
370
563
|
const terminalValidationStatus = await readValidationStatusIfPresent(terminalValidationRef);
|
|
371
|
-
const
|
|
372
|
-
|
|
564
|
+
const latestAttempt = args.runControl.attempt_rows.at(-1) ?? null;
|
|
565
|
+
const failedTransaction = latestAttempt?.attempt_status === "failed"
|
|
566
|
+
? args.runControl.write_transactions.find((row) => row.owner_attempt_id === latestAttempt.attempt_id &&
|
|
567
|
+
isReconstructLlmDispatchFailureRef(args.runControl.session_root, row.artifact_ref))
|
|
568
|
+
: undefined;
|
|
569
|
+
let failedTerminalArtifact = null;
|
|
570
|
+
let failedTerminalArtifactSha256 = null;
|
|
571
|
+
if (failedTransaction) {
|
|
572
|
+
try {
|
|
573
|
+
const failedRead = await readReconstructLlmDispatchFailureArtifactWithHash({
|
|
574
|
+
sessionRoot: args.runControl.session_root,
|
|
575
|
+
artifactRef: failedTransaction.artifact_ref,
|
|
576
|
+
});
|
|
577
|
+
failedTerminalArtifact = failedRead.artifact;
|
|
578
|
+
failedTerminalArtifactSha256 = failedRead.sha256;
|
|
579
|
+
}
|
|
580
|
+
catch {
|
|
581
|
+
failedTerminalArtifact = null;
|
|
582
|
+
failedTerminalArtifactSha256 = null;
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
return validateReconstructRunControl({
|
|
586
|
+
runControl: args.runControl,
|
|
373
587
|
runControlRef: args.runControlPath,
|
|
374
588
|
...(args.expectedSessionId !== undefined
|
|
375
589
|
? { expectedSessionId: args.expectedSessionId }
|
|
@@ -382,10 +596,81 @@ export async function writeReconstructRunControlValidationArtifact(args) {
|
|
|
382
596
|
: {}),
|
|
383
597
|
terminalValidationRef,
|
|
384
598
|
terminalValidationStatus,
|
|
599
|
+
...(failedTransaction
|
|
600
|
+
? {
|
|
601
|
+
failedTerminalArtifactRef: failedTransaction.artifact_ref,
|
|
602
|
+
failedTerminalArtifact,
|
|
603
|
+
failedTerminalArtifactSha256,
|
|
604
|
+
}
|
|
605
|
+
: {}),
|
|
606
|
+
});
|
|
607
|
+
}
|
|
608
|
+
async function buildReconstructRunControlValidationArtifact(args) {
|
|
609
|
+
const runControl = await readYamlDocument(args.runControlPath);
|
|
610
|
+
return buildReconstructRunControlValidationArtifactFromRunControl({
|
|
611
|
+
...args,
|
|
612
|
+
runControl,
|
|
385
613
|
});
|
|
614
|
+
}
|
|
615
|
+
export async function writeReconstructRunControlValidationArtifact(args) {
|
|
616
|
+
const validation = await buildReconstructRunControlValidationArtifact(args);
|
|
386
617
|
await writeYamlDocument(args.outputPath, validation);
|
|
387
618
|
return validation;
|
|
388
619
|
}
|
|
620
|
+
function comparableRunControlValidation(validation, options) {
|
|
621
|
+
if (validation === null || typeof validation !== "object" || Array.isArray(validation)) {
|
|
622
|
+
return null;
|
|
623
|
+
}
|
|
624
|
+
const persisted = { ...validation };
|
|
625
|
+
if (options.stripInMemoryFields) {
|
|
626
|
+
delete persisted.asserted_obligation_ids;
|
|
627
|
+
}
|
|
628
|
+
const parsed = PersistedRunControlValidationSchema.safeParse(persisted);
|
|
629
|
+
if (!parsed.success)
|
|
630
|
+
return null;
|
|
631
|
+
try {
|
|
632
|
+
return JSON.stringify({ ...parsed.data, created_at: null });
|
|
633
|
+
}
|
|
634
|
+
catch {
|
|
635
|
+
return null;
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
const PersistedRunControlValidationSchema = z.object({
|
|
639
|
+
schema_version: z.literal("1"),
|
|
640
|
+
session_id: z.string().min(1),
|
|
641
|
+
created_at: z.string().datetime({ offset: true }),
|
|
642
|
+
reconstruct_run_control_ref: z.string().min(1).nullable(),
|
|
643
|
+
validation_status: z.enum(["valid", "invalid"]),
|
|
644
|
+
request_count: z.number().int().nonnegative(),
|
|
645
|
+
attempt_count: z.number().int().nonnegative(),
|
|
646
|
+
active_lock_count: z.number().int().nonnegative(),
|
|
647
|
+
transaction_count: z.number().int().nonnegative(),
|
|
648
|
+
current_attempt_id: z.string().min(1).nullable(),
|
|
649
|
+
validation_results: z.array(z.string()),
|
|
650
|
+
violations: z.array(z.object({
|
|
651
|
+
code: z.enum([
|
|
652
|
+
"schema_shape_invalid",
|
|
653
|
+
"session_id_mismatch",
|
|
654
|
+
"session_root_missing",
|
|
655
|
+
"request_row_missing",
|
|
656
|
+
"attempt_row_missing",
|
|
657
|
+
"active_attempt_missing",
|
|
658
|
+
"session_lock_missing",
|
|
659
|
+
"conflicting_request",
|
|
660
|
+
"conflicting_lock",
|
|
661
|
+
"invalid_transaction",
|
|
662
|
+
"transaction_hash_missing",
|
|
663
|
+
"terminal_validation_missing",
|
|
664
|
+
"terminal_validation_invalid",
|
|
665
|
+
"expected_transaction_missing",
|
|
666
|
+
"failed_terminal_missing",
|
|
667
|
+
"failed_terminal_invalid",
|
|
668
|
+
"invalid_resume",
|
|
669
|
+
]),
|
|
670
|
+
message: z.string(),
|
|
671
|
+
subject_id: z.string().nullable(),
|
|
672
|
+
}).strict()),
|
|
673
|
+
}).strict();
|
|
389
674
|
export async function writeReconstructRunBootstrapDiagnostic(args) {
|
|
390
675
|
const artifact = {
|
|
391
676
|
schema_version: "1",
|
|
@@ -402,7 +687,11 @@ export async function writeReconstructRunBootstrapDiagnostic(args) {
|
|
|
402
687
|
await writeYamlDocument(args.outputPath, artifact);
|
|
403
688
|
return artifact;
|
|
404
689
|
}
|
|
405
|
-
|
|
690
|
+
async function initializeReconstructRunControlUnlocked(args) {
|
|
691
|
+
await assertDispatchFallbackSessionAdmission({
|
|
692
|
+
sessionRoot: args.sessionRoot,
|
|
693
|
+
enabled: args.dispatchFallbackEnabled === true,
|
|
694
|
+
});
|
|
406
695
|
const requestFingerprint = reconstructRequestFingerprint({
|
|
407
696
|
projectRoot: args.projectRoot,
|
|
408
697
|
sessionRoot: args.sessionRoot,
|
|
@@ -417,6 +706,10 @@ export async function initializeReconstructRunControl(args) {
|
|
|
417
706
|
const idempotencyKeyHash = sha256(`reconstruct:${requestFingerprint}`);
|
|
418
707
|
const existing = await readYamlDocumentIfPresent(args.outputPath);
|
|
419
708
|
if (existing) {
|
|
709
|
+
assertDispatchFallbackRunControlHasNoLiveOwner({
|
|
710
|
+
runControl: existing,
|
|
711
|
+
enabled: args.dispatchFallbackEnabled === true,
|
|
712
|
+
});
|
|
420
713
|
const conflict = existing.request_rows.some((row) => row.request_fingerprint !== requestFingerprint);
|
|
421
714
|
if (conflict) {
|
|
422
715
|
await writeReconstructRunBootstrapDiagnostic({
|
|
@@ -435,11 +728,43 @@ export async function initializeReconstructRunControl(args) {
|
|
|
435
728
|
if (!completedAttempt) {
|
|
436
729
|
const now = isoNow();
|
|
437
730
|
const sourceAttempt = [...existing.attempt_rows].reverse()[0] ?? null;
|
|
438
|
-
const
|
|
731
|
+
const heldSessionLock = existing.lock_rows.find((row) => row.lock_scope === "session_root" && row.lock_status === "held");
|
|
732
|
+
if (sourceAttempt?.attempt_status === "running" ||
|
|
733
|
+
heldSessionLock !== undefined) {
|
|
734
|
+
throw new Error("cannot resume a live reconstruct attempt; lease expiry is not takeover authority");
|
|
735
|
+
}
|
|
736
|
+
if (existing.resume_rows.some((row) => row.source_attempt_id === sourceAttempt?.attempt_id &&
|
|
737
|
+
row.resume_decision === "blocked_partial_write")) {
|
|
738
|
+
throw new Error("cannot resume reconstruct session with blocked_partial_write authority");
|
|
739
|
+
}
|
|
740
|
+
let failureProvenanceRefs = [];
|
|
741
|
+
const sourceFailureTransactions = sourceAttempt?.attempt_status === "failed"
|
|
742
|
+
? existing.write_transactions.filter((row) => row.owner_attempt_id === sourceAttempt.attempt_id &&
|
|
743
|
+
isReconstructLlmDispatchFailureRef(args.sessionRoot, row.artifact_ref))
|
|
744
|
+
: [];
|
|
745
|
+
if (sourceFailureTransactions.length > 0) {
|
|
746
|
+
const failedValidation = await writeReconstructRunControlValidationArtifact({
|
|
747
|
+
runControlPath: args.outputPath,
|
|
748
|
+
outputPath: args.validationOutputPath,
|
|
749
|
+
expectedSessionId: args.sessionId,
|
|
750
|
+
expectedSessionRoot: args.sessionRoot,
|
|
751
|
+
});
|
|
752
|
+
if (failedValidation.validation_status !== "valid") {
|
|
753
|
+
throw new Error(`cannot resume from an untrusted failed terminal: ${failedValidation.violations.map((item) => item.code).join(",")}`);
|
|
754
|
+
}
|
|
755
|
+
failureProvenanceRefs = sourceFailureTransactions
|
|
756
|
+
.filter((row) => row.transaction_status === "committed")
|
|
757
|
+
.map((row) => row.artifact_ref);
|
|
758
|
+
if (failureProvenanceRefs.length !== 1) {
|
|
759
|
+
throw new Error("cannot resume without exactly one trusted failure provenance ref");
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
const resumeId = idFor("resume", `${requestFingerprint}:${now}:${crypto.randomUUID()}`);
|
|
439
763
|
const attemptId = idFor("attempt", `${resumeId}:attempt`);
|
|
440
764
|
const trustedArtifactRefs = existing.write_transactions
|
|
441
765
|
.filter((row) => row.transaction_status === "committed" &&
|
|
442
|
-
row.committed_hash !== null
|
|
766
|
+
row.committed_hash !== null &&
|
|
767
|
+
!isReconstructLlmDispatchFailureRef(args.sessionRoot, row.artifact_ref))
|
|
443
768
|
.map((row) => row.artifact_ref)
|
|
444
769
|
.sort();
|
|
445
770
|
const provenanceMatchRefs = await collectResumeProvenanceMatchRefs(args.sessionRoot);
|
|
@@ -448,6 +773,7 @@ export async function initializeReconstructRunControl(args) {
|
|
|
448
773
|
args.validationOutputPath,
|
|
449
774
|
...provenanceMatchRefs,
|
|
450
775
|
...trustedArtifactRefs,
|
|
776
|
+
...failureProvenanceRefs,
|
|
451
777
|
];
|
|
452
778
|
existing.updated_at = now;
|
|
453
779
|
existing.resume_rows.push({
|
|
@@ -463,6 +789,7 @@ export async function initializeReconstructRunControl(args) {
|
|
|
463
789
|
args.validationOutputPath,
|
|
464
790
|
...provenanceMatchRefs,
|
|
465
791
|
...trustedArtifactRefs,
|
|
792
|
+
...failureProvenanceRefs,
|
|
466
793
|
].sort(),
|
|
467
794
|
resume_decision: "resume_pending_provenance",
|
|
468
795
|
});
|
|
@@ -488,6 +815,7 @@ export async function initializeReconstructRunControl(args) {
|
|
|
488
815
|
args.outputPath,
|
|
489
816
|
args.validationOutputPath,
|
|
490
817
|
...trustedArtifactRefs,
|
|
818
|
+
...failureProvenanceRefs,
|
|
491
819
|
],
|
|
492
820
|
});
|
|
493
821
|
existing.lock_rows = existing.lock_rows.map((row) => row.lock_scope === "session_root" && row.lock_status === "held"
|
|
@@ -503,7 +831,7 @@ export async function initializeReconstructRunControl(args) {
|
|
|
503
831
|
conflict_policy: "recover_expired_lease",
|
|
504
832
|
lock_status: "held",
|
|
505
833
|
});
|
|
506
|
-
await
|
|
834
|
+
await writeRunControlDocument(args.outputPath, existing);
|
|
507
835
|
const validation = await writeReconstructRunControlValidationArtifact({
|
|
508
836
|
runControlPath: args.outputPath,
|
|
509
837
|
outputPath: args.validationOutputPath,
|
|
@@ -578,7 +906,7 @@ export async function initializeReconstructRunControl(args) {
|
|
|
578
906
|
};
|
|
579
907
|
const created = await writeYamlDocumentAtomicCreate(args.outputPath, runControl);
|
|
580
908
|
if (!created) {
|
|
581
|
-
return
|
|
909
|
+
return initializeReconstructRunControlUnlocked(args);
|
|
582
910
|
}
|
|
583
911
|
const validation = await writeReconstructRunControlValidationArtifact({
|
|
584
912
|
runControlPath: args.outputPath,
|
|
@@ -588,7 +916,10 @@ export async function initializeReconstructRunControl(args) {
|
|
|
588
916
|
});
|
|
589
917
|
return { runControl, validation, requestFingerprint, attemptId };
|
|
590
918
|
}
|
|
591
|
-
export async function
|
|
919
|
+
export async function initializeReconstructRunControl(args) {
|
|
920
|
+
return withRunControlMutationLock(args.outputPath, () => initializeReconstructRunControlUnlocked(args));
|
|
921
|
+
}
|
|
922
|
+
async function markReconstructRunControlAttemptFailedUnlocked(args) {
|
|
592
923
|
const runControl = await readYamlDocument(args.runControlPath);
|
|
593
924
|
const failedAt = isoNow();
|
|
594
925
|
let changed = false;
|
|
@@ -608,7 +939,7 @@ export async function markReconstructRunControlAttemptFailed(args) {
|
|
|
608
939
|
runControl.lock_rows = runControl.lock_rows.map((row) => row.owner_attempt_id === args.attemptId && row.lock_status === "held"
|
|
609
940
|
? { ...row, lock_status: "released" }
|
|
610
941
|
: row);
|
|
611
|
-
await
|
|
942
|
+
await writeRunControlDocument(args.runControlPath, runControl);
|
|
612
943
|
}
|
|
613
944
|
const validation = await writeReconstructRunControlValidationArtifact({
|
|
614
945
|
runControlPath: args.runControlPath,
|
|
@@ -618,6 +949,617 @@ export async function markReconstructRunControlAttemptFailed(args) {
|
|
|
618
949
|
});
|
|
619
950
|
return { runControl, validation };
|
|
620
951
|
}
|
|
952
|
+
export async function markReconstructRunControlAttemptFailed(args) {
|
|
953
|
+
return withRunControlMutationLock(args.runControlPath, () => markReconstructRunControlAttemptFailedUnlocked(args));
|
|
954
|
+
}
|
|
955
|
+
async function persistReconstructLlmDispatchFailureUnlocked(args) {
|
|
956
|
+
let runControl = await readYamlDocument(args.runControlPath);
|
|
957
|
+
const latestAttempt = runControl.attempt_rows.at(-1) ?? null;
|
|
958
|
+
const ownerLocks = runControl.lock_rows.filter((row) => row.owner_attempt_id === args.attemptId &&
|
|
959
|
+
row.lock_scope === "session_root" &&
|
|
960
|
+
row.lock_status === "held");
|
|
961
|
+
if (latestAttempt?.attempt_id !== args.attemptId ||
|
|
962
|
+
latestAttempt.attempt_status !== "running" ||
|
|
963
|
+
ownerLocks.length !== 1) {
|
|
964
|
+
throw new Error(`cannot persist LLM dispatch failure for non-owning attempt ${args.attemptId}`);
|
|
965
|
+
}
|
|
966
|
+
const artifact = createReconstructLlmDispatchFailureArtifact({
|
|
967
|
+
sessionId: args.sessionId,
|
|
968
|
+
attemptId: args.attemptId,
|
|
969
|
+
error: args.error,
|
|
970
|
+
});
|
|
971
|
+
const plan = await planReconstructLlmDispatchFailureWrite({
|
|
972
|
+
sessionRoot: args.sessionRoot,
|
|
973
|
+
artifact,
|
|
974
|
+
});
|
|
975
|
+
const temp = await writeReconstructLlmDispatchFailureTemp({
|
|
976
|
+
sessionRoot: args.sessionRoot,
|
|
977
|
+
artifact,
|
|
978
|
+
plan,
|
|
979
|
+
});
|
|
980
|
+
await args.faultInjector?.("after_temp_write");
|
|
981
|
+
const transactionId = idFor("write", `${args.attemptId}:${plan.finalRef}`);
|
|
982
|
+
const existingTransaction = runControl.write_transactions.find((row) => row.transaction_id === transactionId);
|
|
983
|
+
const attempt = runControl.attempt_rows.find((row) => row.attempt_id === args.attemptId);
|
|
984
|
+
const heldLock = runControl.lock_rows.some((row) => row.owner_attempt_id === args.attemptId &&
|
|
985
|
+
row.lock_scope === "session_root" &&
|
|
986
|
+
row.lock_status === "held");
|
|
987
|
+
if ((!attempt || attempt.attempt_status !== "running" || !heldLock) &&
|
|
988
|
+
existingTransaction?.transaction_status !== "committed") {
|
|
989
|
+
throw new Error(`cannot prepare LLM dispatch failure for non-owning attempt ${args.attemptId}`);
|
|
990
|
+
}
|
|
991
|
+
if (existingTransaction) {
|
|
992
|
+
if (existingTransaction.owner_attempt_id !== args.attemptId ||
|
|
993
|
+
path.resolve(existingTransaction.artifact_ref) !== path.resolve(plan.finalRef) ||
|
|
994
|
+
existingTransaction.prepared_content_hash !== plan.contentSha256 ||
|
|
995
|
+
(existingTransaction.transaction_status !== "prepared" &&
|
|
996
|
+
existingTransaction.transaction_status !== "committed")) {
|
|
997
|
+
throw new Error(`LLM dispatch failure transaction conflicts: ${transactionId}`);
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
else {
|
|
1001
|
+
runControl.updated_at = isoNow();
|
|
1002
|
+
runControl.write_transactions.push({
|
|
1003
|
+
transaction_id: transactionId,
|
|
1004
|
+
owner_attempt_id: args.attemptId,
|
|
1005
|
+
artifact_ref: plan.finalRef,
|
|
1006
|
+
temp_ref: plan.tempRef,
|
|
1007
|
+
expected_prior_hash: null,
|
|
1008
|
+
prepared_content_hash: plan.contentSha256,
|
|
1009
|
+
committed_hash: null,
|
|
1010
|
+
commit_method: "append_only",
|
|
1011
|
+
transaction_status: "prepared",
|
|
1012
|
+
recovery_ref: null,
|
|
1013
|
+
});
|
|
1014
|
+
await writeRunControlDocument(args.runControlPath, runControl);
|
|
1015
|
+
}
|
|
1016
|
+
await args.faultInjector?.("after_prepare");
|
|
1017
|
+
await publishReconstructLlmDispatchFailureTemp({
|
|
1018
|
+
sessionRoot: args.sessionRoot,
|
|
1019
|
+
tempRef: temp.tempRef,
|
|
1020
|
+
finalRef: temp.finalRef,
|
|
1021
|
+
});
|
|
1022
|
+
await args.faultInjector?.("after_publish");
|
|
1023
|
+
const publishedRead = await readReconstructLlmDispatchFailureArtifactWithHash({
|
|
1024
|
+
sessionRoot: args.sessionRoot,
|
|
1025
|
+
artifactRef: temp.finalRef,
|
|
1026
|
+
});
|
|
1027
|
+
const published = publishedRead.artifact;
|
|
1028
|
+
const publishedHash = publishedRead.sha256;
|
|
1029
|
+
if (published.failure_id !== artifact.failure_id ||
|
|
1030
|
+
published.owner_attempt_id !== args.attemptId ||
|
|
1031
|
+
publishedHash !== temp.contentSha256) {
|
|
1032
|
+
throw new Error("published LLM dispatch failure artifact failed integrity verification");
|
|
1033
|
+
}
|
|
1034
|
+
runControl = await readYamlDocument(args.runControlPath);
|
|
1035
|
+
const alreadyCommitted = runControl.write_transactions.find((row) => row.transaction_id === transactionId &&
|
|
1036
|
+
row.transaction_status === "committed" &&
|
|
1037
|
+
row.owner_attempt_id === args.attemptId &&
|
|
1038
|
+
row.committed_hash === publishedHash);
|
|
1039
|
+
if (alreadyCommitted) {
|
|
1040
|
+
const validation = await writeReconstructRunControlValidationArtifact({
|
|
1041
|
+
runControlPath: args.runControlPath,
|
|
1042
|
+
outputPath: args.validationOutputPath,
|
|
1043
|
+
expectedSessionId: args.sessionId,
|
|
1044
|
+
expectedSessionRoot: args.sessionRoot,
|
|
1045
|
+
});
|
|
1046
|
+
if (validation.validation_status !== "valid") {
|
|
1047
|
+
throw new Error(`reconciled LLM dispatch failure did not produce a valid failed terminal: ${validation.violations.map((item) => item.code).join(",")}`);
|
|
1048
|
+
}
|
|
1049
|
+
return {
|
|
1050
|
+
artifact: published,
|
|
1051
|
+
artifactRef: temp.finalRef,
|
|
1052
|
+
runControl,
|
|
1053
|
+
validation,
|
|
1054
|
+
};
|
|
1055
|
+
}
|
|
1056
|
+
const preparedTransaction = runControl.write_transactions.find((row) => row.transaction_id === transactionId &&
|
|
1057
|
+
row.transaction_status === "prepared" &&
|
|
1058
|
+
row.owner_attempt_id === args.attemptId &&
|
|
1059
|
+
row.prepared_content_hash === publishedHash);
|
|
1060
|
+
const runningAttempt = runControl.attempt_rows.some((row) => row.attempt_id === args.attemptId && row.attempt_status === "running");
|
|
1061
|
+
const owningLock = runControl.lock_rows.some((row) => row.owner_attempt_id === args.attemptId &&
|
|
1062
|
+
row.lock_scope === "session_root" &&
|
|
1063
|
+
row.lock_status === "held");
|
|
1064
|
+
if (!preparedTransaction || !runningAttempt || !owningLock) {
|
|
1065
|
+
throw new Error(`cannot commit LLM dispatch failure without prepared transaction ownership for ${args.attemptId}`);
|
|
1066
|
+
}
|
|
1067
|
+
const committedAt = isoNow();
|
|
1068
|
+
runControl.updated_at = committedAt;
|
|
1069
|
+
runControl.write_transactions = runControl.write_transactions.map((row) => row.transaction_id === transactionId && row.transaction_status === "prepared"
|
|
1070
|
+
? {
|
|
1071
|
+
...row,
|
|
1072
|
+
temp_ref: null,
|
|
1073
|
+
committed_hash: publishedHash,
|
|
1074
|
+
transaction_status: "committed",
|
|
1075
|
+
}
|
|
1076
|
+
: row);
|
|
1077
|
+
runControl.attempt_rows = runControl.attempt_rows.map((row) => row.attempt_id === args.attemptId && row.attempt_status === "running"
|
|
1078
|
+
? { ...row, completed_at: committedAt, attempt_status: "failed" }
|
|
1079
|
+
: row);
|
|
1080
|
+
runControl.lock_rows = runControl.lock_rows.map((row) => row.owner_attempt_id === args.attemptId && row.lock_status === "held"
|
|
1081
|
+
? { ...row, lock_status: "released" }
|
|
1082
|
+
: row);
|
|
1083
|
+
await writeRunControlDocument(args.runControlPath, runControl);
|
|
1084
|
+
await args.faultInjector?.("after_commit");
|
|
1085
|
+
const validation = await writeReconstructRunControlValidationArtifact({
|
|
1086
|
+
runControlPath: args.runControlPath,
|
|
1087
|
+
outputPath: args.validationOutputPath,
|
|
1088
|
+
expectedSessionId: args.sessionId,
|
|
1089
|
+
expectedSessionRoot: args.sessionRoot,
|
|
1090
|
+
});
|
|
1091
|
+
if (validation.validation_status !== "valid") {
|
|
1092
|
+
throw new Error(`persisted LLM dispatch failure did not produce a valid failed terminal: ${validation.violations.map((item) => item.code).join(",")}`);
|
|
1093
|
+
}
|
|
1094
|
+
return { artifact, artifactRef: temp.finalRef, runControl, validation };
|
|
1095
|
+
}
|
|
1096
|
+
export async function persistReconstructLlmDispatchFailure(args) {
|
|
1097
|
+
return withRunControlMutationLock(args.runControlPath, () => persistReconstructLlmDispatchFailureUnlocked(args));
|
|
1098
|
+
}
|
|
1099
|
+
function blockedPartialWriteResumeRow(args) {
|
|
1100
|
+
const resumeId = idFor("resume", `${args.attemptId}:blocked_partial_write:${args.staleRefs.join(":")}`);
|
|
1101
|
+
return {
|
|
1102
|
+
resume_id: resumeId,
|
|
1103
|
+
resume_token_hash: sha256(resumeId),
|
|
1104
|
+
source_attempt_id: args.attemptId,
|
|
1105
|
+
checkpoint_refs: [args.runControlPath],
|
|
1106
|
+
trusted_artifact_refs: [],
|
|
1107
|
+
stale_artifact_refs: args.staleRefs,
|
|
1108
|
+
required_revalidation_refs: [args.runControlPath],
|
|
1109
|
+
resume_decision: "blocked_partial_write",
|
|
1110
|
+
};
|
|
1111
|
+
}
|
|
1112
|
+
function abandonCurrentAttemptForPartialWrite(args) {
|
|
1113
|
+
const current = args.runControl.attempt_rows.at(-1) ?? null;
|
|
1114
|
+
if (current?.attempt_status !== "running")
|
|
1115
|
+
return false;
|
|
1116
|
+
const abandonedAt = isoNow();
|
|
1117
|
+
args.runControl.attempt_rows = args.runControl.attempt_rows.map((row) => row.attempt_id === current.attempt_id && row.attempt_status === "running"
|
|
1118
|
+
? { ...row, completed_at: abandonedAt, attempt_status: "abandoned" }
|
|
1119
|
+
: row);
|
|
1120
|
+
args.runControl.lock_rows = args.runControl.lock_rows.map((row) => row.owner_attempt_id === current.attempt_id && row.lock_status === "held"
|
|
1121
|
+
? { ...row, lock_status: "released" }
|
|
1122
|
+
: row);
|
|
1123
|
+
if (!args.runControl.resume_rows.some((row) => row.source_attempt_id === current.attempt_id &&
|
|
1124
|
+
row.resume_decision === "blocked_partial_write")) {
|
|
1125
|
+
args.runControl.resume_rows.push(blockedPartialWriteResumeRow({
|
|
1126
|
+
runControlPath: args.runControlPath,
|
|
1127
|
+
attemptId: current.attempt_id,
|
|
1128
|
+
staleRefs: args.staleRefs,
|
|
1129
|
+
}));
|
|
1130
|
+
}
|
|
1131
|
+
return true;
|
|
1132
|
+
}
|
|
1133
|
+
async function pathExists(filePath) {
|
|
1134
|
+
if (!filePath)
|
|
1135
|
+
return false;
|
|
1136
|
+
try {
|
|
1137
|
+
await fs.access(filePath);
|
|
1138
|
+
return true;
|
|
1139
|
+
}
|
|
1140
|
+
catch (error) {
|
|
1141
|
+
if (error.code === "ENOENT")
|
|
1142
|
+
return false;
|
|
1143
|
+
throw error;
|
|
1144
|
+
}
|
|
1145
|
+
}
|
|
1146
|
+
async function removeFailureTempFile(sessionRoot, tempRef) {
|
|
1147
|
+
const directoryPath = await assertReconstructLlmDispatchFailureDirectory(sessionRoot);
|
|
1148
|
+
if (!isReconstructLlmDispatchFailureTempRef(sessionRoot, tempRef) ||
|
|
1149
|
+
path.dirname(path.resolve(tempRef)) !== directoryPath) {
|
|
1150
|
+
throw new Error(`cannot remove failure temp outside session: ${tempRef}`);
|
|
1151
|
+
}
|
|
1152
|
+
const stat = await fs.lstat(tempRef).catch((error) => {
|
|
1153
|
+
if (error.code === "ENOENT")
|
|
1154
|
+
return null;
|
|
1155
|
+
throw error;
|
|
1156
|
+
});
|
|
1157
|
+
if (!stat)
|
|
1158
|
+
return;
|
|
1159
|
+
if (stat.isSymbolicLink() || !stat.isFile()) {
|
|
1160
|
+
throw new Error(`cannot remove non-regular failure temp: ${tempRef}`);
|
|
1161
|
+
}
|
|
1162
|
+
await fs.rm(tempRef);
|
|
1163
|
+
}
|
|
1164
|
+
async function validatePreparedFailureArtifact(args) {
|
|
1165
|
+
const read = await readReconstructLlmDispatchFailureArtifactWithHash({
|
|
1166
|
+
sessionRoot: args.sessionRoot,
|
|
1167
|
+
artifactRef: args.artifactRef,
|
|
1168
|
+
allowTemp: isReconstructLlmDispatchFailureTempRef(args.sessionRoot, args.artifactRef),
|
|
1169
|
+
});
|
|
1170
|
+
const artifact = read.artifact;
|
|
1171
|
+
const hash = read.sha256;
|
|
1172
|
+
if (artifact.session_id !== args.sessionId ||
|
|
1173
|
+
artifact.owner_attempt_id !== args.ownerAttemptId ||
|
|
1174
|
+
!args.preparedContentHash ||
|
|
1175
|
+
hash !== args.preparedContentHash) {
|
|
1176
|
+
throw new Error("prepared failure transaction artifact does not match owner/hash");
|
|
1177
|
+
}
|
|
1178
|
+
if (!isReconstructLlmDispatchFailureTempRef(args.sessionRoot, args.artifactRef) &&
|
|
1179
|
+
path.resolve(args.artifactRef) !== path.resolve(reconstructLlmDispatchFailurePath(args.sessionRoot, artifact.failure_id))) {
|
|
1180
|
+
throw new Error("prepared failure transaction artifact path does not match failure id");
|
|
1181
|
+
}
|
|
1182
|
+
return { artifact, hash };
|
|
1183
|
+
}
|
|
1184
|
+
async function reconcileReconstructLlmDispatchFailuresUnlocked(args) {
|
|
1185
|
+
const sessionRoot = path.resolve(args.sessionRoot);
|
|
1186
|
+
const runControlPath = path.resolve(args.runControlPath ?? path.join(sessionRoot, "reconstruct-run-control.yaml"));
|
|
1187
|
+
const validationOutputPath = path.resolve(args.validationOutputPath ??
|
|
1188
|
+
path.join(sessionRoot, "reconstruct-run-control-validation.yaml"));
|
|
1189
|
+
let runControl;
|
|
1190
|
+
try {
|
|
1191
|
+
runControl = await readYamlDocument(runControlPath);
|
|
1192
|
+
}
|
|
1193
|
+
catch (error) {
|
|
1194
|
+
if (error.code === "ENOENT")
|
|
1195
|
+
return null;
|
|
1196
|
+
throw error;
|
|
1197
|
+
}
|
|
1198
|
+
if (path.resolve(runControl.session_root) !== sessionRoot ||
|
|
1199
|
+
runControl.session_id !== path.basename(sessionRoot) ||
|
|
1200
|
+
runControlPath !== path.join(sessionRoot, "reconstruct-run-control.yaml") ||
|
|
1201
|
+
validationOutputPath !==
|
|
1202
|
+
path.join(sessionRoot, "reconstruct-run-control-validation.yaml")) {
|
|
1203
|
+
throw new Error("reconstruct failure reconciliation session identity mismatch");
|
|
1204
|
+
}
|
|
1205
|
+
let changed = false;
|
|
1206
|
+
const failureDirectory = await assertReconstructLlmDispatchFailureDirectory(sessionRoot);
|
|
1207
|
+
let failureEntries = [];
|
|
1208
|
+
try {
|
|
1209
|
+
failureEntries = await fs.readdir(failureDirectory, { withFileTypes: true });
|
|
1210
|
+
}
|
|
1211
|
+
catch (error) {
|
|
1212
|
+
if (error.code !== "ENOENT")
|
|
1213
|
+
throw error;
|
|
1214
|
+
}
|
|
1215
|
+
const trackedTempRefs = new Set(runControl.write_transactions
|
|
1216
|
+
.map((row) => row.temp_ref)
|
|
1217
|
+
.filter((ref) => Boolean(ref))
|
|
1218
|
+
.map((ref) => path.resolve(ref)));
|
|
1219
|
+
const trackedFinalRefs = new Set(runControl.write_transactions
|
|
1220
|
+
.filter((row) => isReconstructLlmDispatchFailureRef(sessionRoot, row.artifact_ref))
|
|
1221
|
+
.map((row) => path.resolve(row.artifact_ref)));
|
|
1222
|
+
const untrackedPartialRefs = failureEntries
|
|
1223
|
+
.filter((entry) => {
|
|
1224
|
+
const entryRef = path.join(failureDirectory, entry.name);
|
|
1225
|
+
return entry.name.startsWith(".scratch-") ||
|
|
1226
|
+
(entry.name.startsWith(".pending-") && !trackedTempRefs.has(entryRef));
|
|
1227
|
+
})
|
|
1228
|
+
.map((entry) => path.join(failureDirectory, entry.name));
|
|
1229
|
+
const ambiguousPartialWrite = untrackedPartialRefs.length > 1;
|
|
1230
|
+
if (ambiguousPartialWrite) {
|
|
1231
|
+
changed = abandonCurrentAttemptForPartialWrite({
|
|
1232
|
+
runControl,
|
|
1233
|
+
runControlPath,
|
|
1234
|
+
staleRefs: untrackedPartialRefs,
|
|
1235
|
+
}) || changed;
|
|
1236
|
+
}
|
|
1237
|
+
for (const entry of failureEntries) {
|
|
1238
|
+
const entryRef = path.join(failureDirectory, entry.name);
|
|
1239
|
+
if (entry.name.startsWith(".scratch-")) {
|
|
1240
|
+
if (!ambiguousPartialWrite) {
|
|
1241
|
+
changed = abandonCurrentAttemptForPartialWrite({
|
|
1242
|
+
runControl,
|
|
1243
|
+
runControlPath,
|
|
1244
|
+
staleRefs: [entryRef],
|
|
1245
|
+
}) || changed;
|
|
1246
|
+
}
|
|
1247
|
+
continue;
|
|
1248
|
+
}
|
|
1249
|
+
if (entry.name.startsWith(".pending-") && !trackedTempRefs.has(entryRef)) {
|
|
1250
|
+
if (ambiguousPartialWrite)
|
|
1251
|
+
continue;
|
|
1252
|
+
try {
|
|
1253
|
+
const read = await readReconstructLlmDispatchFailureArtifactWithHash({
|
|
1254
|
+
sessionRoot,
|
|
1255
|
+
artifactRef: entryRef,
|
|
1256
|
+
allowTemp: true,
|
|
1257
|
+
});
|
|
1258
|
+
const latestAttempt = runControl.attempt_rows.at(-1) ?? null;
|
|
1259
|
+
const ownerLocks = runControl.lock_rows.filter((row) => row.lock_scope === "session_root" &&
|
|
1260
|
+
row.owner_attempt_id === read.artifact.owner_attempt_id);
|
|
1261
|
+
if (entry.name.startsWith(`.pending-${read.sha256.slice(0, 16)}-`) &&
|
|
1262
|
+
read.artifact.session_id === runControl.session_id &&
|
|
1263
|
+
latestAttempt?.attempt_id === read.artifact.owner_attempt_id &&
|
|
1264
|
+
latestAttempt.attempt_status === "running" &&
|
|
1265
|
+
ownerLocks.length === 1 &&
|
|
1266
|
+
ownerLocks[0]?.lock_status === "held") {
|
|
1267
|
+
const finalRef = reconstructLlmDispatchFailurePath(sessionRoot, read.artifact.failure_id);
|
|
1268
|
+
const transactionId = idFor("write", `${read.artifact.owner_attempt_id}:${finalRef}`);
|
|
1269
|
+
if (!runControl.write_transactions.some((row) => row.transaction_id === transactionId)) {
|
|
1270
|
+
runControl.write_transactions.push({
|
|
1271
|
+
transaction_id: transactionId,
|
|
1272
|
+
owner_attempt_id: read.artifact.owner_attempt_id,
|
|
1273
|
+
artifact_ref: finalRef,
|
|
1274
|
+
temp_ref: entryRef,
|
|
1275
|
+
expected_prior_hash: null,
|
|
1276
|
+
prepared_content_hash: read.sha256,
|
|
1277
|
+
committed_hash: null,
|
|
1278
|
+
commit_method: "append_only",
|
|
1279
|
+
transaction_status: "prepared",
|
|
1280
|
+
recovery_ref: null,
|
|
1281
|
+
});
|
|
1282
|
+
trackedFinalRefs.add(path.resolve(finalRef));
|
|
1283
|
+
changed = true;
|
|
1284
|
+
runControl.updated_at = isoNow();
|
|
1285
|
+
await writeRunControlDocument(runControlPath, runControl);
|
|
1286
|
+
}
|
|
1287
|
+
}
|
|
1288
|
+
else {
|
|
1289
|
+
changed = abandonCurrentAttemptForPartialWrite({
|
|
1290
|
+
runControl,
|
|
1291
|
+
runControlPath,
|
|
1292
|
+
staleRefs: [entryRef],
|
|
1293
|
+
}) || changed;
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
catch {
|
|
1297
|
+
changed = abandonCurrentAttemptForPartialWrite({
|
|
1298
|
+
runControl,
|
|
1299
|
+
runControlPath,
|
|
1300
|
+
staleRefs: [entryRef],
|
|
1301
|
+
}) || changed;
|
|
1302
|
+
}
|
|
1303
|
+
continue;
|
|
1304
|
+
}
|
|
1305
|
+
if (!entry.name.startsWith("failure-") ||
|
|
1306
|
+
path.extname(entry.name) !== ".yaml" ||
|
|
1307
|
+
trackedFinalRefs.has(entryRef)) {
|
|
1308
|
+
continue;
|
|
1309
|
+
}
|
|
1310
|
+
let artifact = null;
|
|
1311
|
+
let artifactHash = null;
|
|
1312
|
+
try {
|
|
1313
|
+
const stat = await fs.lstat(entryRef);
|
|
1314
|
+
if (stat.isFile() && !stat.isSymbolicLink()) {
|
|
1315
|
+
artifactHash = await sha256ReconstructLlmDispatchFailureArtifact(entryRef);
|
|
1316
|
+
}
|
|
1317
|
+
}
|
|
1318
|
+
catch {
|
|
1319
|
+
artifactHash = null;
|
|
1320
|
+
}
|
|
1321
|
+
try {
|
|
1322
|
+
artifact = await readReconstructLlmDispatchFailureArtifact(entryRef);
|
|
1323
|
+
}
|
|
1324
|
+
catch {
|
|
1325
|
+
artifact = null;
|
|
1326
|
+
}
|
|
1327
|
+
const ownerAttemptId = artifact &&
|
|
1328
|
+
artifact.session_id === runControl.session_id &&
|
|
1329
|
+
runControl.attempt_rows.some((row) => row.attempt_id === artifact.owner_attempt_id)
|
|
1330
|
+
? artifact.owner_attempt_id
|
|
1331
|
+
: [...runControl.attempt_rows].reverse().find((row) => row.attempt_status === "running")?.attempt_id ?? null;
|
|
1332
|
+
if (!ownerAttemptId || !artifactHash)
|
|
1333
|
+
continue;
|
|
1334
|
+
const transactionId = idFor("write", `${ownerAttemptId}:${entryRef}:orphan`);
|
|
1335
|
+
if (!runControl.write_transactions.some((row) => row.transaction_id === transactionId)) {
|
|
1336
|
+
runControl.write_transactions.push({
|
|
1337
|
+
transaction_id: transactionId,
|
|
1338
|
+
owner_attempt_id: ownerAttemptId,
|
|
1339
|
+
artifact_ref: entryRef,
|
|
1340
|
+
temp_ref: null,
|
|
1341
|
+
expected_prior_hash: null,
|
|
1342
|
+
prepared_content_hash: artifactHash,
|
|
1343
|
+
committed_hash: null,
|
|
1344
|
+
commit_method: "append_only",
|
|
1345
|
+
transaction_status: "quarantined",
|
|
1346
|
+
recovery_ref: "blocked_partial_write",
|
|
1347
|
+
});
|
|
1348
|
+
}
|
|
1349
|
+
const abandonedAt = isoNow();
|
|
1350
|
+
runControl.attempt_rows = runControl.attempt_rows.map((row) => row.attempt_id === ownerAttemptId && row.attempt_status === "running"
|
|
1351
|
+
? { ...row, completed_at: abandonedAt, attempt_status: "abandoned" }
|
|
1352
|
+
: row);
|
|
1353
|
+
runControl.lock_rows = runControl.lock_rows.map((row) => row.owner_attempt_id === ownerAttemptId && row.lock_status === "held"
|
|
1354
|
+
? { ...row, lock_status: "released" }
|
|
1355
|
+
: row);
|
|
1356
|
+
if (!runControl.resume_rows.some((row) => row.source_attempt_id === ownerAttemptId &&
|
|
1357
|
+
row.resume_decision === "blocked_partial_write")) {
|
|
1358
|
+
runControl.resume_rows.push(blockedPartialWriteResumeRow({
|
|
1359
|
+
runControlPath,
|
|
1360
|
+
attemptId: ownerAttemptId,
|
|
1361
|
+
staleRefs: [entryRef],
|
|
1362
|
+
}));
|
|
1363
|
+
}
|
|
1364
|
+
changed = true;
|
|
1365
|
+
}
|
|
1366
|
+
for (const transaction of runControl.write_transactions) {
|
|
1367
|
+
if (transaction.transaction_status !== "prepared" ||
|
|
1368
|
+
!isReconstructLlmDispatchFailureRef(sessionRoot, transaction.artifact_ref)) {
|
|
1369
|
+
continue;
|
|
1370
|
+
}
|
|
1371
|
+
try {
|
|
1372
|
+
const latestAttempt = runControl.attempt_rows.at(-1) ?? null;
|
|
1373
|
+
const ownerAttempt = runControl.attempt_rows.find((row) => row.attempt_id === transaction.owner_attempt_id);
|
|
1374
|
+
const ownerLocks = runControl.lock_rows.filter((row) => row.lock_scope === "session_root" &&
|
|
1375
|
+
row.owner_attempt_id === transaction.owner_attempt_id);
|
|
1376
|
+
if (latestAttempt?.attempt_id !== transaction.owner_attempt_id ||
|
|
1377
|
+
ownerAttempt?.attempt_status !== "running" ||
|
|
1378
|
+
ownerLocks.length !== 1 ||
|
|
1379
|
+
ownerLocks[0]?.lock_status !== "held") {
|
|
1380
|
+
throw new Error("prepared failure transaction is not owned by the latest running attempt and its unique held lock");
|
|
1381
|
+
}
|
|
1382
|
+
if (transaction.temp_ref &&
|
|
1383
|
+
!isReconstructLlmDispatchFailureTempRef(sessionRoot, transaction.temp_ref)) {
|
|
1384
|
+
throw new Error("prepared failure transaction temp ref escapes failure directory");
|
|
1385
|
+
}
|
|
1386
|
+
const finalExists = await pathExists(transaction.artifact_ref);
|
|
1387
|
+
const tempExists = await pathExists(transaction.temp_ref);
|
|
1388
|
+
if (!finalExists) {
|
|
1389
|
+
if (!tempExists) {
|
|
1390
|
+
if (Date.parse(ownerLocks[0].lease_expires_at) > Date.now()) {
|
|
1391
|
+
continue;
|
|
1392
|
+
}
|
|
1393
|
+
throw new Error("prepared failure transaction has neither temp nor final artifact");
|
|
1394
|
+
}
|
|
1395
|
+
await validatePreparedFailureArtifact({
|
|
1396
|
+
sessionRoot,
|
|
1397
|
+
artifactRef: transaction.temp_ref,
|
|
1398
|
+
sessionId: runControl.session_id,
|
|
1399
|
+
ownerAttemptId: transaction.owner_attempt_id,
|
|
1400
|
+
preparedContentHash: transaction.prepared_content_hash,
|
|
1401
|
+
});
|
|
1402
|
+
await publishReconstructLlmDispatchFailureTemp({
|
|
1403
|
+
sessionRoot,
|
|
1404
|
+
tempRef: transaction.temp_ref,
|
|
1405
|
+
finalRef: transaction.artifact_ref,
|
|
1406
|
+
});
|
|
1407
|
+
}
|
|
1408
|
+
const validated = await validatePreparedFailureArtifact({
|
|
1409
|
+
sessionRoot,
|
|
1410
|
+
artifactRef: transaction.artifact_ref,
|
|
1411
|
+
sessionId: runControl.session_id,
|
|
1412
|
+
ownerAttemptId: transaction.owner_attempt_id,
|
|
1413
|
+
preparedContentHash: transaction.prepared_content_hash,
|
|
1414
|
+
});
|
|
1415
|
+
if (await pathExists(transaction.temp_ref)) {
|
|
1416
|
+
await removeFailureTempFile(sessionRoot, transaction.temp_ref);
|
|
1417
|
+
}
|
|
1418
|
+
transaction.temp_ref = null;
|
|
1419
|
+
transaction.committed_hash = validated.hash;
|
|
1420
|
+
transaction.transaction_status = "committed";
|
|
1421
|
+
const completedAt = isoNow();
|
|
1422
|
+
runControl.attempt_rows = runControl.attempt_rows.map((row) => row.attempt_id === transaction.owner_attempt_id &&
|
|
1423
|
+
row.attempt_status === "running"
|
|
1424
|
+
? { ...row, completed_at: completedAt, attempt_status: "failed" }
|
|
1425
|
+
: row);
|
|
1426
|
+
runControl.lock_rows = runControl.lock_rows.map((row) => row.owner_attempt_id === transaction.owner_attempt_id &&
|
|
1427
|
+
row.lock_status === "held"
|
|
1428
|
+
? { ...row, lock_status: "released" }
|
|
1429
|
+
: row);
|
|
1430
|
+
changed = true;
|
|
1431
|
+
}
|
|
1432
|
+
catch {
|
|
1433
|
+
const staleRefs = [transaction.artifact_ref, transaction.temp_ref]
|
|
1434
|
+
.filter((ref) => Boolean(ref));
|
|
1435
|
+
transaction.transaction_status = "quarantined";
|
|
1436
|
+
transaction.recovery_ref = "blocked_partial_write";
|
|
1437
|
+
const abandonedAt = isoNow();
|
|
1438
|
+
runControl.attempt_rows = runControl.attempt_rows.map((row) => row.attempt_id === transaction.owner_attempt_id &&
|
|
1439
|
+
row.attempt_status === "running"
|
|
1440
|
+
? { ...row, completed_at: abandonedAt, attempt_status: "abandoned" }
|
|
1441
|
+
: row);
|
|
1442
|
+
runControl.lock_rows = runControl.lock_rows.map((row) => row.owner_attempt_id === transaction.owner_attempt_id &&
|
|
1443
|
+
row.lock_status === "held"
|
|
1444
|
+
? { ...row, lock_status: "released" }
|
|
1445
|
+
: row);
|
|
1446
|
+
if (!runControl.resume_rows.some((row) => row.source_attempt_id === transaction.owner_attempt_id &&
|
|
1447
|
+
row.resume_decision === "blocked_partial_write")) {
|
|
1448
|
+
runControl.resume_rows.push(blockedPartialWriteResumeRow({
|
|
1449
|
+
runControlPath,
|
|
1450
|
+
attemptId: transaction.owner_attempt_id,
|
|
1451
|
+
staleRefs,
|
|
1452
|
+
}));
|
|
1453
|
+
}
|
|
1454
|
+
changed = true;
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
if (changed) {
|
|
1458
|
+
runControl.updated_at = isoNow();
|
|
1459
|
+
await writeRunControlDocument(runControlPath, runControl);
|
|
1460
|
+
}
|
|
1461
|
+
const latestAttempt = runControl.attempt_rows.at(-1) ?? null;
|
|
1462
|
+
const validation = changed || latestAttempt?.attempt_status === "failed"
|
|
1463
|
+
? await writeReconstructRunControlValidationArtifact({
|
|
1464
|
+
runControlPath,
|
|
1465
|
+
outputPath: validationOutputPath,
|
|
1466
|
+
expectedSessionId: runControl.session_id,
|
|
1467
|
+
expectedSessionRoot: sessionRoot,
|
|
1468
|
+
})
|
|
1469
|
+
: null;
|
|
1470
|
+
return { runControl, validation };
|
|
1471
|
+
}
|
|
1472
|
+
export async function reconcileReconstructLlmDispatchFailures(args) {
|
|
1473
|
+
const sessionRoot = path.resolve(args.sessionRoot);
|
|
1474
|
+
const runControlPath = path.resolve(args.runControlPath ?? path.join(sessionRoot, "reconstruct-run-control.yaml"));
|
|
1475
|
+
const validationOutputPath = path.resolve(args.validationOutputPath ??
|
|
1476
|
+
path.join(sessionRoot, "reconstruct-run-control-validation.yaml"));
|
|
1477
|
+
let snapshot;
|
|
1478
|
+
try {
|
|
1479
|
+
snapshot = await readYamlDocument(runControlPath);
|
|
1480
|
+
}
|
|
1481
|
+
catch (error) {
|
|
1482
|
+
if (error.code === "ENOENT")
|
|
1483
|
+
return null;
|
|
1484
|
+
throw error;
|
|
1485
|
+
}
|
|
1486
|
+
if (path.resolve(snapshot.session_root) !== sessionRoot ||
|
|
1487
|
+
snapshot.session_id !== path.basename(sessionRoot) ||
|
|
1488
|
+
runControlPath !== path.join(sessionRoot, "reconstruct-run-control.yaml") ||
|
|
1489
|
+
validationOutputPath !==
|
|
1490
|
+
path.join(sessionRoot, "reconstruct-run-control-validation.yaml")) {
|
|
1491
|
+
throw new Error("reconstruct failure reconciliation session identity mismatch");
|
|
1492
|
+
}
|
|
1493
|
+
const failureDirectory = path.join(sessionRoot, RECONSTRUCT_LLM_DISPATCH_FAILURE_DIR);
|
|
1494
|
+
const failureDirectoryStat = await fs.lstat(failureDirectory)
|
|
1495
|
+
.catch((error) => {
|
|
1496
|
+
if (error.code === "ENOENT")
|
|
1497
|
+
return null;
|
|
1498
|
+
throw error;
|
|
1499
|
+
});
|
|
1500
|
+
if (failureDirectoryStat &&
|
|
1501
|
+
(!failureDirectoryStat.isDirectory() || failureDirectoryStat.isSymbolicLink())) {
|
|
1502
|
+
throw new Error(`LLM dispatch failure directory is not a real directory: ${failureDirectory}`);
|
|
1503
|
+
}
|
|
1504
|
+
if (failureDirectoryStat) {
|
|
1505
|
+
const [realSessionRoot, realFailureDirectory] = await Promise.all([
|
|
1506
|
+
fs.realpath(sessionRoot),
|
|
1507
|
+
fs.realpath(failureDirectory),
|
|
1508
|
+
]);
|
|
1509
|
+
if (path.dirname(realFailureDirectory) !== realSessionRoot) {
|
|
1510
|
+
throw new Error(`LLM dispatch failure directory escapes session root: ${realFailureDirectory}`);
|
|
1511
|
+
}
|
|
1512
|
+
}
|
|
1513
|
+
const failureEntries = failureDirectoryStat
|
|
1514
|
+
? await fs.readdir(failureDirectory, { withFileTypes: true })
|
|
1515
|
+
: [];
|
|
1516
|
+
const trackedFinalRefs = new Set(snapshot.write_transactions
|
|
1517
|
+
.filter((row) => isReconstructLlmDispatchFailureRef(sessionRoot, row.artifact_ref))
|
|
1518
|
+
.map((row) => path.resolve(row.artifact_ref)));
|
|
1519
|
+
const needsMutation = snapshot.write_transactions.some((row) => row.transaction_status === "prepared" &&
|
|
1520
|
+
isReconstructLlmDispatchFailureRef(sessionRoot, row.artifact_ref)) || failureEntries.some((entry) => {
|
|
1521
|
+
const entryRef = path.join(failureDirectory, entry.name);
|
|
1522
|
+
return entry.name.startsWith(".scratch-") ||
|
|
1523
|
+
entry.name.startsWith(".pending-") ||
|
|
1524
|
+
(entry.name.startsWith("failure-") &&
|
|
1525
|
+
path.extname(entry.name) === ".yaml" &&
|
|
1526
|
+
!trackedFinalRefs.has(path.resolve(entryRef)));
|
|
1527
|
+
});
|
|
1528
|
+
const latestAttempt = snapshot.attempt_rows.at(-1) ?? null;
|
|
1529
|
+
let persistedValidation = null;
|
|
1530
|
+
if (latestAttempt?.attempt_status === "failed") {
|
|
1531
|
+
try {
|
|
1532
|
+
const raw = await readYamlDocumentIfPresent(validationOutputPath);
|
|
1533
|
+
if (raw !== null &&
|
|
1534
|
+
comparableRunControlValidation(raw, { stripInMemoryFields: false }) !== null) {
|
|
1535
|
+
persistedValidation = raw;
|
|
1536
|
+
}
|
|
1537
|
+
}
|
|
1538
|
+
catch {
|
|
1539
|
+
persistedValidation = null;
|
|
1540
|
+
}
|
|
1541
|
+
}
|
|
1542
|
+
const expectedValidation = latestAttempt?.attempt_status === "failed"
|
|
1543
|
+
? await buildReconstructRunControlValidationArtifactFromRunControl({
|
|
1544
|
+
runControl: snapshot,
|
|
1545
|
+
runControlPath,
|
|
1546
|
+
expectedSessionId: snapshot.session_id,
|
|
1547
|
+
expectedSessionRoot: sessionRoot,
|
|
1548
|
+
})
|
|
1549
|
+
: null;
|
|
1550
|
+
const persistedComparable = comparableRunControlValidation(persistedValidation, { stripInMemoryFields: false });
|
|
1551
|
+
const expectedComparable = comparableRunControlValidation(expectedValidation, { stripInMemoryFields: true });
|
|
1552
|
+
const validationIsCurrent = persistedComparable !== null &&
|
|
1553
|
+
expectedComparable !== null &&
|
|
1554
|
+
persistedComparable === expectedComparable;
|
|
1555
|
+
if (!needsMutation && latestAttempt?.attempt_status !== "failed") {
|
|
1556
|
+
return { runControl: snapshot, validation: null };
|
|
1557
|
+
}
|
|
1558
|
+
if (!needsMutation && validationIsCurrent) {
|
|
1559
|
+
return { runControl: snapshot, validation: persistedValidation };
|
|
1560
|
+
}
|
|
1561
|
+
return withRunControlMutationLock(runControlPath, () => reconcileReconstructLlmDispatchFailuresUnlocked(args));
|
|
1562
|
+
}
|
|
621
1563
|
function artifactRefsForTransactions(artifactRefs, extraRefs) {
|
|
622
1564
|
return [...new Set([
|
|
623
1565
|
...Object.values(artifactRefs),
|
|
@@ -648,7 +1590,7 @@ async function appendWriteTransactions(args) {
|
|
|
648
1590
|
existingTransactionIds.add(transactionId);
|
|
649
1591
|
}
|
|
650
1592
|
}
|
|
651
|
-
|
|
1593
|
+
async function recordReconstructRunControlTransactionsUnlocked(args) {
|
|
652
1594
|
const runControl = await readYamlDocument(args.runControlPath);
|
|
653
1595
|
const attempt = runControl.attempt_rows.find((row) => row.attempt_id === args.attemptId);
|
|
654
1596
|
if (!attempt || attempt.attempt_status !== "running") {
|
|
@@ -666,7 +1608,7 @@ export async function recordReconstructRunControlTransactions(args) {
|
|
|
666
1608
|
attemptId: args.attemptId,
|
|
667
1609
|
refs: [...new Set(args.artifactRefs)].sort(),
|
|
668
1610
|
});
|
|
669
|
-
await
|
|
1611
|
+
await writeRunControlDocument(args.runControlPath, runControl);
|
|
670
1612
|
const validation = await writeReconstructRunControlValidationArtifact({
|
|
671
1613
|
runControlPath: args.runControlPath,
|
|
672
1614
|
outputPath: args.validationOutputPath,
|
|
@@ -678,20 +1620,57 @@ export async function recordReconstructRunControlTransactions(args) {
|
|
|
678
1620
|
});
|
|
679
1621
|
return { runControl, validation };
|
|
680
1622
|
}
|
|
681
|
-
export async function
|
|
682
|
-
|
|
1623
|
+
export async function recordReconstructRunControlTransactions(args) {
|
|
1624
|
+
return withRunControlMutationLock(args.runControlPath, () => recordReconstructRunControlTransactionsUnlocked(args));
|
|
1625
|
+
}
|
|
1626
|
+
async function finalizeReconstructRunControlUnlocked(args) {
|
|
1627
|
+
const attemptStatus = args.attemptStatus ?? "completed";
|
|
1628
|
+
const terminalValidationStatus = await readValidationStatusIfPresent(args.terminalRunManifestValidationPath);
|
|
683
1629
|
if (terminalValidationStatus !== "valid") {
|
|
684
1630
|
throw new Error([
|
|
685
|
-
"reconstruct run-control cannot finalize without valid
|
|
686
|
-
`terminal_validation_ref=${args.
|
|
1631
|
+
"reconstruct run-control cannot finalize without valid terminal run-manifest validation.",
|
|
1632
|
+
`terminal_validation_ref=${args.terminalRunManifestValidationPath}`,
|
|
1633
|
+
`attempt_status=${attemptStatus}`,
|
|
687
1634
|
`validation_status=${terminalValidationStatus ?? "missing"}`,
|
|
688
1635
|
].join(" "));
|
|
689
1636
|
}
|
|
690
1637
|
const runControl = await readYamlDocument(args.runControlPath);
|
|
1638
|
+
if (runControl.resume_rows.some((row) => row.source_attempt_id === args.attemptId &&
|
|
1639
|
+
row.resume_decision === "blocked_partial_write")) {
|
|
1640
|
+
throw new Error("reconstruct run-control cannot finalize a blocked_partial_write attempt");
|
|
1641
|
+
}
|
|
1642
|
+
const activeAttempt = runControl.attempt_rows.find((row) => row.attempt_id === args.attemptId);
|
|
1643
|
+
const activeOwnerLocks = runControl.lock_rows.filter((row) => row.lock_scope === "session_root" &&
|
|
1644
|
+
row.owner_attempt_id === args.attemptId &&
|
|
1645
|
+
row.lock_status === "held");
|
|
1646
|
+
if (activeAttempt?.attempt_status !== "running" || activeOwnerLocks.length !== 1) {
|
|
1647
|
+
throw new Error(`reconstruct run-control finalize requires the exact running owner: ${args.attemptId}`);
|
|
1648
|
+
}
|
|
1649
|
+
if (activeAttempt.attempt_kind === "resume" && activeAttempt.trigger_ref) {
|
|
1650
|
+
const resume = runControl.resume_rows.find((row) => row.resume_id === activeAttempt.trigger_ref);
|
|
1651
|
+
if (!resume || resume.resume_decision !== "resume_pending_provenance") {
|
|
1652
|
+
throw new Error("reconstruct resume is not pending trusted provenance");
|
|
1653
|
+
}
|
|
1654
|
+
for (const failureRef of resume.required_revalidation_refs.filter((ref) => isReconstructLlmDispatchFailureRef(runControl.session_root, ref))) {
|
|
1655
|
+
const sourceTransaction = runControl.write_transactions.find((row) => row.owner_attempt_id === resume.source_attempt_id &&
|
|
1656
|
+
path.resolve(row.artifact_ref) === path.resolve(failureRef) &&
|
|
1657
|
+
row.transaction_status === "committed" &&
|
|
1658
|
+
row.committed_hash !== null);
|
|
1659
|
+
const failureRead = sourceTransaction
|
|
1660
|
+
? await readReconstructLlmDispatchFailureArtifactWithHash({
|
|
1661
|
+
sessionRoot: runControl.session_root,
|
|
1662
|
+
artifactRef: failureRef,
|
|
1663
|
+
})
|
|
1664
|
+
: null;
|
|
1665
|
+
if (!sourceTransaction || failureRead?.sha256 !== sourceTransaction.committed_hash) {
|
|
1666
|
+
throw new Error(`reconstruct resume failure provenance changed before finalize: ${failureRef}`);
|
|
1667
|
+
}
|
|
1668
|
+
}
|
|
1669
|
+
}
|
|
691
1670
|
const completedAt = isoNow();
|
|
692
1671
|
runControl.updated_at = completedAt;
|
|
693
1672
|
runControl.attempt_rows = runControl.attempt_rows.map((row) => row.attempt_id === args.attemptId
|
|
694
|
-
? { ...row, completed_at: completedAt, attempt_status:
|
|
1673
|
+
? { ...row, completed_at: completedAt, attempt_status: attemptStatus }
|
|
695
1674
|
: row);
|
|
696
1675
|
const completedAttempt = runControl.attempt_rows.find((row) => row.attempt_id === args.attemptId);
|
|
697
1676
|
if (completedAttempt?.attempt_kind === "resume" && completedAttempt.trigger_ref) {
|
|
@@ -705,20 +1684,23 @@ export async function finalizeReconstructRunControl(args) {
|
|
|
705
1684
|
: row);
|
|
706
1685
|
const refs = artifactRefsForTransactions(args.artifactRefs, [
|
|
707
1686
|
...(args.extraArtifactRefs ?? []),
|
|
708
|
-
args.
|
|
1687
|
+
args.terminalRunManifestValidationPath,
|
|
709
1688
|
]);
|
|
710
1689
|
await appendWriteTransactions({
|
|
711
1690
|
runControl,
|
|
712
1691
|
attemptId: args.attemptId,
|
|
713
1692
|
refs,
|
|
714
1693
|
});
|
|
715
|
-
await
|
|
1694
|
+
await writeRunControlDocument(args.runControlPath, runControl);
|
|
716
1695
|
const validation = await writeReconstructRunControlValidationArtifact({
|
|
717
1696
|
runControlPath: args.runControlPath,
|
|
718
1697
|
outputPath: args.validationOutputPath,
|
|
719
1698
|
expectedSessionId: args.expectedSessionId,
|
|
720
1699
|
expectedSessionRoot: args.expectedSessionRoot,
|
|
721
|
-
terminalValidationRef: args.
|
|
1700
|
+
terminalValidationRef: args.terminalRunManifestValidationPath,
|
|
722
1701
|
});
|
|
723
1702
|
return { runControl, validation };
|
|
724
1703
|
}
|
|
1704
|
+
export async function finalizeReconstructRunControl(args) {
|
|
1705
|
+
return withRunControlMutationLock(args.runControlPath, () => finalizeReconstructRunControlUnlocked(args));
|
|
1706
|
+
}
|