feature-factory 0.10.1 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/WORKFLOW.md +3 -2
- package/bin/factory.js +60 -12
- package/bin/restore.js +1 -0
- package/bin/snapshot.js +107 -64
- package/core/atomic-write.js +11 -7
- package/core/run-lock.js +14 -4
- package/core/write-core.js +17 -4
- package/observe/repair-reverification.js +3 -3
- package/package.json +1 -1
- package/state/retry-grant-transaction.js +194 -0
- package/state/session-lock.js +3 -3
- package/state/transition.js +3 -1
package/README.md
CHANGED
|
@@ -265,8 +265,9 @@ waves too, but refuses when another slice is blocked or an exhausted post-merge
|
|
|
265
265
|
scopes reopen only the named slice after exact owner, snapshot, REJECT, evidence, base, current clean head,
|
|
266
266
|
and immutable attempt archives. A legacy run missing one prepares it without granting, then requires a
|
|
267
267
|
fresh snapshot and a second grant invocation. Exhaustion parks with reason `blocked-after-retries` rather
|
|
268
|
-
than terminalizing `partial`. The grant
|
|
269
|
-
|
|
268
|
+
than terminalizing `partial`. The grant durably fences the prior canonical snapshot before committing its
|
|
269
|
+
manifest, then removes it only after the commit is proved; restore and parked mutations refuse any
|
|
270
|
+
interrupted fence. Run `factory snapshot <run-id> --repo <operator>` and requalify it before the separate
|
|
270
271
|
explicit resume. Restored blocked slices whose physical refs were intentionally cleared do not qualify.
|
|
271
272
|
|
|
272
273
|
`resolve` and `verify` are consumed now, and the run's recorded `publishing_identity` is compared at the publication guards. Step 6 resolves one selection: a nonblank inherited `FACTORY_PUBLISHING_COMMAND` selects its exact string; that variable set blank or whitespace selects the default; when it is unset, configured `publish` wins if present; otherwise the default wins. Only a selected nondefault command replaces `gh pr create`, after the factory-owned exact push and post-push identity guard. It receives exact `PR_BASE`, `FEATURE_BRANCH`, `PR_DRAFT`, `PR_TITLE`, and absolute `PR_BODY_FILE` environment values. Only exit zero with an absolute HTTPS URL on the last nonempty stdout line is recordable; every other result parks with exact reason `selected publishing command outcome indeterminate; re-observe whether the pull request exists before retry` and no fallback.
|
package/WORKFLOW.md
CHANGED
|
@@ -236,8 +236,9 @@ an archive gets a preparation-only refusal: publish the changed plane and invoke
|
|
|
236
236
|
append the durable authorization, preserve worktree, branch and `base_ref`, clear only the live attempt-bound
|
|
237
237
|
refs, and record `running@(N+1)` while top-level status and `terminal_result` remain parked.
|
|
238
238
|
|
|
239
|
-
The grant
|
|
240
|
-
|
|
239
|
+
The grant durably fences the old canonical snapshot before committing `run.json`, then removes it only
|
|
240
|
+
when the manifest commit is proved. Restore and parked mutations refuse an interrupted fence; `factory
|
|
241
|
+
snapshot` reconciles exact pre-commit or post-commit state under the run lock. Do not dispatch or resume yet. Republish the updated live plane, then require qualified status
|
|
241
242
|
to report the refreshed `park_snapshot`, unchanged owner, the chosen new effective limit, and only the named
|
|
242
243
|
slice at `running@(N+1)`. Only then run the ordinary explicit
|
|
243
244
|
`factory resume "$R" --session "$SESSION_ID" --repo "$RUN_REPO"` and dispatch that attempt. Resume refreshes
|
package/bin/factory.js
CHANGED
|
@@ -12,13 +12,14 @@ import { isDeepStrictEqual } from "node:util";
|
|
|
12
12
|
import { readFileSync } from "node:fs";
|
|
13
13
|
import { nextAction, nextActionRecord, readRun, readRunUnchecked } from "../state/index.js";
|
|
14
14
|
import { transition } from "../state/transition.js";
|
|
15
|
-
import { RUN_JSON_LOCK_DIR } from "../core/run-lock.js";
|
|
15
|
+
import { RUN_JSON_LOCK_DIR, withRunJsonLock } from "../core/run-lock.js";
|
|
16
16
|
import { buildEvidence, deriveReviewReady, EVIDENCE_KEYS, evidenceRef, git, observeAncestry, observeCleanliness, observeTrackedCleanliness, observeWorktree, privilegedPaths, proveInitContainment, resolveWorktree, runBootstrap, unownedPaths } from "../observe/index.js";
|
|
17
17
|
import { assertPublicationReady, assertReviewBinding, isApproving, observeMergeProof, readEvidence, readReview, readValidatorReview } from "../observe/review.js";
|
|
18
18
|
import { readRepositoryConfig, RepositoryConfigError } from "../observe/repository-config.js";
|
|
19
19
|
import { reverifyRepair } from "../observe/repair-reverification.js";
|
|
20
20
|
import { readRepairState } from "../observe/repair-record.js";
|
|
21
21
|
import { archiveReviewAttempt, publishAttemptArchive, qualifyAttemptArchive } from "../state/review-archive.js";
|
|
22
|
+
import { hasRetryGrantTransaction, prepareRetryGrantTransaction, reconcileRetryGrantTransaction } from "../state/retry-grant-transaction.js";
|
|
22
23
|
import { writeProtectedFileAtomic, writeProtectedJsonAtomic } from "../core/atomic-write.js";
|
|
23
24
|
import { enforceEffectivePushTarget } from "../core/effective-push.js";
|
|
24
25
|
import { resolveSpawnExecutable } from "../core/executable.js";
|
|
@@ -81,6 +82,24 @@ export async function run(argv) {
|
|
|
81
82
|
if (!Object.hasOwn(COMMANDS, command)) throw new CliError(`unknown command '${command}' (try --help)`);
|
|
82
83
|
const { positional, flags } = parse(command, rest);
|
|
83
84
|
const handler = HANDLERS[command];
|
|
85
|
+
if (["init", "status", "snapshot", "lock", "heartbeat", "effective-push"].includes(command)) return handler(positional, flags);
|
|
86
|
+
const repo = resolve(flags.repo ?? process.cwd()), runId = positional[0];
|
|
87
|
+
if (command === "restore") {
|
|
88
|
+
const live = [join(repo, CONTROL_PLANE, runId), join(repo, ".factory-sandboxes", runId, CONTROL_PLANE, runId)]
|
|
89
|
+
.find((candidate) => existsSync(join(candidate, "run.json")));
|
|
90
|
+
if (!live) { assertNoRetryGrantTransaction(repo, runId, command); return handler(positional, flags); }
|
|
91
|
+
return withRunJsonLock(live, async () => { assertNoRetryGrantTransaction(repo, runId, command); return handler(positional, flags); },
|
|
92
|
+
{ nonExpiring: true });
|
|
93
|
+
}
|
|
94
|
+
if (["resume", "observe", "reverify-repair"].includes(command)) { assertNoRetryGrantTransaction(repo, runId, command); return handler(positional, flags); }
|
|
95
|
+
if (typeof runId === "string" && /^[a-z0-9](?:[a-z0-9._-]*[a-z0-9])?$/u.test(runId)
|
|
96
|
+
&& existsSync(join(repo, CONTROL_PLANE, runId, "run.json"))) {
|
|
97
|
+
return withRunJsonLock(join(repo, CONTROL_PLANE, runId), async () => {
|
|
98
|
+
assertNoRetryGrantTransaction(repo, runId, command);
|
|
99
|
+
return handler(positional, flags);
|
|
100
|
+
}, { allowReentrant: true, nonExpiring: true });
|
|
101
|
+
}
|
|
102
|
+
assertNoRetryGrantTransaction(repo, runId, command);
|
|
84
103
|
return handler(positional, flags);
|
|
85
104
|
}
|
|
86
105
|
|
|
@@ -213,6 +232,7 @@ function observedParkSnapshot(repo, runId, runDir, liveSkipped = new Set()) {
|
|
|
213
232
|
const container = dirname(repo), sandbox = basename(container) === ".factory-sandboxes" && basename(repo) === runId;
|
|
214
233
|
const operatorRoot = sandbox ? dirname(container) : repo, candidate = join(operatorRoot, CONTROL_PLANE, ".parked", runId);
|
|
215
234
|
try {
|
|
235
|
+
if (hasRetryGrantTransaction(candidate, runId)) return null;
|
|
216
236
|
const live = [join(operatorRoot, CONTROL_PLANE, runId), join(operatorRoot, ".factory-sandboxes", runId, CONTROL_PLANE, runId)]
|
|
217
237
|
.filter((plane) => existsSync(join(plane, "run.json")));
|
|
218
238
|
if (live.length !== 1 || realpathSync(live[0]) !== realpathSync(runDir)) return null;
|
|
@@ -241,12 +261,23 @@ function qualifyRetryGrant(repo, runDir, runId, state, sliceId, { requireSnapsho
|
|
|
241
261
|
if (!cleanliness.clean) throw new CliError(`slice '${sliceId}' ${cleanliness.reason}`);
|
|
242
262
|
const observed = observeWorktree(worktree, slice.base_ref, { ref: slice.branch });
|
|
243
263
|
if (!observed.diff_observed || observed.commit !== evidence.commit) throw new CliError(`slice '${sliceId}' REJECT does not bind the live branch head`);
|
|
244
|
-
const snapshot = requireSnapshot ? observedParkSnapshot(repo, runId, runDir) : null;
|
|
264
|
+
const snapshot = requireSnapshot ? observedParkSnapshot(repo, runId, runDir, new Set([RUN_JSON_LOCK_DIR])) : null;
|
|
245
265
|
if (requireSnapshot && !snapshot) throw new CliError(`grant-retry requires a complete current park snapshot for run '${runId}'`);
|
|
246
266
|
return { slice, limit, review, evidence, worktree, snapshot,
|
|
247
267
|
snapshotDigest: snapshot ? planDigest(Buffer.from(planeInventory(snapshot))) : null };
|
|
248
268
|
}
|
|
249
269
|
|
|
270
|
+
function assertNoRetryGrantTransaction(repo, runId, command) {
|
|
271
|
+
if (typeof runId !== "string" || !/^[a-z0-9](?:[a-z0-9._-]*[a-z0-9])?$/u.test(runId)) return;
|
|
272
|
+
const canonicalRepo = existsSync(repo) ? realpathSync(repo) : repo;
|
|
273
|
+
const container = dirname(canonicalRepo), sandbox = basename(container) === ".factory-sandboxes" && basename(canonicalRepo) === runId;
|
|
274
|
+
const operatorRoot = sandbox ? dirname(container) : canonicalRepo;
|
|
275
|
+
const canonical = join(operatorRoot, CONTROL_PLANE, ".parked", runId);
|
|
276
|
+
if (hasRetryGrantTransaction(canonical, runId)) {
|
|
277
|
+
throw new CliError(`factory ${command} refuses an interrupted retry-grant transaction for '${runId}'; run factory snapshot first`);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
250
281
|
function runDirFor(flags, runId) {
|
|
251
282
|
if (!runId) throw new CliError("a <run-id> is required");
|
|
252
283
|
return join(resolve(flags.repo ?? process.cwd()), CONTROL_PLANE, runId);
|
|
@@ -367,7 +398,7 @@ function branchPoint(run) {
|
|
|
367
398
|
return base;
|
|
368
399
|
}
|
|
369
400
|
|
|
370
|
-
async function writeObservedEvidence({ runDir, runId, subject, attempt, branch, baseRef, worktree, status, blockedReason, claim, testCommand, skipReason, shellCommand, testTimeoutMs }) {
|
|
401
|
+
async function writeObservedEvidence({ repo, runDir, runId, subject, attempt, branch, baseRef, worktree, status, blockedReason, claim, testCommand, skipReason, shellCommand, testTimeoutMs }) {
|
|
371
402
|
const evidence = buildEvidence({
|
|
372
403
|
subject, attempt, branch, baseRef, worktree, status, blockedReason, claim, runId,
|
|
373
404
|
testCommand, skipReason, shellCommand, testTimeoutMs,
|
|
@@ -377,7 +408,10 @@ async function writeObservedEvidence({ runDir, runId, subject, attempt, branch,
|
|
|
377
408
|
evidence.review_ready = false;
|
|
378
409
|
evidence.blocked_reason = evidence.blocked_reason ?? `base ${baseRef} is ${ancestry} of HEAD`;
|
|
379
410
|
}
|
|
380
|
-
await
|
|
411
|
+
await withRunJsonLock(runDir, async () => {
|
|
412
|
+
assertNoRetryGrantTransaction(repo, runId, "observe");
|
|
413
|
+
await writeProtectedJsonAtomic(runDir, evidenceRef(subject), evidence);
|
|
414
|
+
}, { nonExpiring: true, reentrant: true });
|
|
381
415
|
return { evidence, ancestry };
|
|
382
416
|
}
|
|
383
417
|
|
|
@@ -491,7 +525,7 @@ async function runRepositoryVerifyAttempts({ repo, runDir, runId, run, mergeComm
|
|
|
491
525
|
// False-green enforcement: one invocation gets at most two executions, never an unbounded recovery loop.
|
|
492
526
|
for (let attempt = 1; attempt <= 2; attempt += 1) {
|
|
493
527
|
const { evidence } = await writeObservedEvidence({
|
|
494
|
-
runDir, runId, subject: "test-verifier", attempt, branch: run.branch,
|
|
528
|
+
repo, runDir, runId, subject: "test-verifier", attempt, branch: run.branch,
|
|
495
529
|
baseRef, worktree: attemptIntegration.worktree, status: "completed", blockedReason: null,
|
|
496
530
|
claim: null, testCommand: verify.command, skipReason: null, shellCommand: true,
|
|
497
531
|
testTimeoutMs: verify.timeoutMs,
|
|
@@ -541,7 +575,7 @@ const HANDLERS = {
|
|
|
541
575
|
const at = stamp(flags);
|
|
542
576
|
const repo = resolve(flags.repo ?? process.cwd());
|
|
543
577
|
const runDir = runDirFor(flags, runId);
|
|
544
|
-
return emit(flags, await reverifyRepair({ repo, runDir, runId, recordId, at }));
|
|
578
|
+
return emit(flags, await reverifyRepair({ repo, runDir, runId, recordId, at, beforeWrite: () => assertNoRetryGrantTransaction(repo, runId, "reverify-repair") }));
|
|
545
579
|
},
|
|
546
580
|
|
|
547
581
|
"effective-push"(positional) {
|
|
@@ -812,7 +846,18 @@ const HANDLERS = {
|
|
|
812
846
|
const next = await transition(runDir, {
|
|
813
847
|
participants: [{ familyId: "envelope", mode }, { familyId: "slices", mode }],
|
|
814
848
|
reobservers: new Map([["envelope", async () => recheck()]]),
|
|
815
|
-
|
|
849
|
+
// Enforcement: the durable fence is published while the qualified snapshot is still canonical.
|
|
850
|
+
// It hides stale recovery authority across process death until the manifest outcome is reconciled.
|
|
851
|
+
finalGuard: ({ source }) => {
|
|
852
|
+
recheck();
|
|
853
|
+
prepareRetryGrantTransaction({ canonical: qualified.snapshot, runDir, runId, source,
|
|
854
|
+
expectedSnapshotDigest: qualified.snapshotDigest });
|
|
855
|
+
},
|
|
856
|
+
commitFailureGuard: () => reconcileRetryGrantTransaction({ canonical: qualified.snapshot, runDir, runId }),
|
|
857
|
+
afterCommit: () => {
|
|
858
|
+
try { return reconcileRetryGrantTransaction({ canonical: qualified.snapshot, runDir, runId }); }
|
|
859
|
+
catch (error) { throw new CliError("grant-retry committed; snapshot revocation cleanup is pending", { cause: error }); }
|
|
860
|
+
},
|
|
816
861
|
apply: (state) => {
|
|
817
862
|
if (!isDeepStrictEqual(state, current)) throw new CliError("grant-retry refused because run state changed after qualification");
|
|
818
863
|
const existing = state.slices.find((slice) => slice.id === sliceId), previousLimit = effectiveRetryLimit(state, existing);
|
|
@@ -1119,7 +1164,7 @@ const HANDLERS = {
|
|
|
1119
1164
|
: null;
|
|
1120
1165
|
|
|
1121
1166
|
const { evidence, ancestry } = await writeObservedEvidence({
|
|
1122
|
-
runDir, runId, subject,
|
|
1167
|
+
repo, runDir, runId, subject,
|
|
1123
1168
|
attempt,
|
|
1124
1169
|
branch: flags.repositoryVerify ? run.branch : flags.branch ?? null,
|
|
1125
1170
|
baseRef: flags.base, worktree, status: flags.status ?? "completed",
|
|
@@ -1420,9 +1465,11 @@ const HANDLERS = {
|
|
|
1420
1465
|
if (!isDeepStrictEqual(state, current)) throw new CliError("factory resume bootstrap refused: run.json bytes changed while bootstrap ran; current state was preserved");
|
|
1421
1466
|
if (!sameSessionOwner(runDir, boundOwner)) throw new CliError("factory resume bootstrap refused: factory.lock is absent, stale, or no longer names the same owner; current state and owner were preserved");
|
|
1422
1467
|
};
|
|
1423
|
-
// Enforcement:
|
|
1424
|
-
|
|
1425
|
-
|
|
1468
|
+
// Enforcement: bootstrap permits its bound session heartbeat; serialize every later effect with grants.
|
|
1469
|
+
const next = await withRunJsonLock(runDir, async () => {
|
|
1470
|
+
assertNoRetryGrantTransaction(repo, runId, "resume"); assertBinding({ state: validateRun(JSON.parse(readFileSync(join(runDir, "run.json")))) });
|
|
1471
|
+
if (success) await writeProtectedFileAtomic(runDir, "WORKFLOW.md", readFileSync(new URL("../WORKFLOW.md", import.meta.url)));
|
|
1472
|
+
return transition(runDir, {
|
|
1426
1473
|
participants: [{ familyId: "envelope", mode: success ? "resume-needs-human" : "record-bootstrap" }],
|
|
1427
1474
|
reobservers: new Map([["envelope", assertBinding]]), finalGuard: ({ state, source }) => {
|
|
1428
1475
|
if (!readFileSync(join(runDir, "run.json")).equals(boundRunBytes) || !isDeepStrictEqual(state, current)) {
|
|
@@ -1436,7 +1483,8 @@ const HANDLERS = {
|
|
|
1436
1483
|
},
|
|
1437
1484
|
apply: (state) => ({ ...state, ...(success ? { status: "running" } : {}), updated_at: at,
|
|
1438
1485
|
...(outcome ? { bootstrap_command: config.bootstrapCommand, bootstrap_exit: outcome.exit } : {}) }),
|
|
1439
|
-
|
|
1486
|
+
});
|
|
1487
|
+
}, { allowReentrant: true, nonExpiring: true });
|
|
1440
1488
|
if (outcome?.refusal) throw new CliError(`${outcome.refusal}; run remains needs-human and its historical terminal result is preserved`);
|
|
1441
1489
|
return emit(flags, {
|
|
1442
1490
|
run_id: runId, status: next.status, terminal_result: next.terminal_result,
|
package/bin/restore.js
CHANGED
|
@@ -286,6 +286,7 @@ export async function dispatchRestore(positional, flags, operations = {}) {
|
|
|
286
286
|
// Test seam only: production supplies no hook. The final guard below must catch any intervening writer.
|
|
287
287
|
if (operations.beforeManifest) await operations.beforeManifest({ runDir, sandbox });
|
|
288
288
|
const finalGuard = () => {
|
|
289
|
+
if ([`.grant-retry-${runId}.json`, `.grant-retry-${runId}.json.staging`].some((name) => entryState(join(dirname(qualified.source), name)))) throw new RestoreError("restore refuses an interrupted retry-grant transaction");
|
|
289
290
|
if (!readFileSync(join(qualified.source, "run.json")).equals(qualified.bytes) || inventory(qualified.source) !== qualified.inventory) throw new RestoreError("park snapshot changed while restore was running; run.json was not published");
|
|
290
291
|
if (inventory(runDir) !== preparedInventory) throw new RestoreError("restored control plane changed before manifest publication; run.json was not published");
|
|
291
292
|
if (entryState(legacyManifest)) throw new RestoreError(`live run manifest appeared at '${legacyManifest}' while restore was running`);
|
package/bin/snapshot.js
CHANGED
|
@@ -6,11 +6,13 @@
|
|
|
6
6
|
// the plane is worse than none -- `status` reports a path, an operator believes the run is recoverable,
|
|
7
7
|
// and the copy is found partial only when it is needed -- so verify-before-commit cannot be delegated to
|
|
8
8
|
// prose for a caller that is not the driver.
|
|
9
|
-
import {
|
|
9
|
+
import { createHash } from "node:crypto";
|
|
10
|
+
import { mkdirSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync } from "node:fs";
|
|
10
11
|
import { join, resolve } from "node:path";
|
|
11
12
|
import { CONTROL_PLANE, validateRun } from "../state/schema.js";
|
|
13
|
+
import { hasRetryGrantTransaction, isRetryGrantTransition, reconcileRetryGrantTransaction } from "../state/retry-grant-transaction.js";
|
|
12
14
|
import { RUN_JSON_LOCK_DIR, withRunJsonLock } from "../core/run-lock.js";
|
|
13
|
-
import { assertRetryExtensionBindings, copySnapshot, entryState, inventory } from "./restore.js";
|
|
15
|
+
import { assertRetryExtensionBindings, copySnapshot, entryState, inventory, inventoryEntries } from "./restore.js";
|
|
14
16
|
|
|
15
17
|
// The two root entries excluded from publication are coordination, not run state: `factory.lock` is session
|
|
16
18
|
// liveness and `run-json.lock` serializes this copy with state transitions. Only those exact root paths are
|
|
@@ -23,15 +25,57 @@ export class SnapshotError extends Error {
|
|
|
23
25
|
|
|
24
26
|
const ID = /^[a-z0-9](?:[a-z0-9._-]*[a-z0-9])?$/u;
|
|
25
27
|
|
|
26
|
-
//
|
|
27
|
-
//
|
|
28
|
-
//
|
|
29
|
-
function preflight(staging, prior) {
|
|
28
|
+
// Version 0.10.1 also used `.prior-$R` as its grant seam. Exact old bytes prove its grant did not
|
|
29
|
+
// commit; the grant contracts plus the bound snapshot digest prove that it did. Anything else is
|
|
30
|
+
// preserved and refused. New grants use a distinct durable fence and never overload this path.
|
|
31
|
+
function preflight(staging, prior, canonical, plane, runId) {
|
|
30
32
|
if (entryState(staging)) throw new SnapshotError(`residual staging tree '${staging}' must be removed before publishing`);
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
33
|
+
const cleanup = `${prior}.cleanup`;
|
|
34
|
+
if (entryState(cleanup)) rmSync(cleanup, { recursive: true, force: true });
|
|
35
|
+
const priorState = entryState(prior);
|
|
36
|
+
if (!priorState) return;
|
|
37
|
+
if (priorState.isSymbolicLink() || !priorState.isDirectory()) {
|
|
38
|
+
throw new SnapshotError(`residual '${prior}' has an unsafe type`);
|
|
34
39
|
}
|
|
40
|
+
if (!entryState(canonical)) {
|
|
41
|
+
const before = qualifyManifest(prior, runId, "prior park manifest");
|
|
42
|
+
const current = qualifyManifest(plane, runId, "run manifest");
|
|
43
|
+
if (readFileSync(join(prior, "run.json")).equals(readFileSync(join(plane, "run.json")))) {
|
|
44
|
+
removeAbandonedGrantCandidate(plane, prior, before);
|
|
45
|
+
if (inventory(plane, LIVENESS) !== inventory(prior, LIVENESS)) {
|
|
46
|
+
throw new SnapshotError("interrupted retry grant live plane changed from its prior snapshot");
|
|
47
|
+
}
|
|
48
|
+
renameSync(prior, canonical);
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
if (!isCommittedRetryGrant(before, current, prior)) {
|
|
52
|
+
throw new SnapshotError(`residual '${prior}' is not a recoverable retry-grant transaction`);
|
|
53
|
+
}
|
|
54
|
+
} else {
|
|
55
|
+
qualifyManifest(canonical, runId, "canonical park manifest");
|
|
56
|
+
}
|
|
57
|
+
try { renameSync(prior, cleanup); rmSync(cleanup, { recursive: true, force: true }); }
|
|
58
|
+
catch (error) { throw new SnapshotError(`residual '${prior}' could not be quarantined`, { cause: error }); }
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function removeAbandonedGrantCandidate(plane, prior, before) {
|
|
62
|
+
const candidates = readdirSync(plane).filter((name) => /^\.[0-9a-f-]{36}\.tmp$/u.test(name)
|
|
63
|
+
&& !entryState(join(prior, name)));
|
|
64
|
+
if (candidates.length === 0) return;
|
|
65
|
+
if (candidates.length !== 1) throw new SnapshotError("interrupted retry grant has ambiguous atomic candidates");
|
|
66
|
+
const candidatePath = join(plane, candidates[0]), state = entryState(candidatePath);
|
|
67
|
+
let candidate;
|
|
68
|
+
try { candidate = state?.isFile() && !state.isSymbolicLink()
|
|
69
|
+
? validateRun(JSON.parse(readFileSync(candidatePath, "utf8"))) : null; } catch { candidate = null; }
|
|
70
|
+
if (!candidate || !isCommittedRetryGrant(before, candidate, prior)) {
|
|
71
|
+
throw new SnapshotError("interrupted retry grant has an unbound atomic candidate");
|
|
72
|
+
}
|
|
73
|
+
rmSync(candidatePath);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function isCommittedRetryGrant(before, after, prior) {
|
|
77
|
+
const digest = `sha256:${createHash("sha256").update(Buffer.from(inventoryEntries(prior).join("\n"))).digest("hex")}`;
|
|
78
|
+
return isRetryGrantTransition(before, after, digest);
|
|
35
79
|
}
|
|
36
80
|
|
|
37
81
|
// Applied to the live plane before staging and to the staged tree before the commit. The same schema and
|
|
@@ -58,73 +102,72 @@ export async function dispatchSnapshot(positional, flags, operations = {}) {
|
|
|
58
102
|
if (flags.repo !== undefined && (typeof flags.repo !== "string" || !flags.repo.trim())) throw new SnapshotError("--repo must name a directory");
|
|
59
103
|
const runId = positional[0], operatorInput = resolve(flags.repo ?? process.cwd());
|
|
60
104
|
if (!entryState(operatorInput)) throw new SnapshotError(`operator repository '${operatorInput}' is not observable`);
|
|
61
|
-
const operatorRoot = realpathSync(operatorInput);
|
|
105
|
+
const operatorRoot = realpathSync(operatorInput), parked = join(operatorRoot, CONTROL_PLANE, ".parked");
|
|
106
|
+
const canonical = join(parked, runId), fenced = hasRetryGrantTransaction(canonical, runId);
|
|
62
107
|
const candidates = [join(operatorRoot, CONTROL_PLANE, runId), join(operatorRoot, ".factory-sandboxes", runId, CONTROL_PLANE, runId)]
|
|
63
|
-
.filter((candidate) => entryState(join(candidate, "run.json"))
|
|
108
|
+
.filter((candidate) => entryState(join(candidate, "run.json"))
|
|
109
|
+
|| (fenced && entryState(candidate)?.isDirectory() && !entryState(candidate).isSymbolicLink()));
|
|
64
110
|
if (candidates.length !== 1) throw new SnapshotError(candidates.length
|
|
65
111
|
? `factory snapshot found ambiguous live manifests for '${runId}'`
|
|
66
112
|
: `control plane for '${runId}' is not observable`);
|
|
67
113
|
const plane = candidates[0];
|
|
68
|
-
qualifyManifest(plane, runId, "run manifest");
|
|
69
114
|
|
|
70
115
|
return withRunJsonLock(plane, async () => {
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
throw new SnapshotError(`'${parent}' must be a real directory to publish a snapshot`);
|
|
116
|
+
// Created one directory at a time, never written through a symlinked parent: the snapshot must land
|
|
117
|
+
// under the operator's own control plane and nowhere a link could redirect it.
|
|
118
|
+
for (const parent of [join(operatorRoot, CONTROL_PLANE), parked]) {
|
|
119
|
+
const present = entryState(parent);
|
|
120
|
+
if (present && (present.isSymbolicLink() || !present.isDirectory())) {
|
|
121
|
+
throw new SnapshotError(`'${parent}' must be a real directory to publish a snapshot`);
|
|
122
|
+
}
|
|
123
|
+
if (!present) mkdirSync(parent);
|
|
80
124
|
}
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
preflight(staging, prior);
|
|
125
|
+
const staging = join(parked, `.staging-${runId}`), prior = join(parked, `.prior-${runId}`);
|
|
126
|
+
try { reconcileRetryGrantTransaction({ canonical, runDir: plane, runId }); }
|
|
127
|
+
catch (error) { throw new SnapshotError(`factory snapshot could not reconcile retry-grant transaction for '${runId}'`, { cause: error }); }
|
|
128
|
+
qualifyManifest(plane, runId, "run manifest");
|
|
129
|
+
preflight(staging, prior, canonical, plane, runId);
|
|
87
130
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
committed = true;
|
|
103
|
-
} else {
|
|
104
|
-
renameSync(canonical, prior);
|
|
105
|
-
try {
|
|
106
|
-
renameSync(staging, canonical); // commit point, replacing a previous snapshot
|
|
131
|
+
let committed = false;
|
|
132
|
+
try {
|
|
133
|
+
(operations.copy ?? copySnapshot)(plane, staging, LIVENESS);
|
|
134
|
+
// Verify before the commit point. An unverified staging tree is never published, so a copy that
|
|
135
|
+
// raced a write is discarded rather than published as evidence of a run it does not describe.
|
|
136
|
+
if (inventory(plane, LIVENESS) !== inventory(staging, LIVENESS)) {
|
|
137
|
+
throw new SnapshotError("staged snapshot does not match the live control plane; nothing was published");
|
|
138
|
+
}
|
|
139
|
+
// Equality alone cannot catch a manifest replaced between qualification and copy: both trees then
|
|
140
|
+
// hold the same unvalidated bytes and compare equal. Qualify what is about to be renamed.
|
|
141
|
+
qualifyManifest(staging, runId, "staged run manifest");
|
|
142
|
+
await operations.beforeCommit?.({ plane, staging, canonical });
|
|
143
|
+
if (!entryState(canonical)) {
|
|
144
|
+
renameSync(staging, canonical); // commit point, with no snapshot present
|
|
107
145
|
committed = true;
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
146
|
+
} else {
|
|
147
|
+
renameSync(canonical, prior);
|
|
148
|
+
try {
|
|
149
|
+
renameSync(staging, canonical); // commit point, replacing a previous snapshot
|
|
150
|
+
committed = true;
|
|
151
|
+
} catch (error) {
|
|
152
|
+
// The first rename succeeded and the second did not: put the previous snapshot back and report
|
|
153
|
+
// that nothing was committed, rather than leaving the canonical path empty.
|
|
154
|
+
renameSync(prior, canonical);
|
|
155
|
+
throw new SnapshotError("snapshot commit failed; the previous snapshot was restored", { cause: error });
|
|
156
|
+
}
|
|
113
157
|
}
|
|
158
|
+
} finally {
|
|
159
|
+
// Before the commit point every failure removes only the staging tree. After it the published
|
|
160
|
+
// snapshot is authoritative and is never rolled back.
|
|
161
|
+
if (!committed) rmSync(staging, { recursive: true, force: true });
|
|
114
162
|
}
|
|
115
|
-
} finally {
|
|
116
|
-
// Before the commit point every failure removes only the staging tree. After it the published
|
|
117
|
-
// snapshot is authoritative and is never rolled back.
|
|
118
|
-
if (!committed) rmSync(staging, { recursive: true, force: true });
|
|
119
|
-
}
|
|
120
163
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
164
|
+
// Cleanup, not publication: a residual `.prior-$R` is reported and left for the next preflight rather
|
|
165
|
+
// than turning a completed publication into a failure.
|
|
166
|
+
let residual = null;
|
|
167
|
+
if (entryState(prior)) {
|
|
168
|
+
try { rmSync(prior, { recursive: true, force: true }); }
|
|
169
|
+
catch { residual = prior; }
|
|
170
|
+
}
|
|
171
|
+
return { run_id: runId, park_snapshot: canonical, residual };
|
|
129
172
|
}, { nonExpiring: true });
|
|
130
173
|
}
|
package/core/atomic-write.js
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
// become readable. beforeCommit is the last race seam used by CAS and create-only tests.
|
|
8
8
|
import { lstat, open, realpath, rename as fsRename, unlink } from "node:fs/promises";
|
|
9
9
|
import { randomUUID } from "node:crypto";
|
|
10
|
-
import { isAbsolute, join, resolve, sep } from "node:path";
|
|
10
|
+
import { basename, isAbsolute, join, resolve, sep } from "node:path";
|
|
11
11
|
|
|
12
12
|
export class ProtectedWriteError extends Error {
|
|
13
13
|
constructor(message, cause) {
|
|
@@ -23,14 +23,16 @@ export function writeProtectedJsonAtomic(rootDir, relativePath, value, options =
|
|
|
23
23
|
export async function writeProtectedFileAtomic(rootDir, relativePath, data, options = {}) {
|
|
24
24
|
const targetPath = resolveProtectedPath(rootDir, relativePath), parentDir = resolve(targetPath, "..");
|
|
25
25
|
const createOnly = options.createOnly === true, beforeCommit = options.hooks?.beforeCommit;
|
|
26
|
+
const afterRename = options.hooks?.afterRename;
|
|
26
27
|
const openFile = options.fsOps?.open ?? open;
|
|
27
28
|
const bytes = Buffer.isBuffer(data) ? Buffer.from(data) : Buffer.from(String(data), "utf8");
|
|
28
|
-
|
|
29
|
+
const tempParent = options.tempDirectory === undefined ? parentDir : resolveProtectedPath(rootDir, options.tempDirectory);
|
|
30
|
+
for (const parent of new Set([parentDir, tempParent])) await assertSafeParent(rootDir, parent);
|
|
29
31
|
await assertSafeTarget(targetPath, createOnly);
|
|
30
32
|
if (createOnly) return writeProtectedCreate(rootDir, parentDir, targetPath, bytes, beforeCommit, openFile);
|
|
31
33
|
|
|
32
|
-
const tempPath = join(
|
|
33
|
-
let handle = null, published = false,
|
|
34
|
+
const tempPath = join(tempParent, `.${basename(targetPath)}.${randomUUID()}.tmp`), rename = options.fsOps?.rename ?? fsRename;
|
|
35
|
+
let handle = null, published = false, renamed = false;
|
|
34
36
|
try {
|
|
35
37
|
handle = await openFile(tempPath, "wx+", 0o600);
|
|
36
38
|
await handle.writeFile(bytes);
|
|
@@ -40,15 +42,17 @@ export async function writeProtectedFileAtomic(rootDir, relativePath, data, opti
|
|
|
40
42
|
await assertSafeTarget(targetPath, false);
|
|
41
43
|
await assertPublishedInode(handle, tempPath, bytes);
|
|
42
44
|
await rename(tempPath, targetPath);
|
|
43
|
-
|
|
45
|
+
renamed = true;
|
|
46
|
+
if (typeof afterRename === "function") await afterRename({ source: tempPath, destination: targetPath });
|
|
44
47
|
await assertPublishedInode(handle, targetPath, bytes);
|
|
45
48
|
await handle.close();
|
|
46
49
|
handle = null;
|
|
47
50
|
published = true;
|
|
48
51
|
} catch (error) {
|
|
49
52
|
if (handle) try { await handle.close(); } catch { /* the original error is primary */ }
|
|
50
|
-
|
|
51
|
-
|
|
53
|
+
// Once rename succeeds the target is the only durable after-image. Preserve it for validation or
|
|
54
|
+
// transaction recovery instead of converting a post-commit verification failure into lost state.
|
|
55
|
+
if (!published && !renamed) try { await unlink(tempPath); } catch (cleanupError) {
|
|
52
56
|
if (cleanupError?.code !== "ENOENT") throw new ProtectedWriteError("protected temporary file cleanup is indeterminate", cleanupError);
|
|
53
57
|
}
|
|
54
58
|
throw error instanceof ProtectedWriteError ? error : new ProtectedWriteError("protected file commit failed", error);
|
package/core/run-lock.js
CHANGED
|
@@ -2,11 +2,12 @@
|
|
|
2
2
|
// The 0c spike reused this lock as-is, which is why it is lifted rather than
|
|
3
3
|
// rewritten: hand-rolling lock reclaim and steal logic is where subtle crash bugs
|
|
4
4
|
// live. Only the imports and the extracted constants below are new.
|
|
5
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
5
6
|
import { constants } from "node:fs";
|
|
6
7
|
import { lstat, mkdir, open, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
7
8
|
import { randomUUID } from "node:crypto";
|
|
8
9
|
import { hostname } from "node:os";
|
|
9
|
-
import { join } from "node:path";
|
|
10
|
+
import { join, resolve } from "node:path";
|
|
10
11
|
|
|
11
12
|
const DEFAULT_LOCK_TIMEOUT_MS = 1000;
|
|
12
13
|
const DEFAULT_LOCK_RETRY_DELAY_MS = 10;
|
|
@@ -15,11 +16,14 @@ const DEFAULT_MISSING_OWNER_STEAL_MS = 5000;
|
|
|
15
16
|
export const RUN_JSON_LOCK_DIR = "run-json.lock";
|
|
16
17
|
const LOCK_DIR = RUN_JSON_LOCK_DIR;
|
|
17
18
|
const LOCK_OWNER_FILE = "owner.json";
|
|
19
|
+
const LOCK_CONTEXT = new AsyncLocalStorage();
|
|
18
20
|
|
|
19
21
|
export async function withRunJsonLock(runDir, fn, options = {}) {
|
|
20
22
|
if (typeof fn !== "function") throw new Error("withRunJsonLock requires a callback");
|
|
21
|
-
const
|
|
22
|
-
if (
|
|
23
|
+
const key = resolve(runDir), inherited = LOCK_CONTEXT.getStore()?.get(key);
|
|
24
|
+
if (options.reentrant === true && inherited?.active === true) return fn(inherited);
|
|
25
|
+
const { allowReentrant, onBeforeSteal, nonExpiring, reentrant } = options;
|
|
26
|
+
if ([allowReentrant, reentrant, nonExpiring].some((value) => value !== undefined && typeof value !== "boolean")) throw new Error("run lock boolean options must be boolean");
|
|
23
27
|
if (onBeforeSteal !== undefined && typeof onBeforeSteal !== "function") {
|
|
24
28
|
throw new Error("onBeforeSteal must be a function");
|
|
25
29
|
}
|
|
@@ -32,6 +36,7 @@ export async function withRunJsonLock(runDir, fn, options = {}) {
|
|
|
32
36
|
let stealAttempted = false;
|
|
33
37
|
let createdIdentity = null;
|
|
34
38
|
let owner = null;
|
|
39
|
+
let context = null;
|
|
35
40
|
let ownerPublished = false;
|
|
36
41
|
let publishedEvidence = null;
|
|
37
42
|
|
|
@@ -68,8 +73,13 @@ export async function withRunJsonLock(runDir, fn, options = {}) {
|
|
|
68
73
|
publishedEvidence = await readLockOwnerEvidence(ownerPath);
|
|
69
74
|
if (!sameLockOwner(owner, publishedEvidence?.owner)) throw new Error(`run.json lock owner publication failed at ${lockDir}`);
|
|
70
75
|
ownerPublished = true;
|
|
71
|
-
|
|
76
|
+
const lease = { lock_dir: lockDir, owner, active: true };
|
|
77
|
+
if (!allowReentrant) return await fn(lease);
|
|
78
|
+
context = lease;
|
|
79
|
+
const locks = new Map(LOCK_CONTEXT.getStore() ?? []); locks.set(key, context);
|
|
80
|
+
return await LOCK_CONTEXT.run(locks, () => fn(context));
|
|
72
81
|
} finally {
|
|
82
|
+
if (context) context.active = false;
|
|
73
83
|
if (ownerPublished) {
|
|
74
84
|
await releaseOwnedRunJsonLock(runDir, lockDir, createdIdentity, publishedEvidence);
|
|
75
85
|
} else if (!ownerPublished && !(await lockOwnerEntryExists(ownerPath))) {
|
package/core/write-core.js
CHANGED
|
@@ -14,6 +14,8 @@ export async function coordinateRunJsonTransition(runDir, options) {
|
|
|
14
14
|
reobservers = new Map(),
|
|
15
15
|
atomicWriteHooks,
|
|
16
16
|
finalGuard,
|
|
17
|
+
commitFailureGuard,
|
|
18
|
+
afterCommit,
|
|
17
19
|
} = options ?? {};
|
|
18
20
|
const registry = contractRegistry(contracts);
|
|
19
21
|
const participants = participantRegistry(descriptor, registry);
|
|
@@ -70,16 +72,27 @@ export async function coordinateRunJsonTransition(runDir, options) {
|
|
|
70
72
|
// Only the synchronous final guard may run between this line and the rename.
|
|
71
73
|
const finalObserved = deepFreeze(await readRunState(runDir, validateRun));
|
|
72
74
|
assertUnchanged(finalObserved, initial);
|
|
75
|
+
let guarded = false;
|
|
73
76
|
if (typeof finalGuard === "function") {
|
|
74
|
-
const
|
|
75
|
-
if (
|
|
77
|
+
const result = finalGuard({ state: finalObserved, candidate, source });
|
|
78
|
+
if (result && typeof result.then === "function") throw new Error("final commit guard must be synchronous");
|
|
79
|
+
guarded = true;
|
|
80
|
+
}
|
|
81
|
+
try {
|
|
82
|
+
await rename(source, destination);
|
|
83
|
+
} catch (error) {
|
|
84
|
+
if (guarded && typeof commitFailureGuard === "function") {
|
|
85
|
+
const result = commitFailureGuard({ state: finalObserved, candidate, source, error });
|
|
86
|
+
if (result && typeof result.then === "function") throw new Error("commit failure guard must be synchronous");
|
|
87
|
+
}
|
|
88
|
+
throw error;
|
|
76
89
|
}
|
|
77
|
-
await rename(source, destination);
|
|
78
90
|
},
|
|
79
91
|
},
|
|
80
92
|
});
|
|
93
|
+
if (typeof afterCommit === "function") await afterCommit({ state: initial, candidate });
|
|
81
94
|
return candidate;
|
|
82
|
-
});
|
|
95
|
+
}, { reentrant: true });
|
|
83
96
|
}
|
|
84
97
|
|
|
85
98
|
function assertUnchanged(observed, initial) {
|
|
@@ -66,7 +66,7 @@ function removeDetached(repo, temporary) {
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
|
|
69
|
-
export async function reverifyRepair({ repo, runDir, runId, recordId, at }) {
|
|
69
|
+
export async function reverifyRepair({ repo, runDir, runId, recordId, at, beforeWrite = () => {} }) {
|
|
70
70
|
const parsedAt = typeof at === "string" ? Date.parse(at) : NaN;
|
|
71
71
|
if (!Number.isFinite(parsedAt) || new Date(parsedAt).toISOString() !== at) throw new Error("repair re-verification timestamp is not canonical");
|
|
72
72
|
const preread = readRepairState({ repo, runDir, runId, recordId });
|
|
@@ -84,7 +84,7 @@ export async function reverifyRepair({ repo, runDir, runId, recordId, at }) {
|
|
|
84
84
|
let reservation;
|
|
85
85
|
try {
|
|
86
86
|
reservation = await withRunJsonLock(runDir, async () => {
|
|
87
|
-
const current = readRepairState({ repo, runDir, runId, recordId });
|
|
87
|
+
const current = readRepairState({ repo, runDir, runId, recordId }); beforeWrite();
|
|
88
88
|
assertEnvelope(current.run);
|
|
89
89
|
assertDetached(temporary.worktree, current.selected.repair_commit);
|
|
90
90
|
const history = current.selectedHistory;
|
|
@@ -131,7 +131,7 @@ export async function reverifyRepair({ repo, runDir, runId, recordId, at }) {
|
|
|
131
131
|
removeDetached(repo, temporary);
|
|
132
132
|
|
|
133
133
|
const completed = await withRunJsonLock(runDir, async () => {
|
|
134
|
-
const current = readRepairState({ repo, runDir, runId, recordId });
|
|
134
|
+
const current = readRepairState({ repo, runDir, runId, recordId }); beforeWrite();
|
|
135
135
|
assertEnvelope(current.run);
|
|
136
136
|
if (!current.runBytes.equals(reservation.runBytes) || !current.journalBytes.equals(reservation.journalBytes)) {
|
|
137
137
|
throw new Error("run or repair journal bytes changed during re-verification");
|
package/package.json
CHANGED
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { closeSync, fsyncSync, lstatSync, openSync, readFileSync, readdirSync, renameSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
|
+
import { FAMILY_CONTRACTS } from "../core/contracts.js";
|
|
5
|
+
import { validateRun } from "./schema.js";
|
|
6
|
+
import { inventoryEntries } from "../bin/restore.js";
|
|
7
|
+
|
|
8
|
+
const VERSION = 1;
|
|
9
|
+
const UUID_TEMP = /^\.run\.json\.[0-9a-f-]{36}\.tmp$/u;
|
|
10
|
+
const sha256 = (bytes) => `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
|
|
11
|
+
const state = (path) => { try { return lstatSync(path); } catch (error) { if (error?.code === "ENOENT") return null; throw error; } };
|
|
12
|
+
const paths = (canonical, runId) => { const fence = join(dirname(canonical), `.grant-retry-${runId}.json`);
|
|
13
|
+
return { fence, staging: `${fence}.staging`, revoked: join(dirname(canonical), `.revoked-grant-retry-${runId}`) }; };
|
|
14
|
+
const snapshotDigest = (path) => sha256(Buffer.from(inventoryEntries(path).join("\n")));
|
|
15
|
+
|
|
16
|
+
export function hasRetryGrantTransaction(canonical, runId) {
|
|
17
|
+
const { fence, staging } = paths(canonical, runId);
|
|
18
|
+
return Boolean(state(staging) || state(fence));
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function prepareRetryGrantTransaction({ canonical, runDir, runId, source, expectedSnapshotDigest }) {
|
|
22
|
+
const { fence, staging, revoked } = paths(canonical, runId);
|
|
23
|
+
if (state(fence) || state(staging) || state(revoked)) throw new Error("retry-grant transaction already exists");
|
|
24
|
+
const beforeBytes = readFileSync(join(runDir, "run.json")), afterBytes = readFileSync(source);
|
|
25
|
+
const before = validateRun(JSON.parse(beforeBytes)), after = validateRun(JSON.parse(afterBytes));
|
|
26
|
+
if (snapshotDigest(canonical) !== expectedSnapshotDigest
|
|
27
|
+
|| sha256(readFileSync(join(canonical, "run.json"))) !== sha256(beforeBytes)
|
|
28
|
+
|| !isRetryGrantTransition(before, after, expectedSnapshotDigest)) {
|
|
29
|
+
throw new Error("retry-grant transaction does not bind the qualified snapshot and candidate");
|
|
30
|
+
}
|
|
31
|
+
const record = { version: VERSION, run_id: runId, before_sha256: sha256(beforeBytes), after_sha256: sha256(afterBytes),
|
|
32
|
+
snapshot_digest: expectedSnapshotDigest, snapshot_run_sha256: sha256(beforeBytes),
|
|
33
|
+
before_state_sha256: sha256(Buffer.from(JSON.stringify(before))), before_run: before };
|
|
34
|
+
let fd, published = false;
|
|
35
|
+
try {
|
|
36
|
+
fd = openSync(staging, "wx", 0o600);
|
|
37
|
+
writeFileSync(fd, `${JSON.stringify(record, null, 2)}\n`);
|
|
38
|
+
fsyncSync(fd); closeSync(fd); fd = undefined;
|
|
39
|
+
renameSync(staging, fence); published = true;
|
|
40
|
+
syncDir(dirname(canonical));
|
|
41
|
+
} finally {
|
|
42
|
+
if (fd !== undefined) closeSync(fd);
|
|
43
|
+
if (!published) rmSync(staging, { force: true });
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function reconcileRetryGrantTransaction({ canonical, runDir, runId }) {
|
|
48
|
+
const { fence, staging, revoked } = paths(canonical, runId);
|
|
49
|
+
const fenceState = state(fence), stagingState = state(staging); const revokedState = state(revoked);
|
|
50
|
+
if (fenceState && stagingState) throw new Error("retry-grant transaction has duplicate fences");
|
|
51
|
+
if (!fenceState && stagingState && revokedState) throw new Error("retry-grant transaction has impossible staging and revocation artifacts");
|
|
52
|
+
if (!fenceState) {
|
|
53
|
+
if (stagingState) {
|
|
54
|
+
reconcileStagingTransaction({ canonical, runDir, runId, staging, stagingState });
|
|
55
|
+
return "rolled-back";
|
|
56
|
+
}
|
|
57
|
+
// X is non-authoritative cleanup after J was removed; an interrupted recursive removal is resumable.
|
|
58
|
+
if (revokedState) rmSync(revoked, { recursive: true, force: true });
|
|
59
|
+
return "none";
|
|
60
|
+
}
|
|
61
|
+
if (!fenceState.isFile() || fenceState.isSymbolicLink()) throw new Error("retry-grant transaction fence has an unsafe type");
|
|
62
|
+
let record;
|
|
63
|
+
try { record = JSON.parse(readFileSync(fence, "utf8")); } catch { throw new Error("retry-grant transaction fence is malformed"); }
|
|
64
|
+
validateRecord(record, runId);
|
|
65
|
+
const runPath = join(runDir, "run.json"); let runState = state(runPath);
|
|
66
|
+
if (!runState) {
|
|
67
|
+
const candidates = readdirSync(runDir).filter((name) => UUID_TEMP.test(name) && !state(join(canonical, name)));
|
|
68
|
+
if (candidates.length !== 1 || sha256(readCandidate(runDir, candidates[0])) !== record.after_sha256) {
|
|
69
|
+
throw new Error("retry-grant transaction has no recoverable live run manifest");
|
|
70
|
+
}
|
|
71
|
+
renameSync(join(runDir, candidates[0]), runPath); syncDir(runDir); runState = state(runPath);
|
|
72
|
+
}
|
|
73
|
+
if (!runState?.isFile() || runState.isSymbolicLink()) throw new Error("retry-grant transaction has no safe live run manifest");
|
|
74
|
+
const liveBytes = readFileSync(runPath), liveDigest = sha256(liveBytes);
|
|
75
|
+
if (liveDigest === record.before_sha256) {
|
|
76
|
+
if (revokedState || !qualifiedSnapshot(canonical, record)) throw new Error("pre-commit retry-grant transaction has lost its qualified snapshot");
|
|
77
|
+
removeAbandonedCandidate(runDir, canonical, record.after_sha256);
|
|
78
|
+
const livePlaneDigest = sha256(Buffer.from(inventoryEntries(runDir, new Set(["factory.lock", "run-json.lock"])).join("\n")));
|
|
79
|
+
if (livePlaneDigest !== record.snapshot_digest) throw new Error("pre-commit retry-grant live plane changed from its qualified snapshot");
|
|
80
|
+
unlinkSync(fence); syncDir(dirname(canonical));
|
|
81
|
+
return "rolled-back";
|
|
82
|
+
}
|
|
83
|
+
if (liveDigest !== record.after_sha256) throw new Error("retry-grant transaction does not bind the live run manifest");
|
|
84
|
+
const after = validateRun(JSON.parse(liveBytes));
|
|
85
|
+
if (!isRetryGrantTransition(record.before_run, after, record.snapshot_digest)) throw new Error("retry-grant transaction candidate is not an authorized grant");
|
|
86
|
+
syncDir(runDir);
|
|
87
|
+
const canonicalState = state(canonical);
|
|
88
|
+
if (canonicalState && revokedState) throw new Error("retry-grant transaction has ambiguous snapshot copies");
|
|
89
|
+
if (!canonicalState && !revokedState) throw new Error("retry-grant transaction has lost its fenced snapshot");
|
|
90
|
+
if (canonicalState) {
|
|
91
|
+
if (!qualifiedSnapshot(canonical, record)) throw new Error("retry-grant transaction canonical snapshot changed");
|
|
92
|
+
renameSync(canonical, revoked); syncDir(dirname(canonical));
|
|
93
|
+
} else if (revokedState && !qualifiedSnapshot(revoked, record)) {
|
|
94
|
+
throw new Error("retry-grant transaction revoked snapshot changed");
|
|
95
|
+
}
|
|
96
|
+
// C is now unreachable as recovery authority. Remove J before best-effort quarantine cleanup so a
|
|
97
|
+
// process death during recursive removal cannot wedge an otherwise committed grant.
|
|
98
|
+
unlinkSync(fence); syncDir(dirname(canonical));
|
|
99
|
+
if (state(revoked)) rmSync(revoked, { recursive: true, force: true });
|
|
100
|
+
return "committed";
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export function isRetryGrantTransition(before, after, expectedSnapshotDigest) {
|
|
104
|
+
const audit = after.retry_extensions.at(-1), scope = audit?.scope;
|
|
105
|
+
if (!["slice", "all"].includes(scope) || audit.snapshot_digest !== expectedSnapshotDigest) return false;
|
|
106
|
+
const mode = scope === "all" ? "grant-retry-all" : "grant-retry-slice";
|
|
107
|
+
try {
|
|
108
|
+
for (const contract of FAMILY_CONTRACTS) {
|
|
109
|
+
const prior = contract.project(before), next = contract.project(after);
|
|
110
|
+
contract.validateProjection(prior); contract.validateProjection(next);
|
|
111
|
+
contract.validateTransition({ mode: ["envelope", "slices"].includes(contract.id) ? mode : undefined,
|
|
112
|
+
before: prior, after: next, current: before, candidate: after });
|
|
113
|
+
}
|
|
114
|
+
return true;
|
|
115
|
+
} catch { return false; }
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function reconcileStagingTransaction({ canonical, runDir, runId, staging, stagingState }) {
|
|
119
|
+
if (!stagingState.isFile() || stagingState.isSymbolicLink()) {
|
|
120
|
+
throw new Error("retry-grant transaction staging fence has an unsafe type");
|
|
121
|
+
}
|
|
122
|
+
let record; try { record = JSON.parse(readFileSync(staging, "utf8")); validateRecord(record, runId); }
|
|
123
|
+
catch { record = null; }
|
|
124
|
+
const canonicalState = state(canonical), liveBytes = readFileSync(join(runDir, "run.json"));
|
|
125
|
+
if (!canonicalState?.isDirectory() || canonicalState.isSymbolicLink()
|
|
126
|
+
|| !readFileSync(join(canonical, "run.json")).equals(liveBytes)) {
|
|
127
|
+
throw new Error("retry-grant staging transaction has lost its pre-commit snapshot");
|
|
128
|
+
}
|
|
129
|
+
const before = validateRun(JSON.parse(liveBytes)), digest = snapshotDigest(canonical);
|
|
130
|
+
const candidates = readdirSync(runDir).filter((name) => UUID_TEMP.test(name) && !state(join(canonical, name)));
|
|
131
|
+
if (candidates.length > 1 || (!record && candidates.length === 0)) {
|
|
132
|
+
throw new Error("retry-grant transaction staging fence is malformed or ambiguous");
|
|
133
|
+
}
|
|
134
|
+
if (record && (record.before_sha256 !== sha256(liveBytes) || !qualifiedSnapshot(canonical, record))) {
|
|
135
|
+
throw new Error("retry-grant staging transaction record does not bind its pre-commit snapshot");
|
|
136
|
+
}
|
|
137
|
+
if (candidates.length === 1) {
|
|
138
|
+
const candidateBytes = readCandidate(runDir, candidates[0]);
|
|
139
|
+
let after; try { after = validateRun(JSON.parse(candidateBytes)); } catch { after = null; }
|
|
140
|
+
if (!after || (record && sha256(candidateBytes) !== record.after_sha256)
|
|
141
|
+
|| !isRetryGrantTransition(before, after, digest)) {
|
|
142
|
+
throw new Error("retry-grant staging transaction has an unbound atomic candidate");
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
const skipped = new Set(["factory.lock", "run-json.lock", ...candidates]);
|
|
146
|
+
if (sha256(Buffer.from(inventoryEntries(runDir, skipped).join("\n"))) !== digest) {
|
|
147
|
+
throw new Error("retry-grant staging transaction live plane changed from its qualified snapshot");
|
|
148
|
+
}
|
|
149
|
+
if (candidates.length) unlinkSync(join(runDir, candidates[0]));
|
|
150
|
+
unlinkSync(staging); syncDir(dirname(canonical));
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function qualifiedSnapshot(path, record) {
|
|
154
|
+
const value = state(path);
|
|
155
|
+
return Boolean(value?.isDirectory() && !value.isSymbolicLink()
|
|
156
|
+
&& snapshotDigest(path) === record.snapshot_digest
|
|
157
|
+
&& sha256(readFileSync(join(path, "run.json"))) === record.snapshot_run_sha256);
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function removeAbandonedCandidate(runDir, canonical, afterDigest) {
|
|
161
|
+
const candidates = readdirSync(runDir).filter((name) => UUID_TEMP.test(name) && !state(join(canonical, name)));
|
|
162
|
+
if (!candidates.length) return;
|
|
163
|
+
if (candidates.length !== 1 || sha256(readCandidate(runDir, candidates[0])) !== afterDigest) {
|
|
164
|
+
throw new Error("retry-grant transaction has an ambiguous atomic candidate");
|
|
165
|
+
}
|
|
166
|
+
unlinkSync(join(runDir, candidates[0])); syncDir(runDir);
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function readCandidate(runDir, name) {
|
|
170
|
+
const path = join(runDir, name), value = state(path);
|
|
171
|
+
if (!value?.isFile() || value.isSymbolicLink()) throw new Error("retry-grant atomic candidate has an unsafe type");
|
|
172
|
+
return readFileSync(path);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function validateRecord(record, runId) {
|
|
176
|
+
const keys = ["after_sha256", "before_run", "before_sha256", "before_state_sha256", "run_id", "snapshot_digest", "snapshot_run_sha256", "version"];
|
|
177
|
+
if (!record || typeof record !== "object" || Array.isArray(record) || Object.keys(record).sort().join() !== keys.join()
|
|
178
|
+
|| record.version !== VERSION || record.run_id !== runId || record.before_run?.run_id !== runId
|
|
179
|
+
|| record.before_sha256 !== record.snapshot_run_sha256
|
|
180
|
+
|| ![record.after_sha256, record.before_sha256, record.before_state_sha256, record.snapshot_digest,
|
|
181
|
+
record.snapshot_run_sha256].every((value) => /^sha256:[0-9a-f]{64}$/u.test(value))) {
|
|
182
|
+
throw new Error("retry-grant transaction fence is invalid");
|
|
183
|
+
}
|
|
184
|
+
validateRun(record.before_run);
|
|
185
|
+
if (sha256(Buffer.from(JSON.stringify(record.before_run))) !== record.before_state_sha256) {
|
|
186
|
+
throw new Error("retry-grant transaction before state is not bound to its fence");
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function syncDir(path) {
|
|
191
|
+
let fd; try { fd = openSync(path, "r"); fsyncSync(fd); }
|
|
192
|
+
catch (error) { if (!["EINVAL", "EPERM", "EISDIR", "EACCES", "ENOTSUP"].includes(error?.code)) throw error; }
|
|
193
|
+
finally { if (fd !== undefined) closeSync(fd); }
|
|
194
|
+
}
|
package/state/session-lock.js
CHANGED
|
@@ -7,7 +7,7 @@ import { readFileSync } from "node:fs";
|
|
|
7
7
|
import { join } from "node:path";
|
|
8
8
|
import { writeProtectedJsonAtomic } from "../core/atomic-write.js";
|
|
9
9
|
import { rm } from "node:fs/promises";
|
|
10
|
-
import { withRunJsonLock } from "../core/run-lock.js";
|
|
10
|
+
import { RUN_JSON_LOCK_DIR, withRunJsonLock } from "../core/run-lock.js";
|
|
11
11
|
|
|
12
12
|
export const SESSION_LOCK_FILE = "factory.lock";
|
|
13
13
|
export const DEFAULT_SESSION_TTL_MS = 30 * 60 * 1000;
|
|
@@ -65,7 +65,7 @@ export async function claimSessionLock(runDir, { session, runId, branch, now, tt
|
|
|
65
65
|
claimed_at: observed.owner?.session === session ? observed.owner.claimed_at : at,
|
|
66
66
|
heartbeat_at: at,
|
|
67
67
|
};
|
|
68
|
-
await writeProtectedJsonAtomic(runDir, SESSION_LOCK_FILE, owner);
|
|
68
|
+
await writeProtectedJsonAtomic(runDir, SESSION_LOCK_FILE, owner, { tempDirectory: RUN_JSON_LOCK_DIR });
|
|
69
69
|
return { ...owner, stolen_from: observed.state === "stale" || force ? observed.owner : null };
|
|
70
70
|
});
|
|
71
71
|
}
|
|
@@ -77,7 +77,7 @@ export async function refreshSessionLock(runDir, { session, now } = {}) {
|
|
|
77
77
|
// Refreshing someone else's lock would silently extend a run you do not own.
|
|
78
78
|
if (session && owner.session !== session) throw new SessionLockHeldError(owner);
|
|
79
79
|
const next = { ...owner, heartbeat_at: new Date(now ?? Date.now()).toISOString() };
|
|
80
|
-
await writeProtectedJsonAtomic(runDir, SESSION_LOCK_FILE, next);
|
|
80
|
+
await writeProtectedJsonAtomic(runDir, SESSION_LOCK_FILE, next, { tempDirectory: RUN_JSON_LOCK_DIR });
|
|
81
81
|
return next;
|
|
82
82
|
});
|
|
83
83
|
}
|
package/state/transition.js
CHANGED
|
@@ -10,7 +10,7 @@ import { validateRun } from "./schema.js";
|
|
|
10
10
|
import { coordinateRunJsonTransition } from "../core/write-core.js";
|
|
11
11
|
import { FAMILY_CONTRACTS } from "../core/contracts.js";
|
|
12
12
|
|
|
13
|
-
export async function transition(runDir, { participants, apply, reobservers, hooks, finalGuard } = {}) {
|
|
13
|
+
export async function transition(runDir, { participants, apply, reobservers, hooks, finalGuard, commitFailureGuard, afterCommit } = {}) {
|
|
14
14
|
const descriptor = Object.freeze({
|
|
15
15
|
participants: Object.freeze((participants ?? []).map((entry) => Object.freeze({ ...entry }))),
|
|
16
16
|
apply,
|
|
@@ -22,5 +22,7 @@ export async function transition(runDir, { participants, apply, reobservers, hoo
|
|
|
22
22
|
reobservers: reobservers ?? new Map(),
|
|
23
23
|
atomicWriteHooks: hooks,
|
|
24
24
|
finalGuard,
|
|
25
|
+
commitFailureGuard,
|
|
26
|
+
afterCommit,
|
|
25
27
|
});
|
|
26
28
|
}
|