@cotal-ai/workspace 0.171.3 → 0.171.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extension-mutation.d.ts +2 -0
- package/dist/extension-mutation.d.ts.map +1 -1
- package/dist/extension-mutation.js +71 -10
- package/dist/extension-mutation.js.map +1 -1
- package/dist/extensions.d.ts.map +1 -1
- package/dist/extensions.js +14 -4
- package/dist/extensions.js.map +1 -1
- package/dist/maintenance-foundation.d.ts +16 -0
- package/dist/maintenance-foundation.d.ts.map +1 -1
- package/dist/maintenance-foundation.js +86 -1
- package/dist/maintenance-foundation.js.map +1 -1
- package/dist/maintenance-journal.d.ts +1 -0
- package/dist/maintenance-journal.d.ts.map +1 -1
- package/dist/maintenance-journal.js +11 -1
- package/dist/maintenance-journal.js.map +1 -1
- package/dist/maintenance-restore.d.ts +162 -0
- package/dist/maintenance-restore.d.ts.map +1 -0
- package/dist/maintenance-restore.js +912 -0
- package/dist/maintenance-restore.js.map +1 -0
- package/dist/maintenance.d.ts +3 -154
- package/dist/maintenance.d.ts.map +1 -1
- package/dist/maintenance.js +6 -1002
- package/dist/maintenance.js.map +1 -1
- package/dist/mesh-registry.d.ts +3 -0
- package/dist/mesh-registry.d.ts.map +1 -1
- package/dist/mesh-registry.js +3 -0
- package/dist/mesh-registry.js.map +1 -1
- package/dist/mesh-target.d.ts +3 -3
- package/dist/mesh-target.d.ts.map +1 -1
- package/dist/mesh-target.js +61 -6
- package/dist/mesh-target.js.map +1 -1
- package/dist/prepared-spaces.d.ts +1 -1
- package/dist/prepared-spaces.d.ts.map +1 -1
- package/dist/prepared-spaces.js +16 -3
- package/dist/prepared-spaces.js.map +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,912 @@
|
|
|
1
|
+
/** The restore lifecycle: restore contexts and claims, the restore listener and its custody, and the
|
|
2
|
+
* settling, activation and retirement of a restored root, driven by the maintenance operations in
|
|
3
|
+
* `maintenance.ts`. `maintenance.ts` re-exports every public name here, so importers keep their path. */
|
|
4
|
+
import { existsSync, lstatSync, readdirSync, readFileSync, renameSync, rmdirSync, rmSync, statSync, unlinkSync, } from "node:fs";
|
|
5
|
+
import { dirname, join, resolve } from "node:path";
|
|
6
|
+
import { randomUUID } from "node:crypto";
|
|
7
|
+
import { wfjStreamName } from "@cotal-ai/core";
|
|
8
|
+
import { foreignStoreEntries, STORE_ENTRY_NAMES, storeAccounts } from "./store-entries.js";
|
|
9
|
+
import { ATTEMPT_ID, STORE_ID_FILE, assertLock, assertPathsDoNotOverlap, assertStoreIdentity, canonicalAbsentPath, canonicalCandidatePath, defaultOwner, ensureStoreIdentity, errno, fsyncDirectory, identityAt, localProcessOwnerStatus, maintenanceError, maintenancePaths, pathContains, pathExistsStrict, readStoreIdentity, sameStoreIdentity, carriedTokenHandoff, removeOwnedPathsOrFail, requireSettledTokenHandoff, } from "./maintenance-foundation.js";
|
|
10
|
+
import { assertListenerOwnerAlive, canonicalListenerEndpoint, currentJournal, normalizedListenerProof, readMaintenanceResumeDocument, readyFrom, requireCustodyProof, restoreProofScope, reusesListenerIdentity, sameListenerProof, sameManagerCommitEvidence, sameManagerFinalizeEvidence, sameProcessOwner, validJsonObject, validManagerCommitEvidence, validManagerFinalizeEvidence, validOwnedPaths, validRecourse, validRestoreActivationEvidence, validRestoreClaim, writeJournal, requireReady, } from "./maintenance-journal.js";
|
|
11
|
+
function restoreBase(ready, attemptId, method, targetPath, selection) {
|
|
12
|
+
if (!ATTEMPT_ID.test(attemptId))
|
|
13
|
+
maintenanceError("invalid-transition", "restore attempt id must be a safe token", { root: ready.source.path, attemptId });
|
|
14
|
+
return { attemptId, method, targetPath, ...(selection ? { selection } : {}) };
|
|
15
|
+
}
|
|
16
|
+
function newRestoreClaim(input) {
|
|
17
|
+
if (!Number.isFinite(Date.parse(input.deadline)))
|
|
18
|
+
maintenanceError("invalid-transition", "restore claim requires an absolute deadline");
|
|
19
|
+
const claim = { deadline: input.deadline, coordinator: input.coordinator ?? defaultOwner(), owners: [] };
|
|
20
|
+
if (!validRestoreClaim(claim))
|
|
21
|
+
maintenanceError("invalid-transition", "restore claim coordinator is invalid");
|
|
22
|
+
if (input.ownedPaths !== undefined && !validOwnedPaths(input.ownedPaths))
|
|
23
|
+
maintenanceError("invalid-transition", "restore claim owned paths are invalid");
|
|
24
|
+
return { claim, ...(input.ownedPaths ? { ownedPaths: input.ownedPaths } : {}) };
|
|
25
|
+
}
|
|
26
|
+
/** Assess a maintenance claim (a restore's or a backup's). `live` while the deadline has not elapsed
|
|
27
|
+
* or any recorded owner is alive; `stale` once the deadline elapsed AND every owner is proven dead.
|
|
28
|
+
* Before the deadline it is `stale` only when the caller also proves that no process the attempt
|
|
29
|
+
* started outside the recorded owners is alive (`unrecordedOwnersDead`), and every recorded owner,
|
|
30
|
+
* coordinator included, is proven dead. */
|
|
31
|
+
export function assessMaintenanceClaim(claim, options = {}) {
|
|
32
|
+
const now = options.now ?? new Date();
|
|
33
|
+
const status = options.ownerStatus ?? localProcessOwnerStatus;
|
|
34
|
+
if (now.getTime() <= Date.parse(claim.deadline))
|
|
35
|
+
return options.unrecordedOwnersDead === true && [claim.coordinator, ...claim.owners].every((owner) => status(owner) === "dead") ? "stale" : "live";
|
|
36
|
+
const statuses = [claim.coordinator, ...claim.owners].map(status);
|
|
37
|
+
if (statuses.includes("alive"))
|
|
38
|
+
return "live";
|
|
39
|
+
if (statuses.includes("unknown"))
|
|
40
|
+
return "ambiguous";
|
|
41
|
+
return "stale";
|
|
42
|
+
}
|
|
43
|
+
/** Assess the recorded pre-commit restore claim; see {@link assessMaintenanceClaim}. */
|
|
44
|
+
export function assessRestoreClaim(record, options = {}) {
|
|
45
|
+
return assessMaintenanceClaim(record.claim, options);
|
|
46
|
+
}
|
|
47
|
+
export function requireRestoreReady(lock) {
|
|
48
|
+
const record = currentJournal(lock);
|
|
49
|
+
if (record.state !== "restore-ready")
|
|
50
|
+
maintenanceError("invalid-transition", "maintenance journal has no pre-commit restore attempt", { root: lock.root });
|
|
51
|
+
return record;
|
|
52
|
+
}
|
|
53
|
+
/** Append broker/watchdog owners and attempt-owned working trees to the live restore attempt so
|
|
54
|
+
* recovery can refuse while they live and delete exactly these inodes once they are proven dead. */
|
|
55
|
+
export function recordRestoreAttemptResources(lock, input) {
|
|
56
|
+
const record = requireRestoreReady(lock);
|
|
57
|
+
const owners = [...record.claim.owners, ...(input.owners ?? [])];
|
|
58
|
+
const claim = { ...record.claim, owners };
|
|
59
|
+
const ownedPaths = [...(record.restore.ownedPaths ?? [])];
|
|
60
|
+
for (const owned of input.ownedPaths ?? []) {
|
|
61
|
+
const existing = ownedPaths.findIndex((entry) => entry.path === owned.path && entry.label === owned.label);
|
|
62
|
+
if (existing >= 0)
|
|
63
|
+
ownedPaths.splice(existing, 1, owned);
|
|
64
|
+
else
|
|
65
|
+
ownedPaths.push(owned);
|
|
66
|
+
}
|
|
67
|
+
if (!validRestoreClaim(claim))
|
|
68
|
+
maintenanceError("invalid-transition", "restore claim owners are invalid", { attemptId: record.restore.attemptId });
|
|
69
|
+
if (!validOwnedPaths(ownedPaths))
|
|
70
|
+
maintenanceError("invalid-transition", "restore attempt-owned paths are invalid", { attemptId: record.restore.attemptId });
|
|
71
|
+
const restore = { ...record.restore, ...(ownedPaths.length ? { ownedPaths } : {}) };
|
|
72
|
+
return writeJournal(lock, record, {
|
|
73
|
+
...readyFrom(record), state: "restore-ready", phase: record.phase, restore, claim,
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
export function prepareSamePathRestore(lock, input) {
|
|
77
|
+
const ready = requireReady(lock);
|
|
78
|
+
const targetPath = resolve(input.targetPath);
|
|
79
|
+
if (targetPath !== ready.source.path)
|
|
80
|
+
maintenanceError("invalid-path", "same-path restore target must be the recorded source path", {
|
|
81
|
+
paths: [targetPath, ready.source.path],
|
|
82
|
+
});
|
|
83
|
+
const fallbackPath = canonicalAbsentPath(input.fallbackPath);
|
|
84
|
+
if (dirname(fallbackPath) !== dirname(ready.source.path))
|
|
85
|
+
maintenanceError("invalid-path", "same-path fallback must be a sibling of the source", {
|
|
86
|
+
paths: [ready.source.path, fallbackPath],
|
|
87
|
+
});
|
|
88
|
+
const parentDev = statSync(dirname(fallbackPath), { bigint: true }).dev.toString();
|
|
89
|
+
if (parentDev !== ready.source.dev)
|
|
90
|
+
maintenanceError("invalid-path", "same-path fallback must be on the source filesystem", {
|
|
91
|
+
paths: [ready.source.path, fallbackPath],
|
|
92
|
+
});
|
|
93
|
+
const { claim, ownedPaths } = newRestoreClaim(input.claim);
|
|
94
|
+
const restore = {
|
|
95
|
+
...restoreBase(ready, input.attemptId, "same-path", targetPath, input.selection), fallbackPath,
|
|
96
|
+
...(ownedPaths ? { ownedPaths } : {}),
|
|
97
|
+
};
|
|
98
|
+
return writeJournal(lock, ready, { ...readyFrom(ready), state: "restore-ready", phase: "move-pending", restore, claim });
|
|
99
|
+
}
|
|
100
|
+
export function interpretPendingStoreMove(record) {
|
|
101
|
+
if (record.phase !== "move-pending" || record.restore.method !== "same-path" || !record.restore.fallbackPath)
|
|
102
|
+
maintenanceError("invalid-transition", "record is not a pending same-path move", { attemptId: record.restore.attemptId });
|
|
103
|
+
const source = identityAt(record.source.path);
|
|
104
|
+
const fallback = identityAt(record.restore.fallbackPath);
|
|
105
|
+
const sourceMatches = source ? sameStoreIdentity(record.source, source) : false;
|
|
106
|
+
const expectedFallback = { ...record.source, path: record.restore.fallbackPath };
|
|
107
|
+
const fallbackMatches = fallback ? sameStoreIdentity(expectedFallback, fallback) : false;
|
|
108
|
+
if (sourceMatches && !fallback)
|
|
109
|
+
return "not-moved";
|
|
110
|
+
if (!source && fallbackMatches)
|
|
111
|
+
return "moved";
|
|
112
|
+
maintenanceError("ambiguous-filesystem-state", "same-path move cannot be interpreted safely", {
|
|
113
|
+
attemptId: record.restore.attemptId,
|
|
114
|
+
paths: [record.source.path, record.restore.fallbackPath],
|
|
115
|
+
expected: record.source,
|
|
116
|
+
actual: source ?? fallback,
|
|
117
|
+
recourse: [{ action: "inspect", description: "Preserve both paths; do not rename or delete either store.", paths: [record.source.path, record.restore.fallbackPath] }],
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Move the stopped source after interpreting either crash side. This is a cooperative filesystem
|
|
122
|
+
* transition: the caller must first prove every broker and other non-Cotal store user has exited.
|
|
123
|
+
* The maintenance lock excludes Cotal peers; it is not a shared-store lock for raw processes.
|
|
124
|
+
*/
|
|
125
|
+
export function moveSamePathRestoreSource(lock) {
|
|
126
|
+
const record = currentJournal(lock);
|
|
127
|
+
if (record.state !== "restore-ready" || record.phase !== "move-pending" ||
|
|
128
|
+
record.restore.method !== "same-path" || !record.restore.fallbackPath)
|
|
129
|
+
maintenanceError("invalid-transition", "maintenance journal is not awaiting a source move", { root: lock.root });
|
|
130
|
+
if (interpretPendingStoreMove(record) === "not-moved")
|
|
131
|
+
renameSync(record.source.path, record.restore.fallbackPath);
|
|
132
|
+
// Also fsync on crash recovery: the rename may have happened before its original parent fsync.
|
|
133
|
+
fsyncDirectory(dirname(record.source.path));
|
|
134
|
+
const fallback = readStoreIdentity(record.restore.fallbackPath);
|
|
135
|
+
const expected = { ...record.source, path: record.restore.fallbackPath };
|
|
136
|
+
if (!sameStoreIdentity(expected, fallback) || existsSync(record.source.path))
|
|
137
|
+
maintenanceError("ambiguous-filesystem-state", "source rename did not produce the recorded filesystem state", {
|
|
138
|
+
attemptId: record.restore.attemptId,
|
|
139
|
+
paths: [record.source.path, record.restore.fallbackPath], expected, actual: fallback,
|
|
140
|
+
});
|
|
141
|
+
const restore = { ...record.restore, previousSource: { kind: "fallback", identity: fallback } };
|
|
142
|
+
return writeJournal(lock, record, { ...readyFrom(record), state: "restore-ready", phase: "source-moved", restore, claim: record.claim });
|
|
143
|
+
}
|
|
144
|
+
export function prepareAlternateRestore(lock, input) {
|
|
145
|
+
const ready = requireReady(lock);
|
|
146
|
+
const targetPath = canonicalCandidatePath(input.targetPath);
|
|
147
|
+
assertPathsDoNotOverlap(ready.source.path, targetPath, "alternate restore target must not equal, contain, or be nested under the retained source");
|
|
148
|
+
if (pathExistsStrict(targetPath))
|
|
149
|
+
maintenanceError("path-exists", "store path must be absent", {
|
|
150
|
+
paths: [targetPath],
|
|
151
|
+
recourse: [{ action: "inspect", description: "Inspect the unexpected path before retrying.", paths: [targetPath] }],
|
|
152
|
+
});
|
|
153
|
+
const { claim, ownedPaths } = newRestoreClaim(input.claim);
|
|
154
|
+
const restore = {
|
|
155
|
+
...restoreBase(ready, input.attemptId, "alternate", targetPath, input.selection),
|
|
156
|
+
previousSource: { kind: "retained", identity: ready.source },
|
|
157
|
+
...(ownedPaths ? { ownedPaths } : {}),
|
|
158
|
+
};
|
|
159
|
+
return writeJournal(lock, ready, { ...readyFrom(ready), state: "restore-ready", phase: "source-retained", restore, claim });
|
|
160
|
+
}
|
|
161
|
+
export function prepareMissingSourceRestore(lock, input) {
|
|
162
|
+
assertLock(lock);
|
|
163
|
+
const record = currentJournal(lock);
|
|
164
|
+
if (record.state !== "ready")
|
|
165
|
+
maintenanceError("invalid-transition", "missing-source consent requires a ready journal", { root: lock.root });
|
|
166
|
+
const existing = identityAt(record.source.path);
|
|
167
|
+
if (existing)
|
|
168
|
+
maintenanceError("identity-mismatch", "missing-source consent cannot accept an existing or replacement inode", {
|
|
169
|
+
expected: record.source, actual: existing, paths: [record.source.path],
|
|
170
|
+
});
|
|
171
|
+
const targetPath = canonicalAbsentPath(input.targetPath);
|
|
172
|
+
if (targetPath !== record.source.path)
|
|
173
|
+
maintenanceError("invalid-path", "disaster restore target must be the missing canonical source path", {
|
|
174
|
+
paths: [targetPath, record.source.path],
|
|
175
|
+
});
|
|
176
|
+
const { claim, ownedPaths } = newRestoreClaim(input.claim);
|
|
177
|
+
const restore = {
|
|
178
|
+
...restoreBase(record, input.attemptId, "disaster", targetPath, input.selection),
|
|
179
|
+
...(ownedPaths ? { ownedPaths } : {}),
|
|
180
|
+
};
|
|
181
|
+
return writeJournal(lock, record, { ...readyFrom(record), state: "restore-ready", phase: "disaster-source-missing", restore, claim });
|
|
182
|
+
}
|
|
183
|
+
export function bindRestoreTarget(lock) {
|
|
184
|
+
const record = currentJournal(lock);
|
|
185
|
+
if (record.state !== "restore-ready")
|
|
186
|
+
maintenanceError("invalid-transition", "restore target can be bound only before commit intent", { root: lock.root });
|
|
187
|
+
if (record.restore.target)
|
|
188
|
+
maintenanceError("invalid-transition", "restore target is already bound", { attemptId: record.restore.attemptId });
|
|
189
|
+
if (record.restore.method === "same-path" && record.phase !== "source-moved")
|
|
190
|
+
maintenanceError("invalid-transition", "same-path target cannot be bound before the source move", {
|
|
191
|
+
attemptId: record.restore.attemptId,
|
|
192
|
+
});
|
|
193
|
+
const target = ensureStoreIdentity(record.restore.targetPath);
|
|
194
|
+
if (record.restore.method === "alternate")
|
|
195
|
+
assertPathsDoNotOverlap(record.source.path, target.path, "alternate restore target must remain disjoint from the retained source");
|
|
196
|
+
if (record.restore.previousSource &&
|
|
197
|
+
target.dev === record.restore.previousSource.identity.dev && target.ino === record.restore.previousSource.identity.ino)
|
|
198
|
+
maintenanceError("identity-mismatch", "restore target aliases the old source inode", {
|
|
199
|
+
attemptId: record.restore.attemptId,
|
|
200
|
+
expected: record.restore.previousSource.identity,
|
|
201
|
+
actual: target,
|
|
202
|
+
paths: [target.path, record.restore.previousSource.identity.path],
|
|
203
|
+
});
|
|
204
|
+
const restore = { ...record.restore, target };
|
|
205
|
+
return writeJournal(lock, record, { ...readyFrom(record), state: "restore-ready", phase: record.phase, restore, claim: record.claim });
|
|
206
|
+
}
|
|
207
|
+
export function jsonObject(value) {
|
|
208
|
+
if (!validJsonObject(value))
|
|
209
|
+
maintenanceError("invalid-transition", "restore metadata must be finite JSON data");
|
|
210
|
+
return JSON.parse(JSON.stringify(value));
|
|
211
|
+
}
|
|
212
|
+
function preflightRollbackSource(record) {
|
|
213
|
+
if (record.restore.method === "same-path") {
|
|
214
|
+
if (record.phase === "move-pending") {
|
|
215
|
+
interpretPendingStoreMove(record);
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
if (!record.restore.previousSource)
|
|
219
|
+
maintenanceError("journal-corrupt", "moved restore has no previous source", { attemptId: record.restore.attemptId });
|
|
220
|
+
assertStoreIdentity(record.restore.previousSource.identity);
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
if (record.restore.method === "alternate") {
|
|
224
|
+
assertStoreIdentity(record.source);
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
if (identityAt(record.source.path))
|
|
228
|
+
maintenanceError("ambiguous-filesystem-state", "missing-source rollback found an unexpected replacement", {
|
|
229
|
+
attemptId: record.restore.attemptId, paths: [record.source.path], expected: record.source,
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
export function writeRestoreCommitIntent(lock, launch) {
|
|
233
|
+
const record = currentJournal(lock);
|
|
234
|
+
if (record.state !== "restore-ready" || !record.restore.target)
|
|
235
|
+
maintenanceError("invalid-transition", "commit intent requires a bound pre-commit target", { root: lock.root });
|
|
236
|
+
assertStoreIdentity(record.restore.target);
|
|
237
|
+
if (record.restore.previousSource) {
|
|
238
|
+
const previous = assertStoreIdentity(record.restore.previousSource.identity);
|
|
239
|
+
if (previous.dev === record.restore.target.dev && previous.ino === record.restore.target.ino)
|
|
240
|
+
maintenanceError("identity-mismatch", "old source aliases the restore target at commit intent", {
|
|
241
|
+
attemptId: record.restore.attemptId,
|
|
242
|
+
expected: record.restore.previousSource.identity,
|
|
243
|
+
actual: record.restore.target,
|
|
244
|
+
paths: [previous.path, record.restore.target.path],
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
return writeJournal(lock, record, {
|
|
248
|
+
...readyFrom(record), state: "commit-intent", restore: record.restore,
|
|
249
|
+
launch: jsonObject(launch),
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
/** Fsync the exact normal listener identity after spawn and before activation is published. */
|
|
253
|
+
export function bindRestoreListener(lock, proof, options = {}) {
|
|
254
|
+
const record = currentJournal(lock);
|
|
255
|
+
if (record.state !== "commit-intent")
|
|
256
|
+
maintenanceError("invalid-transition", "restore listener can be bound only from commit intent", {
|
|
257
|
+
root: lock.root,
|
|
258
|
+
});
|
|
259
|
+
const normalized = normalizedListenerProof(proof);
|
|
260
|
+
requireCustodyProof(normalized, record.cut, restoreProofScope(record), options.controllerInstanceRunning, lock.root, record.restore.attemptId);
|
|
261
|
+
if (normalized.attemptId !== record.restore.attemptId ||
|
|
262
|
+
!sameStoreIdentity(normalized.target, record.restore.target))
|
|
263
|
+
maintenanceError("listener-proof-mismatch", "restore listener proof does not match the committed attempt and target", {
|
|
264
|
+
root: lock.root, attemptId: record.restore.attemptId,
|
|
265
|
+
paths: [record.restore.target.path], expected: record.restore.target, actual: normalized.target,
|
|
266
|
+
});
|
|
267
|
+
const launchServer = record.launch.server;
|
|
268
|
+
if (typeof launchServer !== "string" || canonicalListenerEndpoint(launchServer) !== normalized.serverEndpoint)
|
|
269
|
+
maintenanceError("listener-proof-mismatch", "restore listener endpoint does not match committed launch provenance", {
|
|
270
|
+
root: lock.root, attemptId: record.restore.attemptId, paths: [record.restore.target.path],
|
|
271
|
+
});
|
|
272
|
+
assertStoreIdentity(record.restore.target);
|
|
273
|
+
if (record.listenerReplacements?.some((replacement) => reusesListenerIdentity(replacement.proof, normalized)))
|
|
274
|
+
maintenanceError("listener-proof-mismatch", "restore listener proof reuses a retired listener identity", {
|
|
275
|
+
root: lock.root, attemptId: record.restore.attemptId, paths: [record.restore.target.path],
|
|
276
|
+
});
|
|
277
|
+
assertListenerOwnerAlive(normalized);
|
|
278
|
+
if (record.listenerProof) {
|
|
279
|
+
if (!sameListenerProof(record.listenerProof, normalized))
|
|
280
|
+
maintenanceError("listener-proof-mismatch", "restore commit intent is already bound to another listener", {
|
|
281
|
+
root: lock.root, attemptId: record.restore.attemptId, paths: [record.restore.target.path],
|
|
282
|
+
});
|
|
283
|
+
return record;
|
|
284
|
+
}
|
|
285
|
+
return writeJournal(lock, record, {
|
|
286
|
+
...readyFrom(record), state: "commit-intent", restore: record.restore,
|
|
287
|
+
launch: record.launch, listenerProof: normalized,
|
|
288
|
+
...(record.listenerReplacements ? { listenerReplacements: record.listenerReplacements } : {}),
|
|
289
|
+
...carriedTokenHandoff(record),
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
/**
|
|
293
|
+
* Retire one exactly bound, provably dead listener without changing the restore attempt. The
|
|
294
|
+
* atomic journal transition clears the current proof before another listener may be spawned.
|
|
295
|
+
*/
|
|
296
|
+
export function replaceDeadRestoreListener(lock, proof, options = {}) {
|
|
297
|
+
const record = currentJournal(lock);
|
|
298
|
+
if (record.state !== "commit-intent" && record.state !== "degraded")
|
|
299
|
+
maintenanceError("invalid-transition", "dead restore listener replacement requires uncommitted commit intent or degraded state", {
|
|
300
|
+
root: lock.root,
|
|
301
|
+
});
|
|
302
|
+
if (record.state === "degraded" &&
|
|
303
|
+
(record.managerCommittedAt !== undefined || record.managerCommit !== undefined || record.activeAt !== undefined))
|
|
304
|
+
maintenanceError("invalid-transition", "manager-committed restore listener cannot be replaced", {
|
|
305
|
+
root: lock.root, attemptId: record.restore.attemptId,
|
|
306
|
+
});
|
|
307
|
+
if (!record.listenerProof)
|
|
308
|
+
maintenanceError("listener-proof-missing", "restore listener replacement requires an existing bound proof", {
|
|
309
|
+
root: lock.root, attemptId: record.restore.attemptId, paths: [record.restore.target.path],
|
|
310
|
+
});
|
|
311
|
+
const normalized = normalizedListenerProof(proof);
|
|
312
|
+
if (!sameListenerProof(record.listenerProof, normalized))
|
|
313
|
+
maintenanceError("listener-proof-mismatch", "restore listener replacement does not exactly match the bound proof", {
|
|
314
|
+
root: lock.root, attemptId: record.restore.attemptId,
|
|
315
|
+
paths: [record.restore.target.path], expected: record.restore.target, actual: normalized.target,
|
|
316
|
+
});
|
|
317
|
+
assertStoreIdentity(record.restore.target);
|
|
318
|
+
const status = (options.ownerStatus ?? localProcessOwnerStatus)(record.listenerProof.processOwner);
|
|
319
|
+
if (status === "alive")
|
|
320
|
+
maintenanceError("listener-owner-alive", "bound restore listener is still alive", {
|
|
321
|
+
root: lock.root, attemptId: record.restore.attemptId,
|
|
322
|
+
recourse: [{ action: "retry", description: "Stop the exact bound listener and prove process exit before replacement." }],
|
|
323
|
+
});
|
|
324
|
+
if (status !== "dead")
|
|
325
|
+
maintenanceError("listener-owner-ambiguous", "bound restore listener death cannot be proven", {
|
|
326
|
+
root: lock.root, attemptId: record.restore.attemptId,
|
|
327
|
+
recourse: [{ action: "inspect", description: "Prove the exact recorded listener process dead before replacement." }],
|
|
328
|
+
});
|
|
329
|
+
const history = record.listenerReplacements ?? [];
|
|
330
|
+
const replacement = {
|
|
331
|
+
generation: history.length + 1,
|
|
332
|
+
replacedAt: new Date().toISOString(),
|
|
333
|
+
proof: record.listenerProof,
|
|
334
|
+
};
|
|
335
|
+
return writeJournal(lock, record, {
|
|
336
|
+
...readyFrom(record), state: "commit-intent", restore: record.restore, launch: record.launch,
|
|
337
|
+
listenerReplacements: [...history, replacement],
|
|
338
|
+
...carriedTokenHandoff(record),
|
|
339
|
+
});
|
|
340
|
+
}
|
|
341
|
+
/**
|
|
342
|
+
* Return a degraded restore that has no bound listener to commit intent, so its next listener can
|
|
343
|
+
* start and bind (#7127). A bound listener is retired through {@link replaceDeadRestoreListener}, and
|
|
344
|
+
* a restore past manager commitment never returns here. The caller first proves that no listener
|
|
345
|
+
* this attempt spawned still runs. A commit intent with no bound listener is returned unchanged.
|
|
346
|
+
*/
|
|
347
|
+
export function rearmUnboundRestoreCommitIntent(lock) {
|
|
348
|
+
const record = currentJournal(lock);
|
|
349
|
+
if (record.state === "commit-intent" && !record.listenerProof)
|
|
350
|
+
return record;
|
|
351
|
+
if (record.state !== "degraded" || record.listenerProof)
|
|
352
|
+
maintenanceError("invalid-transition", "restore commit intent is re-armed only from a degraded restore with no bound listener", {
|
|
353
|
+
root: lock.root,
|
|
354
|
+
});
|
|
355
|
+
if (record.managerCommittedAt !== undefined || record.managerCommit !== undefined || record.activeAt !== undefined)
|
|
356
|
+
maintenanceError("invalid-transition", "manager-committed restore cannot return to commit intent", {
|
|
357
|
+
root: lock.root, attemptId: record.restore.attemptId,
|
|
358
|
+
});
|
|
359
|
+
assertStoreIdentity(record.restore.target);
|
|
360
|
+
return writeJournal(lock, record, {
|
|
361
|
+
...readyFrom(record), state: "commit-intent", restore: record.restore, launch: record.launch,
|
|
362
|
+
...(record.listenerReplacements ? { listenerReplacements: record.listenerReplacements } : {}),
|
|
363
|
+
...carriedTokenHandoff(record),
|
|
364
|
+
});
|
|
365
|
+
}
|
|
366
|
+
/** Fsync the manager's suppression-retaining commit token before requesting finalization. */
|
|
367
|
+
export function recordRestoreManagerCommit(lock, proof, evidence) {
|
|
368
|
+
const record = currentJournal(lock);
|
|
369
|
+
if (record.state === "manager-committed") {
|
|
370
|
+
const normalized = normalizedListenerProof(proof);
|
|
371
|
+
if (!sameListenerProof(record.listenerProof, normalized))
|
|
372
|
+
maintenanceError("listener-proof-mismatch", "restore commit retry does not match the durable listener proof", {
|
|
373
|
+
root: lock.root, attemptId: record.restore.attemptId, paths: [record.restore.target.path],
|
|
374
|
+
});
|
|
375
|
+
if (!validManagerCommitEvidence(evidence, record.restore.attemptId) ||
|
|
376
|
+
!sameManagerCommitEvidence(record.managerCommit, evidence))
|
|
377
|
+
maintenanceError("activation-evidence-invalid", "restore commit retry does not match the durable manager token", {
|
|
378
|
+
root: lock.root, attemptId: record.restore.attemptId, paths: [record.restore.target.path],
|
|
379
|
+
});
|
|
380
|
+
assertStoreIdentity(record.restore.target);
|
|
381
|
+
return record;
|
|
382
|
+
}
|
|
383
|
+
if (record.state !== "commit-intent" && record.state !== "degraded")
|
|
384
|
+
maintenanceError("invalid-transition", "restore manager commit requires commit-intent or uncommitted degraded state", {
|
|
385
|
+
root: lock.root,
|
|
386
|
+
});
|
|
387
|
+
if (record.state === "degraded" &&
|
|
388
|
+
(record.managerCommittedAt !== undefined || record.managerCommit !== undefined || record.activeAt !== undefined))
|
|
389
|
+
maintenanceError("invalid-transition", "restore manager commitment is already durable", {
|
|
390
|
+
root: lock.root, attemptId: record.restore.attemptId,
|
|
391
|
+
});
|
|
392
|
+
if (!record.listenerProof)
|
|
393
|
+
maintenanceError("listener-proof-missing", "restore manager commit requires a durably bound listener proof", {
|
|
394
|
+
root: lock.root, attemptId: record.restore.attemptId, paths: [record.restore.target.path],
|
|
395
|
+
});
|
|
396
|
+
const normalized = normalizedListenerProof(proof);
|
|
397
|
+
if (!sameListenerProof(record.listenerProof, normalized))
|
|
398
|
+
maintenanceError("listener-proof-mismatch", "restore manager commit proof does not exactly match the bound listener", {
|
|
399
|
+
root: lock.root, attemptId: record.restore.attemptId, paths: [record.restore.target.path],
|
|
400
|
+
});
|
|
401
|
+
if (!validManagerCommitEvidence(evidence, record.restore.attemptId))
|
|
402
|
+
maintenanceError("activation-evidence-invalid", "restore manager commit evidence is invalid or belongs to another attempt", {
|
|
403
|
+
root: lock.root, attemptId: record.restore.attemptId, paths: [record.restore.target.path],
|
|
404
|
+
});
|
|
405
|
+
assertStoreIdentity(record.restore.target);
|
|
406
|
+
assertListenerOwnerAlive(record.listenerProof);
|
|
407
|
+
return writeJournal(lock, record, {
|
|
408
|
+
...readyFrom(record), state: "manager-committed", restore: record.restore,
|
|
409
|
+
launch: record.launch, listenerProof: record.listenerProof,
|
|
410
|
+
...(record.listenerReplacements ? { listenerReplacements: record.listenerReplacements } : {}),
|
|
411
|
+
...carriedTokenHandoff(record),
|
|
412
|
+
managerCommittedAt: new Date().toISOString(), managerCommit: evidence,
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
export function markRestoreActive(lock, proof, evidence) {
|
|
416
|
+
const record = currentJournal(lock);
|
|
417
|
+
const normalizedProof = normalizedListenerProof(proof);
|
|
418
|
+
if (record.state === "active") {
|
|
419
|
+
if (!record.listenerProof || !sameListenerProof(record.listenerProof, normalizedProof))
|
|
420
|
+
maintenanceError("listener-proof-mismatch", "active restore retry does not match the durable listener proof", {
|
|
421
|
+
root: lock.root, attemptId: record.restore.attemptId, paths: [record.restore.target.path],
|
|
422
|
+
});
|
|
423
|
+
if (!validManagerFinalizeEvidence(evidence, record.restore.attemptId, record.details.managerCommit.durableCommitToken) || !sameManagerFinalizeEvidence(record.details.managerFinalize, evidence))
|
|
424
|
+
maintenanceError("activation-evidence-invalid", "active restore retry does not match the durable manager finalize evidence", {
|
|
425
|
+
root: lock.root, attemptId: record.restore.attemptId, paths: [record.restore.target.path],
|
|
426
|
+
});
|
|
427
|
+
assertStoreIdentity(record.restore.target);
|
|
428
|
+
return record;
|
|
429
|
+
}
|
|
430
|
+
if (record.state !== "manager-committed")
|
|
431
|
+
maintenanceError("invalid-transition", "active restore requires manager-committed state", { root: lock.root });
|
|
432
|
+
if (!sameListenerProof(record.listenerProof, normalizedProof))
|
|
433
|
+
maintenanceError("listener-proof-mismatch", "active restore proof does not exactly match the bound listener", {
|
|
434
|
+
root: lock.root, attemptId: record.restore.attemptId, paths: [record.restore.target.path],
|
|
435
|
+
});
|
|
436
|
+
if (!validManagerFinalizeEvidence(evidence, record.restore.attemptId, record.managerCommit.durableCommitToken))
|
|
437
|
+
maintenanceError("activation-evidence-invalid", "active restore requires exact token-bound manager finalize evidence", {
|
|
438
|
+
root: lock.root, attemptId: record.restore.attemptId, paths: [record.restore.target.path],
|
|
439
|
+
});
|
|
440
|
+
assertStoreIdentity(record.restore.target);
|
|
441
|
+
assertListenerOwnerAlive(record.listenerProof);
|
|
442
|
+
requireSettledTokenHandoff(record, lock.root);
|
|
443
|
+
const details = {
|
|
444
|
+
attemptId: record.restore.attemptId,
|
|
445
|
+
listenerReady: true,
|
|
446
|
+
observedAt: new Date().toISOString(),
|
|
447
|
+
managerCommit: record.managerCommit,
|
|
448
|
+
managerFinalize: evidence,
|
|
449
|
+
};
|
|
450
|
+
return writeJournal(lock, record, {
|
|
451
|
+
...readyFrom(record), state: "active", restore: record.restore,
|
|
452
|
+
launch: record.launch, listenerProof: record.listenerProof,
|
|
453
|
+
...(record.listenerReplacements ? { listenerReplacements: record.listenerReplacements } : {}),
|
|
454
|
+
...carriedTokenHandoff(record),
|
|
455
|
+
managerCommittedAt: record.managerCommittedAt, managerCommit: record.managerCommit,
|
|
456
|
+
activeAt: new Date().toISOString(), details,
|
|
457
|
+
});
|
|
458
|
+
}
|
|
459
|
+
export function markRestoreDegraded(lock, reason, recourse) {
|
|
460
|
+
const record = currentJournal(lock);
|
|
461
|
+
if (record.state !== "commit-intent" && record.state !== "manager-committed" && record.state !== "active")
|
|
462
|
+
maintenanceError("invalid-transition", "degraded restore requires commit intent, manager-committed, or active state", { root: lock.root });
|
|
463
|
+
if (!reason || !validRecourse(recourse))
|
|
464
|
+
maintenanceError("invalid-transition", "degraded restore requires a reason and structured recourse", { root: lock.root });
|
|
465
|
+
return writeJournal(lock, record, {
|
|
466
|
+
...readyFrom(record), state: "degraded", restore: record.restore,
|
|
467
|
+
launch: record.launch, ...(record.listenerProof ? { listenerProof: record.listenerProof } : {}),
|
|
468
|
+
...(record.listenerReplacements ? { listenerReplacements: record.listenerReplacements } : {}),
|
|
469
|
+
...carriedTokenHandoff(record),
|
|
470
|
+
...(record.state === "manager-committed"
|
|
471
|
+
? { managerCommittedAt: record.managerCommittedAt, managerCommit: record.managerCommit }
|
|
472
|
+
: {}),
|
|
473
|
+
...(record.state === "active"
|
|
474
|
+
? {
|
|
475
|
+
managerCommittedAt: record.managerCommittedAt,
|
|
476
|
+
managerCommit: record.managerCommit,
|
|
477
|
+
details: record.details,
|
|
478
|
+
activeAt: record.activeAt,
|
|
479
|
+
}
|
|
480
|
+
: {}),
|
|
481
|
+
degradedAt: new Date().toISOString(), reason, recourse: [...recourse],
|
|
482
|
+
});
|
|
483
|
+
}
|
|
484
|
+
/** Recover forward only by re-presenting the exact bound listener plus fresh readiness evidence. */
|
|
485
|
+
export function repairRestoreDegradedToActive(lock, proof, evidence) {
|
|
486
|
+
const record = currentJournal(lock);
|
|
487
|
+
if (record.state !== "degraded")
|
|
488
|
+
maintenanceError("invalid-transition", "restore repair requires degraded state", { root: lock.root });
|
|
489
|
+
if (!record.listenerProof)
|
|
490
|
+
maintenanceError("listener-proof-missing", "degraded restore has no bound listener proof", {
|
|
491
|
+
root: lock.root, attemptId: record.restore.attemptId,
|
|
492
|
+
paths: [record.restore.target.path, record.restore.previousSource?.identity.path]
|
|
493
|
+
.filter((path) => Boolean(path)),
|
|
494
|
+
recourse: [{ action: "inspect", description: "Preserve both stores; listener ownership was never durably bound." }],
|
|
495
|
+
});
|
|
496
|
+
const normalized = normalizedListenerProof(proof);
|
|
497
|
+
if (!sameListenerProof(record.listenerProof, normalized))
|
|
498
|
+
maintenanceError("listener-proof-mismatch", "restore repair proof does not exactly match the bound listener", {
|
|
499
|
+
root: lock.root, attemptId: record.restore.attemptId,
|
|
500
|
+
paths: [record.restore.target.path, record.restore.previousSource?.identity.path]
|
|
501
|
+
.filter((path) => Boolean(path)),
|
|
502
|
+
});
|
|
503
|
+
assertStoreIdentity(record.restore.target);
|
|
504
|
+
assertListenerOwnerAlive(record.listenerProof);
|
|
505
|
+
requireSettledTokenHandoff(record, lock.root);
|
|
506
|
+
if (!record.managerCommit || !record.managerCommittedAt)
|
|
507
|
+
maintenanceError("invalid-transition", "uncommitted degraded restore must record manager commitment before finalization", {
|
|
508
|
+
root: lock.root, attemptId: record.restore.attemptId, paths: [record.restore.target.path],
|
|
509
|
+
});
|
|
510
|
+
if (!validManagerFinalizeEvidence(evidence, record.restore.attemptId, record.managerCommit.durableCommitToken))
|
|
511
|
+
maintenanceError("activation-evidence-invalid", "restore repair requires exact token-bound manager finalize evidence", {
|
|
512
|
+
root: lock.root, attemptId: record.restore.attemptId, paths: [record.restore.target.path],
|
|
513
|
+
});
|
|
514
|
+
const details = {
|
|
515
|
+
attemptId: record.restore.attemptId,
|
|
516
|
+
listenerReady: true,
|
|
517
|
+
observedAt: new Date().toISOString(),
|
|
518
|
+
managerCommit: record.managerCommit,
|
|
519
|
+
managerFinalize: evidence,
|
|
520
|
+
};
|
|
521
|
+
return writeJournal(lock, record, {
|
|
522
|
+
...readyFrom(record), state: "active", restore: record.restore, launch: record.launch,
|
|
523
|
+
listenerProof: record.listenerProof,
|
|
524
|
+
managerCommittedAt: record.managerCommittedAt, managerCommit: record.managerCommit,
|
|
525
|
+
activeAt: new Date().toISOString(), details,
|
|
526
|
+
...(record.listenerReplacements ? { listenerReplacements: record.listenerReplacements } : {}),
|
|
527
|
+
...carriedTokenHandoff(record),
|
|
528
|
+
});
|
|
529
|
+
}
|
|
530
|
+
function cleanupTomb(record, kind, identity) {
|
|
531
|
+
return join(dirname(identity.path), `.cotal-clean-${record.restore.attemptId}-${kind}-${randomUUID()}`);
|
|
532
|
+
}
|
|
533
|
+
function startCleanup(lock, record, kind, identity) {
|
|
534
|
+
const cleanup = {
|
|
535
|
+
kind, status: "pending", originalPath: identity.path,
|
|
536
|
+
tombPath: cleanupTomb(record, kind, identity), identity,
|
|
537
|
+
};
|
|
538
|
+
const restore = { ...record.restore, cleanup };
|
|
539
|
+
if (record.state === "restore-ready")
|
|
540
|
+
return writeJournal(lock, record, { ...readyFrom(record), state: "restore-ready", phase: record.phase, restore, claim: record.claim });
|
|
541
|
+
return writeJournal(lock, record, {
|
|
542
|
+
...readyFrom(record), state: "active", restore: restore,
|
|
543
|
+
launch: record.launch, ...(record.listenerProof ? { listenerProof: record.listenerProof } : {}),
|
|
544
|
+
...(record.listenerReplacements ? { listenerReplacements: record.listenerReplacements } : {}),
|
|
545
|
+
...carriedTokenHandoff(record),
|
|
546
|
+
managerCommittedAt: record.managerCommittedAt, managerCommit: record.managerCommit,
|
|
547
|
+
activeAt: record.activeAt, details: record.details,
|
|
548
|
+
});
|
|
549
|
+
}
|
|
550
|
+
function continueCleanup(lock, record) {
|
|
551
|
+
const cleanup = record.restore.cleanup;
|
|
552
|
+
if (!cleanup)
|
|
553
|
+
maintenanceError("invalid-transition", "no cleanup is recorded", { attemptId: record.restore.attemptId });
|
|
554
|
+
if (cleanup.status === "complete")
|
|
555
|
+
return record;
|
|
556
|
+
if (record.state === "active") {
|
|
557
|
+
assertStoreIdentity(record.restore.target);
|
|
558
|
+
if (pathContains(cleanup.originalPath, record.restore.target.path) ||
|
|
559
|
+
pathContains(cleanup.tombPath, record.restore.target.path))
|
|
560
|
+
maintenanceError("cleanup-forbidden", "recursive cleanup tree contains the active target", {
|
|
561
|
+
root: lock.root, attemptId: record.restore.attemptId,
|
|
562
|
+
paths: [cleanup.originalPath, cleanup.tombPath, record.restore.target.path],
|
|
563
|
+
});
|
|
564
|
+
}
|
|
565
|
+
const original = identityAt(cleanup.originalPath);
|
|
566
|
+
const tomb = identityAt(cleanup.tombPath);
|
|
567
|
+
const expectedOriginal = cleanup.identity;
|
|
568
|
+
const expectedTomb = { ...cleanup.identity, path: cleanup.tombPath };
|
|
569
|
+
if (original && sameStoreIdentity(expectedOriginal, original) && !tomb) {
|
|
570
|
+
renameSync(cleanup.originalPath, cleanup.tombPath);
|
|
571
|
+
fsyncDirectory(dirname(cleanup.originalPath));
|
|
572
|
+
}
|
|
573
|
+
else if (!original && tomb && sameStoreIdentity(expectedTomb, tomb)) {
|
|
574
|
+
// Crash after rename: continue with the recorded inode.
|
|
575
|
+
}
|
|
576
|
+
else if (!original && !tomb) {
|
|
577
|
+
// Crash after removal but before the final journal write: the requested cleanup is complete.
|
|
578
|
+
}
|
|
579
|
+
else {
|
|
580
|
+
maintenanceError("ambiguous-filesystem-state", "recorded cleanup paths do not match the journal", {
|
|
581
|
+
attemptId: record.restore.attemptId,
|
|
582
|
+
paths: [cleanup.originalPath, cleanup.tombPath], expected: cleanup.identity, actual: original ?? tomb,
|
|
583
|
+
recourse: [{ action: "inspect", description: "Preserve every matching path and repair the journal explicitly.", paths: [cleanup.originalPath, cleanup.tombPath] }],
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
const moved = identityAt(cleanup.tombPath);
|
|
587
|
+
if (moved) {
|
|
588
|
+
if (!sameStoreIdentity(expectedTomb, moved))
|
|
589
|
+
maintenanceError("identity-mismatch", "cleanup tomb inode changed", {
|
|
590
|
+
attemptId: record.restore.attemptId, expected: expectedTomb, actual: moved, paths: [cleanup.tombPath],
|
|
591
|
+
});
|
|
592
|
+
if (record.state === "active")
|
|
593
|
+
assertStoreIdentity(record.restore.target);
|
|
594
|
+
if (cleanup.kind === "previous-source")
|
|
595
|
+
removeStoreTomb(lock, record, cleanup);
|
|
596
|
+
else
|
|
597
|
+
rmSync(cleanup.tombPath, { recursive: true });
|
|
598
|
+
fsyncDirectory(dirname(cleanup.tombPath));
|
|
599
|
+
}
|
|
600
|
+
const restore = { ...record.restore, cleanup: { ...cleanup, status: "complete" } };
|
|
601
|
+
if (record.state === "restore-ready")
|
|
602
|
+
return writeJournal(lock, record, { ...readyFrom(record), state: "restore-ready", phase: record.phase, restore, claim: record.claim });
|
|
603
|
+
return writeJournal(lock, record, {
|
|
604
|
+
...readyFrom(record), state: "active", restore: restore,
|
|
605
|
+
launch: record.launch, ...(record.listenerProof ? { listenerProof: record.listenerProof } : {}),
|
|
606
|
+
...(record.listenerReplacements ? { listenerReplacements: record.listenerReplacements } : {}),
|
|
607
|
+
...carriedTokenHandoff(record),
|
|
608
|
+
managerCommittedAt: record.managerCommittedAt, managerCommit: record.managerCommit,
|
|
609
|
+
activeAt: record.activeAt, details: record.details,
|
|
610
|
+
});
|
|
611
|
+
}
|
|
612
|
+
/** Remove an old store's tomb by its store entries, then the folder once it is empty. An entry no
|
|
613
|
+
* Cotal store writes, there before the removal or put there since, moves the folder back to where it
|
|
614
|
+
* was and refuses, so the old source loses nothing but its store. */
|
|
615
|
+
function removeStoreTomb(lock, record, cleanup) {
|
|
616
|
+
const refuse = (entries) => {
|
|
617
|
+
const back = !existsSync(cleanup.originalPath);
|
|
618
|
+
if (back) {
|
|
619
|
+
renameSync(cleanup.tombPath, cleanup.originalPath);
|
|
620
|
+
fsyncDirectory(dirname(cleanup.originalPath));
|
|
621
|
+
}
|
|
622
|
+
const path = back ? cleanup.originalPath : cleanup.tombPath;
|
|
623
|
+
return maintenanceError("store-holds-foreign-entries", `the restore's old source ${path} also holds ${entries.join(", ")}, which no Cotal store writes`, {
|
|
624
|
+
root: lock.root, attemptId: record.restore.attemptId, paths: [path],
|
|
625
|
+
recourse: [{ action: "inspect", description: "Move those entries out of the old source, then run the cleanup again.", paths: [path] }],
|
|
626
|
+
});
|
|
627
|
+
};
|
|
628
|
+
const foreign = foreignStoreEntries(cleanup.tombPath);
|
|
629
|
+
if (foreign.length)
|
|
630
|
+
refuse(foreign);
|
|
631
|
+
for (const entry of STORE_ENTRY_NAMES)
|
|
632
|
+
rmSync(join(cleanup.tombPath, entry), { recursive: true, force: true });
|
|
633
|
+
try {
|
|
634
|
+
rmdirSync(cleanup.tombPath);
|
|
635
|
+
}
|
|
636
|
+
catch (error) {
|
|
637
|
+
if (errno(error) !== "ENOTEMPTY" && errno(error) !== "EEXIST")
|
|
638
|
+
throw error;
|
|
639
|
+
refuse(foreignStoreEntries(cleanup.tombPath));
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
function cleanupAttemptTarget(lock, record) {
|
|
643
|
+
const target = record.restore.target;
|
|
644
|
+
if (!target)
|
|
645
|
+
return record;
|
|
646
|
+
let current = record;
|
|
647
|
+
if (!current.restore.cleanup)
|
|
648
|
+
current = startCleanup(lock, current, "attempt-target", target);
|
|
649
|
+
if (current.restore.cleanup?.kind !== "attempt-target")
|
|
650
|
+
maintenanceError("cleanup-forbidden", "pre-commit rollback cannot run previous-source cleanup", {
|
|
651
|
+
attemptId: current.restore.attemptId,
|
|
652
|
+
});
|
|
653
|
+
return continueCleanup(lock, current);
|
|
654
|
+
}
|
|
655
|
+
/** Refuse to touch a LIVE restore attempt: only the exact recorded coordinator, or a caller that
|
|
656
|
+
* proves the attempt stale (every owner dead, and the deadline elapsed or no unrecorded owner
|
|
657
|
+
* alive), may roll it back. */
|
|
658
|
+
export function assertRestoreClaimRollbackable(lock, record, options) {
|
|
659
|
+
if (options.asCoordinator) {
|
|
660
|
+
if (!sameProcessOwner(options.asCoordinator, record.claim.coordinator))
|
|
661
|
+
maintenanceError("claim-live", "rollback caller is not the recorded restore coordinator", {
|
|
662
|
+
root: lock.root, attemptId: record.restore.attemptId,
|
|
663
|
+
recourse: [{ action: "retry", description: "Wait for the live restore attempt to finish or become provably stale." }],
|
|
664
|
+
});
|
|
665
|
+
return;
|
|
666
|
+
}
|
|
667
|
+
const assessment = assessRestoreClaim(record, options);
|
|
668
|
+
if (assessment === "live")
|
|
669
|
+
maintenanceError("claim-live", "a restore attempt claim is still live", {
|
|
670
|
+
root: lock.root, attemptId: record.restore.attemptId,
|
|
671
|
+
recourse: [{ action: "retry", description: `Retry after ${record.claim.deadline} once the restore coordinator, watchdogs, and brokers have exited.` }],
|
|
672
|
+
});
|
|
673
|
+
if (assessment === "ambiguous")
|
|
674
|
+
maintenanceError("claim-owner-ambiguous", "not every restore attempt owner is proven dead", {
|
|
675
|
+
root: lock.root, attemptId: record.restore.attemptId,
|
|
676
|
+
recourse: [{ action: "inspect", description: "Prove the recorded restore coordinator, watchdogs, and brokers dead before recovery." }],
|
|
677
|
+
});
|
|
678
|
+
}
|
|
679
|
+
export function rollbackRestore(lock, options = {}) {
|
|
680
|
+
const initial = currentJournal(lock);
|
|
681
|
+
if (initial.state === "commit-intent" || initial.state === "manager-committed" || initial.state === "active" || initial.state === "degraded") {
|
|
682
|
+
const paths = [initial.restore.target.path, initial.restore.previousSource?.identity.path].filter((p) => Boolean(p));
|
|
683
|
+
maintenanceError("rollback-forbidden", "restore commit intent makes rollback unsafe", {
|
|
684
|
+
root: lock.root, attemptId: initial.restore.attemptId, paths,
|
|
685
|
+
recourse: [
|
|
686
|
+
{ action: "repair", description: "Preserve the restored target and old source; recover forward.", paths },
|
|
687
|
+
{ action: "cleanup", description: "After healthy activation, explicitly clean the recorded old source.", command: `cotal clean restore-fallback --attempt ${initial.restore.attemptId} --force`, paths },
|
|
688
|
+
],
|
|
689
|
+
});
|
|
690
|
+
}
|
|
691
|
+
if (initial.state !== "restore-ready")
|
|
692
|
+
maintenanceError("invalid-transition", "no pre-commit restore is available to roll back", { root: lock.root });
|
|
693
|
+
assertRestoreClaimRollbackable(lock, initial, options);
|
|
694
|
+
// The attempt-owned config is deleted below only once no live broker definition names it: either
|
|
695
|
+
// the forward rewrite never landed, or the definition rollback already recorded its revision.
|
|
696
|
+
const reconfiguration = initial.restore.controllerReconfiguration;
|
|
697
|
+
const live = options.controllerDefinition?.liveRevision;
|
|
698
|
+
if (reconfiguration && !(reconfiguration.phase === "rendered" && live === reconfiguration.fromRevision) &&
|
|
699
|
+
!(reconfiguration.phase === "rolled-back" && live === reconfiguration.rollbackRevision))
|
|
700
|
+
maintenanceError("rollback-forbidden", "the controller broker definition is not back at the cut's revision; its definition rollback must run first", {
|
|
701
|
+
root: lock.root, attemptId: initial.restore.attemptId,
|
|
702
|
+
recourse: [{ action: "inspect", description: "Run the controller restore recovery, which rolls the broker definition back before the store." }],
|
|
703
|
+
});
|
|
704
|
+
if (!initial.restore.target && pathExistsStrict(initial.restore.targetPath) &&
|
|
705
|
+
!(initial.restore.method === "same-path" && initial.phase === "move-pending" &&
|
|
706
|
+
sameStoreIdentity(initial.source, readStoreIdentity(initial.restore.targetPath)))) {
|
|
707
|
+
// A crash between target creation and the bind journal write leaves an unbound target that is
|
|
708
|
+
// either EMPTY (pre-marker) or holds EXACTLY the attempt's own generation marker — the single
|
|
709
|
+
// durable side effect of ensureStoreIdentity before writeJournal. Both shapes are content-
|
|
710
|
+
// bounded and provably data-free, so removal is safe; anything else fails closed.
|
|
711
|
+
const stat = lstatSync(initial.restore.targetPath);
|
|
712
|
+
const removableUnboundTarget = () => {
|
|
713
|
+
if (!stat.isDirectory() || stat.isSymbolicLink())
|
|
714
|
+
return false;
|
|
715
|
+
const entries = readdirSync(initial.restore.targetPath);
|
|
716
|
+
if (entries.length === 0)
|
|
717
|
+
return true;
|
|
718
|
+
if (entries.length !== 1 || entries[0] !== STORE_ID_FILE)
|
|
719
|
+
return false;
|
|
720
|
+
const marker = lstatSync(join(initial.restore.targetPath, STORE_ID_FILE));
|
|
721
|
+
if (!marker.isFile() || marker.isSymbolicLink() || marker.size > 128)
|
|
722
|
+
return false;
|
|
723
|
+
const generation = readFileSync(join(initial.restore.targetPath, STORE_ID_FILE), "utf8").trim();
|
|
724
|
+
return /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(generation);
|
|
725
|
+
};
|
|
726
|
+
if (removableUnboundTarget()) {
|
|
727
|
+
rmSync(initial.restore.targetPath, { recursive: true });
|
|
728
|
+
fsyncDirectory(dirname(initial.restore.targetPath));
|
|
729
|
+
}
|
|
730
|
+
else {
|
|
731
|
+
maintenanceError("ambiguous-filesystem-state", "unbound restore target appeared before rollback", {
|
|
732
|
+
attemptId: initial.restore.attemptId,
|
|
733
|
+
paths: [initial.restore.targetPath],
|
|
734
|
+
recourse: [{ action: "inspect", description: "Preserve the unbound path; its ownership was never journaled.", paths: [initial.restore.targetPath] }],
|
|
735
|
+
});
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
// Never delete the restored target until the exact old-source rollback path is proven usable.
|
|
739
|
+
preflightRollbackSource(initial);
|
|
740
|
+
let record = cleanupAttemptTarget(lock, initial);
|
|
741
|
+
if (record.restore.method === "same-path") {
|
|
742
|
+
const fallbackPath = record.restore.fallbackPath;
|
|
743
|
+
const source = identityAt(record.source.path);
|
|
744
|
+
const fallback = identityAt(fallbackPath);
|
|
745
|
+
const expectedFallback = { ...record.source, path: fallbackPath };
|
|
746
|
+
if (source && sameStoreIdentity(record.source, source) && !fallback) {
|
|
747
|
+
// Crash after the fallback was already returned.
|
|
748
|
+
}
|
|
749
|
+
else if (!source && fallback && sameStoreIdentity(expectedFallback, fallback)) {
|
|
750
|
+
renameSync(fallbackPath, record.source.path);
|
|
751
|
+
fsyncDirectory(dirname(record.source.path));
|
|
752
|
+
assertStoreIdentity(record.source);
|
|
753
|
+
}
|
|
754
|
+
else {
|
|
755
|
+
maintenanceError("ambiguous-filesystem-state", "same-path rollback cannot identify one unchanged source", {
|
|
756
|
+
attemptId: record.restore.attemptId,
|
|
757
|
+
paths: [record.source.path, fallbackPath], expected: record.source, actual: source ?? fallback,
|
|
758
|
+
recourse: [{ action: "inspect", description: "Preserve both paths and repair manually; do not overwrite either store.", paths: [record.source.path, fallbackPath] }],
|
|
759
|
+
});
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
else if (record.restore.method === "alternate") {
|
|
763
|
+
assertStoreIdentity(record.source);
|
|
764
|
+
}
|
|
765
|
+
else if (identityAt(record.source.path)) {
|
|
766
|
+
maintenanceError("ambiguous-filesystem-state", "missing-source rollback found an unexpected replacement", {
|
|
767
|
+
attemptId: record.restore.attemptId, paths: [record.source.path], expected: record.source,
|
|
768
|
+
});
|
|
769
|
+
}
|
|
770
|
+
removeOwnedPathsOrFail(lock, record.restore.attemptId, record.restore.ownedPaths);
|
|
771
|
+
return writeJournal(lock, record, readyFrom(record));
|
|
772
|
+
}
|
|
773
|
+
/** Whether a restore's launch records that the cut it replaced was preserved after its artifact was
|
|
774
|
+
* made, so the store it set aside holds history newer than the artifact. */
|
|
775
|
+
export function restoreReplacedNewerCut(record) {
|
|
776
|
+
return record.launch?.previousCutNewer === true;
|
|
777
|
+
}
|
|
778
|
+
/** The other JetStream accounts whose streams a restore left in the store it set aside, such as an
|
|
779
|
+
* open mesh's account beside a static space's after both ran in one folder. A backup holds one
|
|
780
|
+
* account, so that store holds their only copy. The restore's launch records them. When it has no
|
|
781
|
+
* such record, as in a restore journaled by an earlier release, they are read from that store while
|
|
782
|
+
* it is still the recorded one, and only the accounts the journal's auth mode rules out are named:
|
|
783
|
+
* an open restore restored `$G`, and a static or user restore restored a data account, never `$G`.
|
|
784
|
+
* Which data account that was is not recorded, so after a static or user restore only `$G` is named.
|
|
785
|
+
* A store at that path that is not the recorded one throws. */
|
|
786
|
+
export function restoreOtherAccounts(record) {
|
|
787
|
+
const recorded = record.launch?.otherAccounts;
|
|
788
|
+
if (Array.isArray(recorded))
|
|
789
|
+
return recorded.filter((account) => typeof account === "string");
|
|
790
|
+
const previous = record.restore.previousSource?.identity;
|
|
791
|
+
if (!record.launch || !previous || !identityAt(previous.path))
|
|
792
|
+
return [];
|
|
793
|
+
assertStoreIdentity(previous);
|
|
794
|
+
const held = storeAccounts(previous.path);
|
|
795
|
+
return record.mode === "open" ? held.filter((account) => account !== "$G") : held.filter((account) => account === "$G");
|
|
796
|
+
}
|
|
797
|
+
/** What a restore left only in the store it set aside. A registry restore left out every other
|
|
798
|
+
* stream, and a full restore whose launch names no workflow journal, of a user-mode artifact or one
|
|
799
|
+
* made before full backups held runs, left out the workflow runs. A full restore that replaced a
|
|
800
|
+
* newer cut ({@link restoreReplacedNewerCut}) also left out what was written after its artifact.
|
|
801
|
+
* Any restore left out the streams of the other accounts in that store ({@link restoreOtherAccounts}).
|
|
802
|
+
* Undefined when it left out nothing. */
|
|
803
|
+
export function restoreLeftOut(record) {
|
|
804
|
+
const accounts = restoreOtherAccounts(record);
|
|
805
|
+
const theirs = accounts.length ? `streams of ${accounts.length === 1 ? "account" : "accounts"} ${accounts.join(", ")}` : undefined;
|
|
806
|
+
const restored = record.launch?.restoredStreams;
|
|
807
|
+
const runs = Array.isArray(restored) && restored.includes(wfjStreamName(record.space));
|
|
808
|
+
const own = record.restore.selection === "registry" ? "chat and DM history and workflow runs"
|
|
809
|
+
: restoreReplacedNewerCut(record) ? (runs ? "newer chat and DM history and workflow runs" : "workflow runs and newer chat and DM history")
|
|
810
|
+
: runs ? undefined : "workflow runs";
|
|
811
|
+
return own && theirs ? `${own} and the ${theirs}` : own ?? theirs;
|
|
812
|
+
}
|
|
813
|
+
/** The runs a restore's launch records it rolled back, and how many more it counted than it listed.
|
|
814
|
+
* Empty when it recorded none. */
|
|
815
|
+
export function restoreRolledBackRuns(record) {
|
|
816
|
+
const recorded = record.launch?.rolledBackRuns, more = record.launch?.rolledBackRunsMore;
|
|
817
|
+
const runs = Array.isArray(recorded) ? recorded.flatMap((row) => {
|
|
818
|
+
const value = row;
|
|
819
|
+
if (!value || typeof value !== "object" || typeof value.run !== "string" || !Array.isArray(value.reopened) || !value.reopened.every((key) => typeof key === "string"))
|
|
820
|
+
return [];
|
|
821
|
+
return [{ run: value.run, reopened: value.reopened, ...(value.missing === true ? { missing: true } : {}) }];
|
|
822
|
+
}) : [];
|
|
823
|
+
return { runs, more: typeof more === "number" && Number.isSafeInteger(more) && more > 0 ? more : 0 };
|
|
824
|
+
}
|
|
825
|
+
/** Why a restore left something only in the store it set aside ({@link restoreLeftOut}), as a clause
|
|
826
|
+
* about the restore. Undefined when it left out nothing. */
|
|
827
|
+
export function restoreLeftOutReason(record) {
|
|
828
|
+
const leftOut = restoreLeftOut(record);
|
|
829
|
+
if (!leftOut)
|
|
830
|
+
return undefined;
|
|
831
|
+
const accounts = restoreOtherAccounts(record).length ? ["held only one of the accounts in the store it replaced"] : [];
|
|
832
|
+
if (record.restore.selection === "registry")
|
|
833
|
+
return `this restore was registry-only${accounts.length ? ` and its artifact ${accounts[0]}` : ""}`;
|
|
834
|
+
const why = [...(restoreReplacedNewerCut(record) ? ["was made before the state it replaced was preserved"] : []), ...(leftOut.startsWith("workflow runs") ? ["held no workflow runs"] : []), ...accounts];
|
|
835
|
+
return `this restore's artifact ${why.join(" and ")}`;
|
|
836
|
+
}
|
|
837
|
+
/** Delete the old store a healthy active restore set aside, then retire its journal. When the
|
|
838
|
+
* restore left something out ({@link restoreLeftOut}), that store holds its only copy and is
|
|
839
|
+
* deleted only with `discardHistory`. A store that also holds an entry no Cotal store writes
|
|
840
|
+
* ({@link foreignStoreEntries}) is not deleted at all. */
|
|
841
|
+
export function cleanupRestoreFallback(lock, attemptId, options = {}) {
|
|
842
|
+
let record = currentJournal(lock);
|
|
843
|
+
if (record.state !== "active" || record.restore.attemptId !== attemptId)
|
|
844
|
+
maintenanceError("cleanup-forbidden", "old-source cleanup requires the matching healthy active restore", {
|
|
845
|
+
root: lock.root, attemptId,
|
|
846
|
+
recourse: [{ action: "repair", description: "Recover the restore to healthy active state before cleanup." }],
|
|
847
|
+
});
|
|
848
|
+
assertStoreIdentity(record.restore.target);
|
|
849
|
+
const previous = record.restore.previousSource;
|
|
850
|
+
// A disaster restore replaced a missing source, so it retains no old store: retiring its journal is
|
|
851
|
+
// the whole cleanup, after the same finalization checks.
|
|
852
|
+
if (!previous && record.restore.method === "disaster" && !record.restore.cleanup)
|
|
853
|
+
return retireCleanedRestore(lock, record);
|
|
854
|
+
if (!previous)
|
|
855
|
+
maintenanceError("cleanup-forbidden", "this restore has no retained old source", { root: lock.root, attemptId });
|
|
856
|
+
if (sameStoreIdentity(previous.identity, record.restore.target) || previous.identity.path === record.restore.target.path)
|
|
857
|
+
maintenanceError("cleanup-forbidden", "recorded old source aliases the active target", {
|
|
858
|
+
root: lock.root, attemptId, paths: [previous.identity.path, record.restore.target.path],
|
|
859
|
+
});
|
|
860
|
+
if (pathContains(previous.identity.path, record.restore.target.path))
|
|
861
|
+
maintenanceError("cleanup-forbidden", "old-source cleanup would recursively delete the active target", {
|
|
862
|
+
root: lock.root, attemptId, paths: [previous.identity.path, record.restore.target.path],
|
|
863
|
+
recourse: [{ action: "repair", description: "Preserve both trees and repair the overlapping restore journal.", paths: [previous.identity.path, record.restore.target.path] }],
|
|
864
|
+
});
|
|
865
|
+
if (record.restore.cleanup?.status === "complete")
|
|
866
|
+
return retireCleanedRestore(lock, record);
|
|
867
|
+
// A cleanup already under way had this consent when it started.
|
|
868
|
+
const leftOut = restoreLeftOut(record);
|
|
869
|
+
if (!record.restore.cleanup && leftOut && !options.discardHistory)
|
|
870
|
+
maintenanceError("cleanup-forbidden", `the restore's old source holds the only copy of the ${leftOut} it left out`, {
|
|
871
|
+
root: lock.root, attemptId, paths: [previous.identity.path],
|
|
872
|
+
recourse: [{ action: "cleanup", description: "Keep the old source, or repeat the cleanup with explicit consent to discard that history.", paths: [previous.identity.path] }],
|
|
873
|
+
});
|
|
874
|
+
// A file in the old source that no Cotal store writes is someone's, so nothing there is deleted.
|
|
875
|
+
if (!record.restore.cleanup && existsSync(previous.identity.path)) {
|
|
876
|
+
const foreign = foreignStoreEntries(previous.identity.path);
|
|
877
|
+
if (foreign.length)
|
|
878
|
+
maintenanceError("store-holds-foreign-entries", `the restore's old source ${previous.identity.path} also holds ${foreign.join(", ")}, which no Cotal store writes`, {
|
|
879
|
+
root: lock.root, attemptId, paths: [previous.identity.path],
|
|
880
|
+
recourse: [{ action: "inspect", description: "Move those entries out of the old source, then run the cleanup again.", paths: [previous.identity.path] }],
|
|
881
|
+
});
|
|
882
|
+
}
|
|
883
|
+
if (!record.restore.cleanup)
|
|
884
|
+
record = startCleanup(lock, record, "previous-source", previous.identity);
|
|
885
|
+
if (record.restore.cleanup?.kind !== "previous-source")
|
|
886
|
+
maintenanceError("cleanup-forbidden", "active record contains an attempt-target cleanup", { root: lock.root, attemptId });
|
|
887
|
+
record = continueCleanup(lock, record);
|
|
888
|
+
return retireCleanedRestore(lock, record);
|
|
889
|
+
}
|
|
890
|
+
/** Consume a healthy restore only after exact old-source cleanup is durably complete. A disaster
|
|
891
|
+
* restore retained no old source, so it has no cleanup to complete. */
|
|
892
|
+
function retireCleanedRestore(lock, record) {
|
|
893
|
+
const cleaned = record.restore.method === "disaster"
|
|
894
|
+
? record.restore.cleanup === undefined
|
|
895
|
+
: record.restore.cleanup?.kind === "previous-source" && record.restore.cleanup.status === "complete";
|
|
896
|
+
if (!cleaned)
|
|
897
|
+
maintenanceError("cleanup-forbidden", "restore retirement requires completed previous-source cleanup", {
|
|
898
|
+
root: lock.root, attemptId: record.restore.attemptId,
|
|
899
|
+
});
|
|
900
|
+
assertStoreIdentity(record.restore.target);
|
|
901
|
+
if (!validRestoreActivationEvidence(record.details, record.restore.attemptId) ||
|
|
902
|
+
!sameManagerCommitEvidence(record.managerCommit, record.details.managerCommit))
|
|
903
|
+
maintenanceError("activation-evidence-invalid", "restore retirement requires exact healthy manager finalization evidence", {
|
|
904
|
+
root: lock.root, attemptId: record.restore.attemptId, paths: [record.restore.target.path],
|
|
905
|
+
});
|
|
906
|
+
readMaintenanceResumeDocument(lock.root, record.resume);
|
|
907
|
+
const paths = maintenancePaths(lock.root);
|
|
908
|
+
unlinkSync(paths.journal);
|
|
909
|
+
fsyncDirectory(paths.versionDir);
|
|
910
|
+
return record;
|
|
911
|
+
}
|
|
912
|
+
//# sourceMappingURL=maintenance-restore.js.map
|