@indigoai-us/hq-cloud 6.15.73 → 6.15.75
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/journal-store.test.js +12 -0
- package/dist/journal-store.test.js.map +1 -1
- package/dist/sync/state-store.d.ts +30 -5
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +830 -128
- package/dist/sync/state-store.js.map +1 -1
- package/dist/sync/state-store.test.js +331 -3
- package/dist/sync/state-store.test.js.map +1 -1
- package/package.json +1 -1
package/dist/sync/state-store.js
CHANGED
|
@@ -38,15 +38,66 @@ const MAX_FRAME_BYTES = 256 * 1024 * 1024;
|
|
|
38
38
|
const DEFAULT_MAX_WAL_BYTES_BEFORE_COMPACTION = 16 * 1024 * 1024;
|
|
39
39
|
const LOCK_RETRY_MS = 10;
|
|
40
40
|
const LOCK_TIMEOUT_MS = 5_000;
|
|
41
|
+
const REPLACE_STATE_MAX_ATTEMPTS = 3;
|
|
42
|
+
const OPTIMISTIC_RECOVERY_MAX_RESAMPLES = 3;
|
|
41
43
|
// Test instrumentation for materialisation regressions. Kept at the storage
|
|
42
44
|
// seam so a fixture counts actual snapshot JSON decodes, not API invocations.
|
|
43
45
|
let snapshotRecoveryCount = 0;
|
|
46
|
+
let snapshotMaterializationCount = 0;
|
|
47
|
+
let snapshotEncodeCount = 0;
|
|
48
|
+
let snapshotWriteHookForTest;
|
|
49
|
+
let snapshotReadHookForTest;
|
|
50
|
+
let walReadHookForTest;
|
|
51
|
+
let afterGenerationStageHookForTest;
|
|
52
|
+
let reflinkCopyHookForTest;
|
|
53
|
+
let positionalWriteForTest;
|
|
54
|
+
let rotationClaimWriteHookForTest;
|
|
44
55
|
export function resetStateStoreSnapshotRecoveryCountForTest() {
|
|
45
56
|
snapshotRecoveryCount = 0;
|
|
46
57
|
}
|
|
47
58
|
export function getStateStoreSnapshotRecoveryCountForTest() {
|
|
48
59
|
return snapshotRecoveryCount;
|
|
49
60
|
}
|
|
61
|
+
export function resetStateStoreSnapshotMaterializationCountForTest() {
|
|
62
|
+
snapshotMaterializationCount = 0;
|
|
63
|
+
}
|
|
64
|
+
export function getStateStoreSnapshotMaterializationCountForTest() {
|
|
65
|
+
return snapshotMaterializationCount;
|
|
66
|
+
}
|
|
67
|
+
export function resetStateStoreSnapshotEncodeCountForTest() {
|
|
68
|
+
snapshotEncodeCount = 0;
|
|
69
|
+
}
|
|
70
|
+
export function getStateStoreSnapshotEncodeCountForTest() {
|
|
71
|
+
return snapshotEncodeCount;
|
|
72
|
+
}
|
|
73
|
+
/** Test-only seam placed immediately before the expensive snapshot write. */
|
|
74
|
+
export function setStateStoreSnapshotWriteHookForTest(hook) {
|
|
75
|
+
snapshotWriteHookForTest = hook;
|
|
76
|
+
}
|
|
77
|
+
/** Test-only seam after snapshot validation and before its large payload read. */
|
|
78
|
+
export function setStateStoreSnapshotReadHookForTest(hook) {
|
|
79
|
+
snapshotReadHookForTest = hook;
|
|
80
|
+
}
|
|
81
|
+
/** Test-only seam after the WAL length is sampled during cold recovery. */
|
|
82
|
+
export function setStateStoreWalReadHookForTest(hook) {
|
|
83
|
+
walReadHookForTest = hook;
|
|
84
|
+
}
|
|
85
|
+
/** Test-only crash seam after durable staging and before lock-protected publish. */
|
|
86
|
+
export function setStateStoreAfterGenerationStageHookForTest(hook) {
|
|
87
|
+
afterGenerationStageHookForTest = hook;
|
|
88
|
+
}
|
|
89
|
+
/** Test-only seam for filesystems that do not support forced reflinks. */
|
|
90
|
+
export function setStateStoreReflinkCopyHookForTest(hook) {
|
|
91
|
+
reflinkCopyHookForTest = hook;
|
|
92
|
+
}
|
|
93
|
+
/** Test-only seam for short positional snapshot writes. */
|
|
94
|
+
export function setStateStorePositionalWriteForTest(write) {
|
|
95
|
+
positionalWriteForTest = write;
|
|
96
|
+
}
|
|
97
|
+
/** Test-only seam for rotation-claim PID write failures. */
|
|
98
|
+
export function setStateStoreRotationClaimWriteHookForTest(hook) {
|
|
99
|
+
rotationClaimWriteHookForTest = hook;
|
|
100
|
+
}
|
|
50
101
|
export class StateStoreCorruptionError extends Error {
|
|
51
102
|
constructor(message, options) {
|
|
52
103
|
super(`state-store corruption: ${message}`, options);
|
|
@@ -65,6 +116,9 @@ class StateStoreReducerError extends Error {
|
|
|
65
116
|
this.name = "StateStoreReducerError";
|
|
66
117
|
}
|
|
67
118
|
}
|
|
119
|
+
/** A sampled generation disappeared while it was being read off-lock. */
|
|
120
|
+
class StateStoreGenerationMissingError extends StateStoreCorruptionError {
|
|
121
|
+
}
|
|
68
122
|
/** A recovered v3 store. `append` is the ordinary hot path. */
|
|
69
123
|
export class StateStore {
|
|
70
124
|
options;
|
|
@@ -77,6 +131,7 @@ export class StateStore {
|
|
|
77
131
|
recordCount;
|
|
78
132
|
walBytes;
|
|
79
133
|
maxRecoveryReadBytes;
|
|
134
|
+
observedHighestSnapshotGeneration;
|
|
80
135
|
constructor(options, recovered, wasCreated) {
|
|
81
136
|
this.options = options;
|
|
82
137
|
this.scopeDigest = digestScope(options.scopeId);
|
|
@@ -87,6 +142,7 @@ export class StateStore {
|
|
|
87
142
|
this.recordCount = recovered.recordCount;
|
|
88
143
|
this.walBytes = recovered.walBytes;
|
|
89
144
|
this.maxRecoveryReadBytes = recovered.maxReadBytes;
|
|
145
|
+
this.observedHighestSnapshotGeneration = recovered.observedHighestSnapshotGeneration ?? recovered.generation;
|
|
90
146
|
this.wasCreated = wasCreated;
|
|
91
147
|
}
|
|
92
148
|
static exists(rootDir, scopeId) {
|
|
@@ -105,24 +161,32 @@ export class StateStore {
|
|
|
105
161
|
const scopeDigest = digestScope(normalized.scopeId);
|
|
106
162
|
const dir = stateDirectory(normalized.rootDir, scopeDigest);
|
|
107
163
|
fs.mkdirSync(dir, { recursive: true });
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
if (recovered) {
|
|
164
|
+
while (true) {
|
|
165
|
+
const recovered = readNewestGenerationOptimistically(dir, scopeDigest, normalized.reduce);
|
|
166
|
+
if (recovered)
|
|
112
167
|
return new StateStore(normalized, recovered, false);
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
168
|
+
const created = withScopeLock(dir, () => {
|
|
169
|
+
cleanupUnpublishedInitialGeneration(dir);
|
|
170
|
+
if (highestCompleteGenerationOnDisk(dir) !== undefined)
|
|
171
|
+
return undefined;
|
|
172
|
+
if (hasStateStoreArtifacts(dir)) {
|
|
173
|
+
throw new StateStoreCorruptionError("no complete generation remains while opening");
|
|
174
|
+
}
|
|
175
|
+
const initial = {
|
|
176
|
+
generation: 1,
|
|
177
|
+
state: normalized.initialState,
|
|
178
|
+
lastRecordSequence: 0,
|
|
179
|
+
recordCount: 0,
|
|
180
|
+
walBytes: WAL_HEADER_BYTES,
|
|
181
|
+
maxReadBytes: 0,
|
|
182
|
+
};
|
|
183
|
+
writeGeneration(dir, scopeDigest, initial);
|
|
184
|
+
writeCurrentPointer(dir, initial.generation);
|
|
185
|
+
return new StateStore(normalized, initial, true);
|
|
186
|
+
});
|
|
187
|
+
if (created)
|
|
188
|
+
return created;
|
|
189
|
+
}
|
|
126
190
|
}
|
|
127
191
|
/** Yielding first-open variant for callback-originated projections. */
|
|
128
192
|
static async openAsync(options) {
|
|
@@ -130,28 +194,38 @@ export class StateStore {
|
|
|
130
194
|
const scopeDigest = digestScope(normalized.scopeId);
|
|
131
195
|
const dir = stateDirectory(normalized.rootDir, scopeDigest);
|
|
132
196
|
await fs.promises.mkdir(dir, { recursive: true });
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
const recovered = readNewestGeneration(dir, scopeDigest, normalized.reduce);
|
|
197
|
+
while (true) {
|
|
198
|
+
const recovered = await readNewestGenerationOptimisticallyAsync(dir, scopeDigest, normalized.reduce);
|
|
136
199
|
if (recovered)
|
|
137
200
|
return new StateStore(normalized, recovered, false);
|
|
138
|
-
const
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
201
|
+
const created = await withScopeLockAsync(dir, () => {
|
|
202
|
+
cleanupUnpublishedInitialGeneration(dir);
|
|
203
|
+
if (highestCompleteGenerationOnDisk(dir) !== undefined)
|
|
204
|
+
return undefined;
|
|
205
|
+
if (hasStateStoreArtifacts(dir)) {
|
|
206
|
+
throw new StateStoreCorruptionError("no complete generation remains while opening");
|
|
207
|
+
}
|
|
208
|
+
const initial = {
|
|
209
|
+
generation: 1,
|
|
210
|
+
state: normalized.initialState,
|
|
211
|
+
lastRecordSequence: 0,
|
|
212
|
+
recordCount: 0,
|
|
213
|
+
walBytes: WAL_HEADER_BYTES,
|
|
214
|
+
maxReadBytes: 0,
|
|
215
|
+
};
|
|
216
|
+
writeGeneration(dir, scopeDigest, initial);
|
|
217
|
+
writeCurrentPointer(dir, initial.generation);
|
|
218
|
+
return new StateStore(normalized, initial, true);
|
|
219
|
+
});
|
|
220
|
+
if (created)
|
|
221
|
+
return created;
|
|
222
|
+
}
|
|
150
223
|
}
|
|
151
224
|
/**
|
|
152
|
-
* Repair an over-budget generation without
|
|
153
|
-
*
|
|
154
|
-
* the
|
|
225
|
+
* Repair an over-budget generation without making appenders wait for snapshot
|
|
226
|
+
* I/O. Recovery captures a stable generation while holding the scope lock,
|
|
227
|
+
* then materialises the replacement off-lock; publication is conditional on
|
|
228
|
+
* that source generation still being current.
|
|
155
229
|
*/
|
|
156
230
|
static repairIfNeeded(options, maintenance = {}) {
|
|
157
231
|
const normalized = normalizeOptions(options);
|
|
@@ -159,50 +233,91 @@ export class StateStore {
|
|
|
159
233
|
const dir = stateDirectory(normalized.rootDir, scopeDigest);
|
|
160
234
|
if (!hasSnapshotGeneration(dir))
|
|
161
235
|
return { status: "missing" };
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
const
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
try {
|
|
172
|
-
walBytes = fs.statSync(walPath(dir, newestGeneration)).size;
|
|
173
|
-
snapshotBytes = fs.statSync(snapshotPath(dir, newestGeneration)).size;
|
|
236
|
+
let plan;
|
|
237
|
+
while (!plan) {
|
|
238
|
+
const preflight = withScopeLock(dir, () => {
|
|
239
|
+
cleanupUnpublishedInitialGeneration(dir);
|
|
240
|
+
const newestGeneration = highestCompleteGenerationOnDisk(dir);
|
|
241
|
+
if (newestGeneration === undefined)
|
|
242
|
+
return undefined;
|
|
243
|
+
if (!maintenance.force && !needsMaintenance(dir, newestGeneration, normalized)) {
|
|
244
|
+
return false;
|
|
174
245
|
}
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
246
|
+
return newestGeneration;
|
|
247
|
+
});
|
|
248
|
+
if (preflight === undefined) {
|
|
249
|
+
// Preserve recovery's corruption classification for a crashed or
|
|
250
|
+
// malformed store that has snapshots but no complete generation.
|
|
251
|
+
if (hasSnapshotGeneration(dir)) {
|
|
252
|
+
readNewestGeneration(dir, scopeDigest, normalized.reduce, maintenance.onProgress, false);
|
|
253
|
+
continue;
|
|
178
254
|
}
|
|
179
|
-
|
|
180
|
-
snapshotBytes === undefined ||
|
|
181
|
-
retainedGenerationHistoryNeedsRepair(dir, newestGeneration, currentGeneration) ||
|
|
182
|
-
walBytes > normalized.maxWalBytesBeforeCompaction;
|
|
183
|
-
if (!overBudget)
|
|
184
|
-
return { status: "not-needed" };
|
|
255
|
+
return { status: "missing" };
|
|
185
256
|
}
|
|
186
|
-
|
|
257
|
+
if (preflight === false)
|
|
258
|
+
return { status: "not-needed" };
|
|
259
|
+
// Snapshot and WAL replay are read-only. Do not make appenders wait for
|
|
260
|
+
// this potentially multi-second materialisation; validate its view before
|
|
261
|
+
// using it to build a replacement generation.
|
|
262
|
+
const recovered = readNewestGeneration(dir, scopeDigest, normalized.reduce, maintenance.onProgress, false);
|
|
187
263
|
if (!recovered)
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
}
|
|
264
|
+
continue;
|
|
265
|
+
plan = withScopeLock(dir, () => {
|
|
266
|
+
const validated = validateOptimisticRecovery(dir, scopeDigest, normalized.reduce, preflight, recovered);
|
|
267
|
+
if (!validated)
|
|
268
|
+
return undefined;
|
|
269
|
+
if (!maintenance.force && !needsMaintenance(dir, preflight, normalized))
|
|
270
|
+
return undefined;
|
|
271
|
+
return { recovered: validated, highestGeneration: highestGenerationOnDisk(dir) };
|
|
272
|
+
});
|
|
273
|
+
}
|
|
274
|
+
const first = {
|
|
275
|
+
generation: plan.highestGeneration + 1,
|
|
276
|
+
state: plan.recovered.state,
|
|
277
|
+
lastRecordSequence: plan.recovered.lastRecordSequence,
|
|
278
|
+
recordCount: 0,
|
|
279
|
+
walBytes: WAL_HEADER_BYTES,
|
|
280
|
+
maxReadBytes: 0,
|
|
281
|
+
};
|
|
282
|
+
// Keep two rollback-readable generations, but derive the second snapshot
|
|
283
|
+
// from a copy-on-write clone of the first so one logical repair has one
|
|
284
|
+
// full snapshot materialisation on filesystems that support reflinks.
|
|
285
|
+
const next = { ...first, generation: first.generation + 1 };
|
|
286
|
+
const stagedFirst = stageGeneration(dir, scopeDigest, first);
|
|
287
|
+
let staged;
|
|
288
|
+
try {
|
|
289
|
+
staged = stageGenerationFromSnapshot(dir, scopeDigest, stagedFirst.snapshotTemporary, next);
|
|
290
|
+
}
|
|
291
|
+
catch (error) {
|
|
292
|
+
discardStagedGeneration(stagedFirst);
|
|
293
|
+
throw error;
|
|
294
|
+
}
|
|
295
|
+
try {
|
|
296
|
+
afterGenerationStageHookForTest?.();
|
|
297
|
+
return withScopeLock(dir, () => {
|
|
298
|
+
// A writer won while the snapshot was being built. It has kept the old
|
|
299
|
+
// WAL authoritative, so leave it alone and let a later maintenance pass
|
|
300
|
+
// compact a fresh view rather than publishing a stale snapshot.
|
|
301
|
+
if (highestGenerationOnDisk(dir) !== plan.highestGeneration ||
|
|
302
|
+
generationFileSize(walPath(dir, plan.recovered.generation), "WAL", plan.recovered.generation) !== plan.recovered.walBytes)
|
|
303
|
+
return { status: "not-needed" };
|
|
304
|
+
publishStagedGeneration(dir, stagedFirst);
|
|
305
|
+
publishStagedGeneration(dir, staged);
|
|
306
|
+
writeCurrentPointer(dir, next.generation);
|
|
307
|
+
pruneGenerations(dir, [first.generation, next.generation]);
|
|
308
|
+
return {
|
|
309
|
+
status: "repaired",
|
|
310
|
+
recoveredGeneration: plan.recovered.generation,
|
|
311
|
+
recoveredWalBytes: plan.recovered.walBytes,
|
|
312
|
+
firstCompactGeneration: first.generation,
|
|
313
|
+
currentGeneration: next.generation,
|
|
314
|
+
};
|
|
315
|
+
});
|
|
316
|
+
}
|
|
317
|
+
finally {
|
|
318
|
+
discardStagedGeneration(stagedFirst);
|
|
319
|
+
discardStagedGeneration(staged);
|
|
320
|
+
}
|
|
206
321
|
}
|
|
207
322
|
getState() {
|
|
208
323
|
return this.state;
|
|
@@ -219,7 +334,26 @@ export class StateStore {
|
|
|
219
334
|
append(type, payload, flags = 0) {
|
|
220
335
|
assertUint16(type, "record type");
|
|
221
336
|
assertUint16(flags, "record flags");
|
|
222
|
-
|
|
337
|
+
let recovered = false;
|
|
338
|
+
while (true) {
|
|
339
|
+
let compact = false;
|
|
340
|
+
const result = withScopeLock(this.stateDir, () => {
|
|
341
|
+
const refresh = this.refreshFromDisk();
|
|
342
|
+
if (!refresh)
|
|
343
|
+
return undefined;
|
|
344
|
+
const appended = this.appendUnlocked(type, payload, flags, recovered ? { records: [], recovered: true } : refresh);
|
|
345
|
+
compact = this.needsCompaction();
|
|
346
|
+
return appended;
|
|
347
|
+
});
|
|
348
|
+
if (!result) {
|
|
349
|
+
this.recoverNewestGenerationOutsideLock();
|
|
350
|
+
recovered = true;
|
|
351
|
+
continue;
|
|
352
|
+
}
|
|
353
|
+
if (compact)
|
|
354
|
+
this.compact();
|
|
355
|
+
return result;
|
|
356
|
+
}
|
|
223
357
|
}
|
|
224
358
|
/**
|
|
225
359
|
* Async lock acquisition for callback-originated work. The write itself
|
|
@@ -229,24 +363,77 @@ export class StateStore {
|
|
|
229
363
|
async appendAsync(type, payload, flags = 0) {
|
|
230
364
|
assertUint16(type, "record type");
|
|
231
365
|
assertUint16(flags, "record flags");
|
|
232
|
-
|
|
366
|
+
let recovered = false;
|
|
367
|
+
while (true) {
|
|
368
|
+
let compact = false;
|
|
369
|
+
const result = await withScopeLockAsync(this.stateDir, () => {
|
|
370
|
+
const refresh = this.refreshFromDisk();
|
|
371
|
+
if (!refresh)
|
|
372
|
+
return undefined;
|
|
373
|
+
const appended = this.appendUnlocked(type, payload, flags, recovered ? { records: [], recovered: true } : refresh);
|
|
374
|
+
compact = this.needsCompaction();
|
|
375
|
+
return appended;
|
|
376
|
+
});
|
|
377
|
+
if (!result) {
|
|
378
|
+
await this.recoverNewestGenerationOutsideLockAsync();
|
|
379
|
+
recovered = true;
|
|
380
|
+
continue;
|
|
381
|
+
}
|
|
382
|
+
if (compact)
|
|
383
|
+
this.compact();
|
|
384
|
+
return result;
|
|
385
|
+
}
|
|
233
386
|
}
|
|
234
387
|
/** Conditionally append after obtaining the async scope lock. */
|
|
235
388
|
async appendIfAsync(type, payload, shouldAppend, flags = 0) {
|
|
236
389
|
assertUint16(type, "record type");
|
|
237
390
|
assertUint16(flags, "record flags");
|
|
238
|
-
|
|
391
|
+
let recovered = false;
|
|
392
|
+
while (true) {
|
|
393
|
+
let compact = false;
|
|
394
|
+
const result = await withScopeLockAsync(this.stateDir, () => {
|
|
395
|
+
const refresh = this.refreshFromDisk();
|
|
396
|
+
if (!refresh)
|
|
397
|
+
return { retry: true };
|
|
398
|
+
if (!shouldAppend())
|
|
399
|
+
return { retry: false, result: undefined };
|
|
400
|
+
const appended = this.appendUnlocked(type, payload, flags, recovered ? { records: [], recovered: true } : refresh);
|
|
401
|
+
compact = this.needsCompaction();
|
|
402
|
+
return { retry: false, result: appended };
|
|
403
|
+
});
|
|
404
|
+
if (result.retry) {
|
|
405
|
+
await this.recoverNewestGenerationOutsideLockAsync();
|
|
406
|
+
recovered = true;
|
|
407
|
+
continue;
|
|
408
|
+
}
|
|
409
|
+
if (compact)
|
|
410
|
+
this.compact();
|
|
411
|
+
return result.result;
|
|
412
|
+
}
|
|
239
413
|
}
|
|
240
414
|
/** Refresh only the authenticated unseen WAL suffix when possible. */
|
|
241
415
|
refresh() {
|
|
242
|
-
|
|
416
|
+
let recovered = false;
|
|
417
|
+
while (true) {
|
|
418
|
+
const refreshed = withScopeLock(this.stateDir, () => this.refreshFromDisk());
|
|
419
|
+
if (refreshed)
|
|
420
|
+
return recovered ? { records: [], recovered: true } : refreshed;
|
|
421
|
+
this.recoverNewestGenerationOutsideLock();
|
|
422
|
+
recovered = true;
|
|
423
|
+
}
|
|
243
424
|
}
|
|
244
425
|
/** Async counterpart for callback-originated projections. */
|
|
245
426
|
async refreshAsync() {
|
|
246
|
-
|
|
427
|
+
let recovered = false;
|
|
428
|
+
while (true) {
|
|
429
|
+
const refreshed = await withScopeLockAsync(this.stateDir, () => this.refreshFromDisk());
|
|
430
|
+
if (refreshed)
|
|
431
|
+
return recovered ? { records: [], recovered: true } : refreshed;
|
|
432
|
+
await this.recoverNewestGenerationOutsideLockAsync();
|
|
433
|
+
recovered = true;
|
|
434
|
+
}
|
|
247
435
|
}
|
|
248
|
-
appendUnlocked(type, payload, flags) {
|
|
249
|
-
const refresh = this.refreshFromDisk();
|
|
436
|
+
appendUnlocked(type, payload, flags, refresh) {
|
|
250
437
|
const record = {
|
|
251
438
|
sequence: this.lastRecordSequence + 1,
|
|
252
439
|
type,
|
|
@@ -271,9 +458,6 @@ export class StateStore {
|
|
|
271
458
|
this.lastRecordSequence = record.sequence;
|
|
272
459
|
this.recordCount++;
|
|
273
460
|
this.walBytes += frame.length;
|
|
274
|
-
if (this.recordCount >= this.options.maxRecordsBeforeCompaction ||
|
|
275
|
-
this.walBytes >= this.options.maxWalBytesBeforeCompaction)
|
|
276
|
-
this.compactUnlocked();
|
|
277
461
|
return { ...record, refresh };
|
|
278
462
|
}
|
|
279
463
|
/**
|
|
@@ -286,33 +470,104 @@ export class StateStore {
|
|
|
286
470
|
* and retain the prior compact copy.
|
|
287
471
|
*/
|
|
288
472
|
replaceState(nextState) {
|
|
289
|
-
|
|
290
|
-
this.
|
|
473
|
+
for (let attempt = 0; attempt < REPLACE_STATE_MAX_ATTEMPTS; attempt++) {
|
|
474
|
+
if (this.replaceStateAttempt(nextState))
|
|
475
|
+
return;
|
|
476
|
+
}
|
|
477
|
+
throw new StateStoreLockError(`cannot replace state after ${REPLACE_STATE_MAX_ATTEMPTS} concurrent writes`);
|
|
478
|
+
}
|
|
479
|
+
replaceStateAttempt(nextState) {
|
|
480
|
+
const plan = withScopeLock(this.stateDir, () => {
|
|
481
|
+
if (!this.refreshFromDisk())
|
|
482
|
+
return undefined;
|
|
291
483
|
const previousGeneration = this.generation;
|
|
292
484
|
const previousWalHasRecords = this.walBytes > WAL_HEADER_BYTES;
|
|
293
|
-
const first =
|
|
485
|
+
const first = {
|
|
486
|
+
generation: highestGenerationOnDisk(this.stateDir) + 1,
|
|
487
|
+
state: nextState,
|
|
488
|
+
lastRecordSequence: this.lastRecordSequence,
|
|
489
|
+
recordCount: 0,
|
|
490
|
+
walBytes: WAL_HEADER_BYTES,
|
|
491
|
+
maxReadBytes: 0,
|
|
492
|
+
};
|
|
294
493
|
let retainedGeneration = previousGeneration;
|
|
295
494
|
let current = first;
|
|
296
495
|
if (previousWalHasRecords) {
|
|
297
|
-
current =
|
|
496
|
+
current = { ...first, generation: first.generation + 1 };
|
|
298
497
|
retainedGeneration = first.generation;
|
|
299
498
|
}
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
499
|
+
return {
|
|
500
|
+
sourceGeneration: previousGeneration,
|
|
501
|
+
sourceSequence: this.lastRecordSequence,
|
|
502
|
+
sourceWalBytes: this.walBytes,
|
|
503
|
+
first,
|
|
504
|
+
current,
|
|
505
|
+
retainedGeneration,
|
|
506
|
+
};
|
|
303
507
|
});
|
|
508
|
+
if (!plan) {
|
|
509
|
+
this.recoverNewestGenerationOutsideLock();
|
|
510
|
+
return false;
|
|
511
|
+
}
|
|
512
|
+
const first = stageGeneration(this.stateDir, this.scopeDigest, plan.first);
|
|
513
|
+
let second;
|
|
514
|
+
try {
|
|
515
|
+
second = plan.current.generation === plan.first.generation
|
|
516
|
+
? undefined
|
|
517
|
+
: stageGenerationFromSnapshot(this.stateDir, this.scopeDigest, first.snapshotTemporary, plan.current);
|
|
518
|
+
}
|
|
519
|
+
catch (error) {
|
|
520
|
+
discardStagedGeneration(first);
|
|
521
|
+
throw error;
|
|
522
|
+
}
|
|
523
|
+
try {
|
|
524
|
+
return withScopeLock(this.stateDir, () => {
|
|
525
|
+
if (!this.refreshFromDisk())
|
|
526
|
+
return false;
|
|
527
|
+
if (this.generation !== plan.sourceGeneration ||
|
|
528
|
+
this.lastRecordSequence !== plan.sourceSequence ||
|
|
529
|
+
this.walBytes !== plan.sourceWalBytes)
|
|
530
|
+
return false;
|
|
531
|
+
publishStagedGeneration(this.stateDir, first);
|
|
532
|
+
if (second)
|
|
533
|
+
publishStagedGeneration(this.stateDir, second);
|
|
534
|
+
writeCurrentPointer(this.stateDir, plan.current.generation);
|
|
535
|
+
pruneGenerations(this.stateDir, [plan.retainedGeneration, plan.current.generation]);
|
|
536
|
+
this.installGeneration(plan.current);
|
|
537
|
+
return true;
|
|
538
|
+
});
|
|
539
|
+
}
|
|
540
|
+
finally {
|
|
541
|
+
discardStagedGeneration(first);
|
|
542
|
+
if (second)
|
|
543
|
+
discardStagedGeneration(second);
|
|
544
|
+
}
|
|
304
545
|
}
|
|
305
546
|
/** Create G+1 without ever merging two generations during recovery. */
|
|
306
547
|
compact() {
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
this
|
|
548
|
+
withRotationClaim(this.stateDir, () => {
|
|
549
|
+
// Appenders do not wait for rotation.lock. If one advances the source
|
|
550
|
+
// while this claim owns off-lock staging, it will skip its own compact;
|
|
551
|
+
// keep this claim and retry from the refreshed source instead.
|
|
552
|
+
while (true) {
|
|
553
|
+
const plan = withScopeLock(this.stateDir, () => {
|
|
554
|
+
if (!this.refreshFromDisk())
|
|
555
|
+
return undefined;
|
|
556
|
+
return this.compactionPlan(this.state, this.generation, this.lastRecordSequence, this.walBytes);
|
|
557
|
+
});
|
|
558
|
+
if (!plan) {
|
|
559
|
+
this.recoverNewestGenerationOutsideLock();
|
|
560
|
+
continue;
|
|
561
|
+
}
|
|
562
|
+
if (this.publishCompactionPlan(plan))
|
|
563
|
+
return;
|
|
564
|
+
}
|
|
310
565
|
});
|
|
311
566
|
}
|
|
312
567
|
refreshFromDisk() {
|
|
313
568
|
// The current pointer is deliberately not consulted: ordinary append does
|
|
314
569
|
// not rewrite it. A newer snapshot generation is the publication signal.
|
|
315
|
-
if (highestSnapshotGenerationOnDisk(this.stateDir) === this.
|
|
570
|
+
if (highestSnapshotGenerationOnDisk(this.stateDir) === this.observedHighestSnapshotGeneration) {
|
|
316
571
|
try {
|
|
317
572
|
const tail = readWalTail(walPath(this.stateDir, this.generation), this.scopeDigest, this.generation, this.walBytes, this.lastRecordSequence, this.state, this.options.reduce);
|
|
318
573
|
this.state = tail.state;
|
|
@@ -330,30 +585,61 @@ export class StateStore {
|
|
|
330
585
|
// stale-pointer and corrupt-newest-generation selection semantics.
|
|
331
586
|
}
|
|
332
587
|
}
|
|
333
|
-
|
|
588
|
+
return undefined;
|
|
589
|
+
}
|
|
590
|
+
recoverNewestGenerationOutsideLock() {
|
|
591
|
+
const recovered = readNewestGenerationOptimistically(this.stateDir, this.scopeDigest, this.options.reduce);
|
|
334
592
|
if (!recovered)
|
|
335
593
|
throw new StateStoreCorruptionError("no valid generation remains while appending");
|
|
336
594
|
this.installGeneration(recovered);
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
595
|
+
}
|
|
596
|
+
async recoverNewestGenerationOutsideLockAsync() {
|
|
597
|
+
const recovered = await readNewestGenerationOptimisticallyAsync(this.stateDir, this.scopeDigest, this.options.reduce);
|
|
598
|
+
if (!recovered)
|
|
599
|
+
throw new StateStoreCorruptionError("no valid generation remains while appending");
|
|
600
|
+
this.installGeneration(recovered);
|
|
601
|
+
}
|
|
602
|
+
needsCompaction() {
|
|
603
|
+
return this.recordCount >= this.options.maxRecordsBeforeCompaction ||
|
|
604
|
+
this.walBytes >= this.options.maxWalBytesBeforeCompaction;
|
|
605
|
+
}
|
|
606
|
+
compactionPlan(state, sourceGeneration, sourceSequence, sourceWalBytes) {
|
|
607
|
+
return {
|
|
608
|
+
sourceGeneration,
|
|
609
|
+
sourceSequence,
|
|
610
|
+
sourceWalBytes,
|
|
611
|
+
next: {
|
|
612
|
+
generation: highestGenerationOnDisk(this.stateDir) + 1,
|
|
613
|
+
state,
|
|
614
|
+
lastRecordSequence: sourceSequence,
|
|
615
|
+
recordCount: 0,
|
|
616
|
+
walBytes: WAL_HEADER_BYTES,
|
|
617
|
+
maxReadBytes: 0,
|
|
618
|
+
},
|
|
354
619
|
};
|
|
355
|
-
|
|
356
|
-
|
|
620
|
+
}
|
|
621
|
+
/** Materialise off-lock, then publish only if no writer advanced the source. */
|
|
622
|
+
publishCompactionPlan(plan) {
|
|
623
|
+
const staged = stageGeneration(this.stateDir, this.scopeDigest, plan.next);
|
|
624
|
+
try {
|
|
625
|
+
afterGenerationStageHookForTest?.();
|
|
626
|
+
return withScopeLock(this.stateDir, () => {
|
|
627
|
+
if (!this.refreshFromDisk())
|
|
628
|
+
return false;
|
|
629
|
+
if (this.generation !== plan.sourceGeneration ||
|
|
630
|
+
this.lastRecordSequence !== plan.sourceSequence ||
|
|
631
|
+
this.walBytes !== plan.sourceWalBytes)
|
|
632
|
+
return false;
|
|
633
|
+
publishStagedGeneration(this.stateDir, staged);
|
|
634
|
+
writeCurrentPointer(this.stateDir, plan.next.generation);
|
|
635
|
+
pruneGenerations(this.stateDir, [plan.sourceGeneration, plan.next.generation]);
|
|
636
|
+
this.installGeneration(plan.next);
|
|
637
|
+
return true;
|
|
638
|
+
});
|
|
639
|
+
}
|
|
640
|
+
finally {
|
|
641
|
+
discardStagedGeneration(staged);
|
|
642
|
+
}
|
|
357
643
|
}
|
|
358
644
|
installGeneration(generation) {
|
|
359
645
|
this.generation = generation.generation;
|
|
@@ -362,9 +648,10 @@ export class StateStore {
|
|
|
362
648
|
this.recordCount = generation.recordCount;
|
|
363
649
|
this.walBytes = generation.walBytes;
|
|
364
650
|
this.maxRecoveryReadBytes = generation.maxReadBytes;
|
|
651
|
+
this.observedHighestSnapshotGeneration = generation.observedHighestSnapshotGeneration ?? generation.generation;
|
|
365
652
|
}
|
|
366
653
|
}
|
|
367
|
-
function readNewestGeneration(dir, scopeDigest, reduce, onProgress) {
|
|
654
|
+
function readNewestGeneration(dir, scopeDigest, reduce, onProgress, repairTornTail = true) {
|
|
368
655
|
const names = fs.readdirSync(dir);
|
|
369
656
|
const generations = names
|
|
370
657
|
.map((name) => /^snapshot-(\d+)\.bin$/.exec(name)?.[1])
|
|
@@ -373,9 +660,13 @@ function readNewestGeneration(dir, scopeDigest, reduce, onProgress) {
|
|
|
373
660
|
.filter(Number.isSafeInteger)
|
|
374
661
|
.sort((a, b) => b - a);
|
|
375
662
|
const errors = [];
|
|
663
|
+
let sawMissingGeneration = false;
|
|
376
664
|
for (const generation of generations) {
|
|
377
665
|
try {
|
|
378
|
-
return
|
|
666
|
+
return {
|
|
667
|
+
...readGeneration(dir, scopeDigest, generation, reduce, onProgress, repairTornTail),
|
|
668
|
+
observedHighestSnapshotGeneration: generations[0],
|
|
669
|
+
};
|
|
379
670
|
}
|
|
380
671
|
catch (error) {
|
|
381
672
|
if (error instanceof StateStoreReducerError) {
|
|
@@ -383,22 +674,27 @@ function readNewestGeneration(dir, scopeDigest, reduce, onProgress) {
|
|
|
383
674
|
}
|
|
384
675
|
if (!(error instanceof StateStoreCorruptionError))
|
|
385
676
|
throw error;
|
|
677
|
+
if (error instanceof StateStoreGenerationMissingError)
|
|
678
|
+
sawMissingGeneration = true;
|
|
386
679
|
errors.push(error);
|
|
387
680
|
}
|
|
388
681
|
}
|
|
389
682
|
if (generations.length > 0) {
|
|
390
|
-
|
|
683
|
+
const ErrorType = sawMissingGeneration
|
|
684
|
+
? StateStoreGenerationMissingError
|
|
685
|
+
: StateStoreCorruptionError;
|
|
686
|
+
throw new ErrorType("no valid snapshot/WAL generation remains", {
|
|
391
687
|
cause: errors[0],
|
|
392
688
|
});
|
|
393
689
|
}
|
|
394
690
|
return undefined;
|
|
395
691
|
}
|
|
396
|
-
function readGeneration(dir, scopeDigest, generation, reduce, onProgress) {
|
|
692
|
+
function readGeneration(dir, scopeDigest, generation, reduce, onProgress, repairTornTail = true) {
|
|
397
693
|
const snapshotBytes = generationFileSize(snapshotPath(dir, generation), "snapshot", generation);
|
|
398
694
|
const totalBytes = snapshotBytes + generationFileSize(walPath(dir, generation), "WAL", generation);
|
|
399
695
|
const snapshot = readSnapshot(snapshotPath(dir, generation), scopeDigest, generation);
|
|
400
696
|
onProgress?.({ bytesRead: snapshotBytes, totalBytes });
|
|
401
|
-
const parsed = readWal(walPath(dir, generation), scopeDigest, generation, snapshot.payload.lastRecordSequence, snapshot.payload.state, reduce, snapshotBytes, totalBytes, onProgress);
|
|
697
|
+
const parsed = readWal(walPath(dir, generation), scopeDigest, generation, snapshot.payload.lastRecordSequence, snapshot.payload.state, reduce, snapshotBytes, totalBytes, onProgress, repairTornTail);
|
|
402
698
|
return {
|
|
403
699
|
generation,
|
|
404
700
|
state: parsed.state,
|
|
@@ -406,8 +702,141 @@ function readGeneration(dir, scopeDigest, generation, reduce, onProgress) {
|
|
|
406
702
|
recordCount: parsed.recordCount,
|
|
407
703
|
walBytes: parsed.walBytes,
|
|
408
704
|
maxReadBytes: Math.max(snapshot.maxReadBytes, parsed.maxReadBytes),
|
|
705
|
+
observedWalBytes: parsed.observedWalBytes,
|
|
706
|
+
tornWalTailBoundary: parsed.tornWalTailBoundary,
|
|
707
|
+
};
|
|
708
|
+
}
|
|
709
|
+
/**
|
|
710
|
+
* Snapshots are immutable once published. Cold recovery therefore parses the
|
|
711
|
+
* selected snapshot and WAL without append.lock, then validates its generation
|
|
712
|
+
* under the lock. Appends made during the replay are read as one short locked
|
|
713
|
+
* WAL suffix rather than restarting the expensive immutable snapshot read.
|
|
714
|
+
* Rotation or a torn suffix remains lock-protected.
|
|
715
|
+
*/
|
|
716
|
+
function readNewestGenerationOptimistically(dir, scopeDigest, reduce, onProgress) {
|
|
717
|
+
let resamples = 0;
|
|
718
|
+
while (true) {
|
|
719
|
+
const candidate = withScopeLock(dir, () => {
|
|
720
|
+
cleanupUnpublishedInitialGeneration(dir);
|
|
721
|
+
return highestCompleteGenerationOnDisk(dir);
|
|
722
|
+
});
|
|
723
|
+
if (candidate === undefined)
|
|
724
|
+
return undefined;
|
|
725
|
+
let recovered;
|
|
726
|
+
try {
|
|
727
|
+
recovered = readNewestGeneration(dir, scopeDigest, reduce, onProgress, false);
|
|
728
|
+
}
|
|
729
|
+
catch (error) {
|
|
730
|
+
if (!(error instanceof StateStoreGenerationMissingError))
|
|
731
|
+
throw error;
|
|
732
|
+
const freshCandidate = withScopeLock(dir, () => highestCompleteGenerationOnDisk(dir));
|
|
733
|
+
if (freshCandidate === candidate)
|
|
734
|
+
throw error;
|
|
735
|
+
if (resamples++ >= OPTIMISTIC_RECOVERY_MAX_RESAMPLES) {
|
|
736
|
+
throw new StateStoreLockError("generation changed repeatedly during optimistic recovery", { cause: error });
|
|
737
|
+
}
|
|
738
|
+
continue;
|
|
739
|
+
}
|
|
740
|
+
if (!recovered)
|
|
741
|
+
continue;
|
|
742
|
+
const validated = withScopeLock(dir, () => validateOptimisticRecovery(dir, scopeDigest, reduce, candidate, recovered));
|
|
743
|
+
if (validated)
|
|
744
|
+
return validated;
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
async function readNewestGenerationOptimisticallyAsync(dir, scopeDigest, reduce, onProgress) {
|
|
748
|
+
let resamples = 0;
|
|
749
|
+
while (true) {
|
|
750
|
+
const candidate = await withScopeLockAsync(dir, () => {
|
|
751
|
+
cleanupUnpublishedInitialGeneration(dir);
|
|
752
|
+
return highestCompleteGenerationOnDisk(dir);
|
|
753
|
+
});
|
|
754
|
+
if (candidate === undefined)
|
|
755
|
+
return undefined;
|
|
756
|
+
let recovered;
|
|
757
|
+
try {
|
|
758
|
+
recovered = readNewestGeneration(dir, scopeDigest, reduce, onProgress, false);
|
|
759
|
+
}
|
|
760
|
+
catch (error) {
|
|
761
|
+
if (!(error instanceof StateStoreGenerationMissingError))
|
|
762
|
+
throw error;
|
|
763
|
+
const freshCandidate = await withScopeLockAsync(dir, () => highestCompleteGenerationOnDisk(dir));
|
|
764
|
+
if (freshCandidate === candidate)
|
|
765
|
+
throw error;
|
|
766
|
+
if (resamples++ >= OPTIMISTIC_RECOVERY_MAX_RESAMPLES) {
|
|
767
|
+
throw new StateStoreLockError("generation changed repeatedly during optimistic recovery", { cause: error });
|
|
768
|
+
}
|
|
769
|
+
continue;
|
|
770
|
+
}
|
|
771
|
+
if (!recovered)
|
|
772
|
+
continue;
|
|
773
|
+
const validated = await withScopeLockAsync(dir, () => validateOptimisticRecovery(dir, scopeDigest, reduce, candidate, recovered));
|
|
774
|
+
if (validated)
|
|
775
|
+
return validated;
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
function validateOptimisticRecovery(dir, scopeDigest, reduce, candidate, recovered) {
|
|
779
|
+
cleanupUnpublishedInitialGeneration(dir);
|
|
780
|
+
if (highestCompleteGenerationOnDisk(dir) !== candidate)
|
|
781
|
+
return undefined;
|
|
782
|
+
const observedWalBytes = recovered.observedWalBytes ?? recovered.walBytes;
|
|
783
|
+
const filePath = walPath(dir, recovered.generation);
|
|
784
|
+
const currentWalBytes = generationFileSize(filePath, "WAL", recovered.generation);
|
|
785
|
+
if (recovered.tornWalTailBoundary !== undefined) {
|
|
786
|
+
if (currentWalBytes !== observedWalBytes)
|
|
787
|
+
return undefined;
|
|
788
|
+
truncateWal(filePath, recovered.tornWalTailBoundary);
|
|
789
|
+
return {
|
|
790
|
+
...recovered,
|
|
791
|
+
observedWalBytes: recovered.tornWalTailBoundary,
|
|
792
|
+
tornWalTailBoundary: undefined,
|
|
793
|
+
};
|
|
794
|
+
}
|
|
795
|
+
if (currentWalBytes < observedWalBytes)
|
|
796
|
+
return undefined;
|
|
797
|
+
if (currentWalBytes === observedWalBytes)
|
|
798
|
+
return recovered;
|
|
799
|
+
const tail = (() => {
|
|
800
|
+
try {
|
|
801
|
+
return readWalTail(filePath, scopeDigest, recovered.generation, recovered.walBytes, recovered.lastRecordSequence, recovered.state, reduce);
|
|
802
|
+
}
|
|
803
|
+
catch (error) {
|
|
804
|
+
// A complete but invalid concurrent suffix makes this generation invalid;
|
|
805
|
+
// replay selection below can still choose a retained valid generation.
|
|
806
|
+
if (error instanceof StateStoreCorruptionError)
|
|
807
|
+
return undefined;
|
|
808
|
+
throw error;
|
|
809
|
+
}
|
|
810
|
+
})();
|
|
811
|
+
if (!tail)
|
|
812
|
+
return undefined;
|
|
813
|
+
return {
|
|
814
|
+
...recovered,
|
|
815
|
+
state: tail.state,
|
|
816
|
+
lastRecordSequence: tail.lastRecordSequence,
|
|
817
|
+
recordCount: recovered.recordCount + tail.recordCount,
|
|
818
|
+
walBytes: tail.walBytes,
|
|
819
|
+
maxReadBytes: Math.max(recovered.maxReadBytes, tail.maxReadBytes),
|
|
820
|
+
observedWalBytes: tail.walBytes,
|
|
409
821
|
};
|
|
410
822
|
}
|
|
823
|
+
function needsMaintenance(dir, newestGeneration, options) {
|
|
824
|
+
let walBytes;
|
|
825
|
+
let snapshotBytes;
|
|
826
|
+
const currentGeneration = currentGenerationOnDisk(dir);
|
|
827
|
+
try {
|
|
828
|
+
walBytes = fs.statSync(walPath(dir, newestGeneration)).size;
|
|
829
|
+
snapshotBytes = fs.statSync(snapshotPath(dir, newestGeneration)).size;
|
|
830
|
+
}
|
|
831
|
+
catch {
|
|
832
|
+
// Structural recovery below classifies a missing or incomplete generation;
|
|
833
|
+
// it must not be mistaken for a healthy no-op.
|
|
834
|
+
}
|
|
835
|
+
return walBytes === undefined ||
|
|
836
|
+
snapshotBytes === undefined ||
|
|
837
|
+
retainedGenerationHistoryNeedsRepair(dir, newestGeneration, currentGeneration) ||
|
|
838
|
+
walBytes > options.maxWalBytesBeforeCompaction;
|
|
839
|
+
}
|
|
411
840
|
function writeGeneration(dir, scopeDigest, generation) {
|
|
412
841
|
const snapshot = encodeSnapshot(generation.generation, scopeDigest, {
|
|
413
842
|
lastRecordSequence: generation.lastRecordSequence,
|
|
@@ -418,6 +847,130 @@ function writeGeneration(dir, scopeDigest, generation) {
|
|
|
418
847
|
const header = encodeWalHeader(generation.generation, scopeDigest);
|
|
419
848
|
writeNewDurably(walPath(dir, generation.generation), header);
|
|
420
849
|
}
|
|
850
|
+
/**
|
|
851
|
+
* Build both files for a generation before taking append.lock for publication.
|
|
852
|
+
* Temporary names never match the recovery generation pattern, so a process
|
|
853
|
+
* crash here leaves the prior complete generation authoritative.
|
|
854
|
+
*/
|
|
855
|
+
function stageGeneration(dir, scopeDigest, generation) {
|
|
856
|
+
snapshotMaterializationCount++;
|
|
857
|
+
const snapshot = encodeSnapshot(generation.generation, scopeDigest, {
|
|
858
|
+
lastRecordSequence: generation.lastRecordSequence,
|
|
859
|
+
state: generation.state,
|
|
860
|
+
});
|
|
861
|
+
let snapshotTemporary;
|
|
862
|
+
let walTemporary;
|
|
863
|
+
try {
|
|
864
|
+
snapshotTemporary = writeDurableTemporary(dir, `snapshot-${generation.generation}`, snapshot);
|
|
865
|
+
walTemporary = writeDurableTemporary(dir, `wal-${generation.generation}`, encodeWalHeader(generation.generation, scopeDigest));
|
|
866
|
+
return { generation, snapshotTemporary, walTemporary };
|
|
867
|
+
}
|
|
868
|
+
catch (error) {
|
|
869
|
+
if (snapshotTemporary !== undefined)
|
|
870
|
+
try {
|
|
871
|
+
fs.rmSync(snapshotTemporary, { force: true });
|
|
872
|
+
}
|
|
873
|
+
catch { /* preserve staging error */ }
|
|
874
|
+
if (walTemporary !== undefined)
|
|
875
|
+
try {
|
|
876
|
+
fs.rmSync(walTemporary, { force: true });
|
|
877
|
+
}
|
|
878
|
+
catch { /* preserve staging error */ }
|
|
879
|
+
throw error;
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
/**
|
|
883
|
+
* Make the rollback copy without serialising or writing the snapshot payload a
|
|
884
|
+
* second time when the store directory is on a reflink-capable filesystem.
|
|
885
|
+
*/
|
|
886
|
+
function stageGenerationFromSnapshot(dir, scopeDigest, sourceSnapshot, generation) {
|
|
887
|
+
const snapshotTemporary = temporaryPath(dir, `snapshot-${generation.generation}`);
|
|
888
|
+
let walTemporary;
|
|
889
|
+
try {
|
|
890
|
+
try {
|
|
891
|
+
reflinkCopyHookForTest?.();
|
|
892
|
+
fs.copyFileSync(sourceSnapshot, snapshotTemporary, fs.constants.COPYFILE_FICLONE_FORCE);
|
|
893
|
+
rewriteStagedSnapshotGeneration(snapshotTemporary, generation.generation);
|
|
894
|
+
}
|
|
895
|
+
catch {
|
|
896
|
+
// Reflinks are an optimisation, not a storage prerequisite. Reuse the
|
|
897
|
+
// bytes already encoded for the first snapshot on filesystems without
|
|
898
|
+
// clone support rather than materialising the state a second time.
|
|
899
|
+
if (fs.existsSync(snapshotTemporary))
|
|
900
|
+
fs.rmSync(snapshotTemporary, { force: true });
|
|
901
|
+
fs.copyFileSync(sourceSnapshot, snapshotTemporary);
|
|
902
|
+
rewriteStagedSnapshotGeneration(snapshotTemporary, generation.generation);
|
|
903
|
+
}
|
|
904
|
+
walTemporary = writeDurableTemporary(dir, `wal-${generation.generation}`, encodeWalHeader(generation.generation, scopeDigest));
|
|
905
|
+
return { generation, snapshotTemporary, walTemporary };
|
|
906
|
+
}
|
|
907
|
+
catch (error) {
|
|
908
|
+
try {
|
|
909
|
+
fs.rmSync(snapshotTemporary, { force: true });
|
|
910
|
+
}
|
|
911
|
+
catch { /* preserve staging error */ }
|
|
912
|
+
if (walTemporary !== undefined)
|
|
913
|
+
try {
|
|
914
|
+
fs.rmSync(walTemporary, { force: true });
|
|
915
|
+
}
|
|
916
|
+
catch { /* preserve staging error */ }
|
|
917
|
+
throw error;
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
function rewriteStagedSnapshotGeneration(filePath, generation) {
|
|
921
|
+
let fd;
|
|
922
|
+
try {
|
|
923
|
+
fd = fs.openSync(filePath, "r+");
|
|
924
|
+
const header = Buffer.alloc(52);
|
|
925
|
+
if (readIntoSync(fd, header, 0) !== header.length)
|
|
926
|
+
throw new Error("state-store: cloned snapshot header is torn");
|
|
927
|
+
writeU64(header, generation, 8);
|
|
928
|
+
writeAllAtSync(fd, header, 0);
|
|
929
|
+
const payloadLength = header.readUInt32BE(16);
|
|
930
|
+
const hash = crypto.createHash("sha256").update(header);
|
|
931
|
+
const chunk = Buffer.allocUnsafe(Math.min(payloadLength, 1024 * 1024));
|
|
932
|
+
let offset = 0;
|
|
933
|
+
while (offset < payloadLength) {
|
|
934
|
+
const length = Math.min(chunk.length, payloadLength - offset);
|
|
935
|
+
if (readIntoSync(fd, chunk.subarray(0, length), 52 + offset) !== length) {
|
|
936
|
+
throw new Error("state-store: cloned snapshot payload is torn");
|
|
937
|
+
}
|
|
938
|
+
hash.update(chunk.subarray(0, length));
|
|
939
|
+
offset += length;
|
|
940
|
+
}
|
|
941
|
+
const checksum = hash.digest();
|
|
942
|
+
writeAllAtSync(fd, checksum, 52 + payloadLength);
|
|
943
|
+
fs.fdatasyncSync(fd);
|
|
944
|
+
}
|
|
945
|
+
catch (error) {
|
|
946
|
+
throw new Error(`state-store: failed to prepare cloned snapshot generation ${generation}`, { cause: error });
|
|
947
|
+
}
|
|
948
|
+
finally {
|
|
949
|
+
if (fd !== undefined)
|
|
950
|
+
fs.closeSync(fd);
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
/** Publish already-fsynced files; this is the only snapshot-rotation I/O under the lock. */
|
|
954
|
+
function publishStagedGeneration(dir, staged) {
|
|
955
|
+
try {
|
|
956
|
+
fs.renameSync(staged.snapshotTemporary, snapshotPath(dir, staged.generation.generation));
|
|
957
|
+
fs.renameSync(staged.walTemporary, walPath(dir, staged.generation.generation));
|
|
958
|
+
fsyncDirectory(dir);
|
|
959
|
+
}
|
|
960
|
+
catch (error) {
|
|
961
|
+
throw new Error(`state-store: staged generation publication failed for ${staged.generation.generation}`, { cause: error });
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
function discardStagedGeneration(staged) {
|
|
965
|
+
try {
|
|
966
|
+
fs.rmSync(staged.snapshotTemporary, { force: true });
|
|
967
|
+
}
|
|
968
|
+
catch { /* staged files are best-effort cleanup */ }
|
|
969
|
+
try {
|
|
970
|
+
fs.rmSync(staged.walTemporary, { force: true });
|
|
971
|
+
}
|
|
972
|
+
catch { /* staged files are best-effort cleanup */ }
|
|
973
|
+
}
|
|
421
974
|
function readSnapshot(filePath, scopeDigest, generation) {
|
|
422
975
|
snapshotRecoveryCount++;
|
|
423
976
|
let fd;
|
|
@@ -426,7 +979,7 @@ function readSnapshot(filePath, scopeDigest, generation) {
|
|
|
426
979
|
}
|
|
427
980
|
catch (error) {
|
|
428
981
|
if (isMissing(error)) {
|
|
429
|
-
throw new
|
|
982
|
+
throw new StateStoreGenerationMissingError(`snapshot generation ${generation} is missing`, { cause: error });
|
|
430
983
|
}
|
|
431
984
|
throw new Error(`state-store: cannot read snapshot generation ${generation}`, { cause: error });
|
|
432
985
|
}
|
|
@@ -449,6 +1002,7 @@ function readSnapshot(filePath, scopeDigest, generation) {
|
|
|
449
1002
|
if (!header.subarray(20, 52).equals(scopeDigest)) {
|
|
450
1003
|
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a scope digest mismatch`);
|
|
451
1004
|
}
|
|
1005
|
+
snapshotReadHookForTest?.();
|
|
452
1006
|
const data = Buffer.allocUnsafe(payloadLength);
|
|
453
1007
|
if (readIntoSync(fd, data, 52) !== data.length) {
|
|
454
1008
|
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn payload`);
|
|
@@ -482,20 +1036,21 @@ function readSnapshot(filePath, scopeDigest, generation) {
|
|
|
482
1036
|
fs.closeSync(fd);
|
|
483
1037
|
}
|
|
484
1038
|
}
|
|
485
|
-
function readWal(filePath, scopeDigest, generation, snapshotSequence, initialState, reduce, recoveryBytesBeforeWal = 0, recoveryTotalBytes, onProgress) {
|
|
1039
|
+
function readWal(filePath, scopeDigest, generation, snapshotSequence, initialState, reduce, recoveryBytesBeforeWal = 0, recoveryTotalBytes, onProgress, repairTornTail = true) {
|
|
486
1040
|
let fd;
|
|
487
1041
|
try {
|
|
488
1042
|
fd = fs.openSync(filePath, "r");
|
|
489
1043
|
}
|
|
490
1044
|
catch (error) {
|
|
491
1045
|
if (isMissing(error)) {
|
|
492
|
-
throw new
|
|
1046
|
+
throw new StateStoreGenerationMissingError(`WAL generation ${generation} is missing`, { cause: error });
|
|
493
1047
|
}
|
|
494
1048
|
throw new Error(`state-store: cannot read WAL generation ${generation}`, { cause: error });
|
|
495
1049
|
}
|
|
496
1050
|
let truncateBoundary;
|
|
497
1051
|
try {
|
|
498
1052
|
const walBytes = fs.fstatSync(fd).size;
|
|
1053
|
+
walReadHookForTest?.();
|
|
499
1054
|
const header = Buffer.alloc(WAL_HEADER_BYTES);
|
|
500
1055
|
const headerBytes = readIntoSync(fd, header, 0);
|
|
501
1056
|
if (headerBytes < WAL_HEADER_BYTES) {
|
|
@@ -572,6 +1127,8 @@ function readWal(filePath, scopeDigest, generation, snapshotSequence, initialSta
|
|
|
572
1127
|
lastRecordSequence: sequence,
|
|
573
1128
|
recordCount,
|
|
574
1129
|
walBytes: truncateBoundary ?? walBytes,
|
|
1130
|
+
observedWalBytes: walBytes,
|
|
1131
|
+
tornWalTailBoundary: truncateBoundary,
|
|
575
1132
|
maxReadBytes,
|
|
576
1133
|
};
|
|
577
1134
|
}
|
|
@@ -583,7 +1140,7 @@ function readWal(filePath, scopeDigest, generation, snapshotSequence, initialSta
|
|
|
583
1140
|
}
|
|
584
1141
|
finally {
|
|
585
1142
|
fs.closeSync(fd);
|
|
586
|
-
if (truncateBoundary !== undefined)
|
|
1143
|
+
if (repairTornTail && truncateBoundary !== undefined)
|
|
587
1144
|
truncateWal(filePath, truncateBoundary);
|
|
588
1145
|
}
|
|
589
1146
|
}
|
|
@@ -759,6 +1316,7 @@ function decodeFrame(bytes, offset) {
|
|
|
759
1316
|
};
|
|
760
1317
|
}
|
|
761
1318
|
function encodeSnapshot(generation, scopeDigest, payload) {
|
|
1319
|
+
snapshotEncodeCount++;
|
|
762
1320
|
const data = Buffer.from(canonicalJson(payload), "utf8");
|
|
763
1321
|
const header = Buffer.alloc(52);
|
|
764
1322
|
SNAPSHOT_MAGIC.copy(header, 0);
|
|
@@ -845,6 +1403,35 @@ function writeDurableTempThenRename(dir, destination, bytes) {
|
|
|
845
1403
|
throw new Error(`state-store: snapshot write/rename failed for ${destination}`, { cause: error });
|
|
846
1404
|
}
|
|
847
1405
|
}
|
|
1406
|
+
function writeDurableTemporary(dir, label, bytes) {
|
|
1407
|
+
const temporary = temporaryPath(dir, label);
|
|
1408
|
+
writeDurableTemporaryAt(temporary, bytes, label);
|
|
1409
|
+
return temporary;
|
|
1410
|
+
}
|
|
1411
|
+
function temporaryPath(dir, label) {
|
|
1412
|
+
return path.join(dir, `.${label}.${process.pid}.${crypto.randomBytes(6).toString("hex")}.tmp`);
|
|
1413
|
+
}
|
|
1414
|
+
function writeDurableTemporaryAt(temporary, bytes, label) {
|
|
1415
|
+
let fd;
|
|
1416
|
+
try {
|
|
1417
|
+
fd = fs.openSync(temporary, "wx");
|
|
1418
|
+
if (label.startsWith("snapshot-"))
|
|
1419
|
+
snapshotWriteHookForTest?.();
|
|
1420
|
+
writeAllSync(fd, bytes);
|
|
1421
|
+
fs.fdatasyncSync(fd);
|
|
1422
|
+
}
|
|
1423
|
+
catch (error) {
|
|
1424
|
+
try {
|
|
1425
|
+
fs.rmSync(temporary, { force: true });
|
|
1426
|
+
}
|
|
1427
|
+
catch { /* preserve durable-write error */ }
|
|
1428
|
+
throw new Error(`state-store: staged write failed for ${label}`, { cause: error });
|
|
1429
|
+
}
|
|
1430
|
+
finally {
|
|
1431
|
+
if (fd !== undefined)
|
|
1432
|
+
fs.closeSync(fd);
|
|
1433
|
+
}
|
|
1434
|
+
}
|
|
848
1435
|
function writeCurrentPointer(dir, generation) {
|
|
849
1436
|
const pointer = path.join(dir, "current");
|
|
850
1437
|
writeDurableTempThenRename(dir, pointer, Buffer.from(`${generation}\n`, "ascii"));
|
|
@@ -858,6 +1445,18 @@ function writeAllSync(fd, bytes) {
|
|
|
858
1445
|
offset += written;
|
|
859
1446
|
}
|
|
860
1447
|
}
|
|
1448
|
+
function writeAllAtSync(fd, bytes, position) {
|
|
1449
|
+
let offset = 0;
|
|
1450
|
+
while (offset < bytes.length) {
|
|
1451
|
+
const length = bytes.length - offset;
|
|
1452
|
+
const written = positionalWriteForTest
|
|
1453
|
+
? positionalWriteForTest(fd, bytes, offset, length, position + offset)
|
|
1454
|
+
: fs.writeSync(fd, bytes, offset, length, position + offset);
|
|
1455
|
+
if (written <= 0)
|
|
1456
|
+
throw new Error("state-store: short positional write made no progress");
|
|
1457
|
+
offset += written;
|
|
1458
|
+
}
|
|
1459
|
+
}
|
|
861
1460
|
function readIntoSync(fd, target, position) {
|
|
862
1461
|
let offset = 0;
|
|
863
1462
|
while (offset < target.length) {
|
|
@@ -911,6 +1510,73 @@ function withScopeLock(dir, action) {
|
|
|
911
1510
|
throw actionError;
|
|
912
1511
|
return result;
|
|
913
1512
|
}
|
|
1513
|
+
/**
|
|
1514
|
+
* A best-effort, non-blocking claim for off-lock snapshot materialisation.
|
|
1515
|
+
* Appenders deliberately ignore it; it only prevents several stale handles
|
|
1516
|
+
* from concurrently staging the same logical rotation.
|
|
1517
|
+
*/
|
|
1518
|
+
function withRotationClaim(dir, action) {
|
|
1519
|
+
const claimPath = path.join(dir, "rotation.lock");
|
|
1520
|
+
let fd;
|
|
1521
|
+
for (let attempt = 0; attempt < 2 && fd === undefined; attempt++) {
|
|
1522
|
+
try {
|
|
1523
|
+
fd = fs.openSync(claimPath, "wx");
|
|
1524
|
+
rotationClaimWriteHookForTest?.();
|
|
1525
|
+
fs.writeFileSync(fd, String(process.pid));
|
|
1526
|
+
fs.fdatasyncSync(fd);
|
|
1527
|
+
}
|
|
1528
|
+
catch (error) {
|
|
1529
|
+
if (fd !== undefined) {
|
|
1530
|
+
let closeError;
|
|
1531
|
+
try {
|
|
1532
|
+
fs.closeSync(fd);
|
|
1533
|
+
}
|
|
1534
|
+
catch (cleanupError) {
|
|
1535
|
+
closeError = cleanupError;
|
|
1536
|
+
}
|
|
1537
|
+
finally {
|
|
1538
|
+
fd = undefined;
|
|
1539
|
+
}
|
|
1540
|
+
try {
|
|
1541
|
+
fs.unlinkSync(claimPath);
|
|
1542
|
+
}
|
|
1543
|
+
catch (cleanupError) {
|
|
1544
|
+
if (!isMissing(cleanupError)) {
|
|
1545
|
+
throw new StateStoreLockError(`cannot clean up ${claimPath}`, { cause: cleanupError });
|
|
1546
|
+
}
|
|
1547
|
+
}
|
|
1548
|
+
if (closeError !== undefined) {
|
|
1549
|
+
throw new StateStoreLockError(`cannot close ${claimPath}`, { cause: closeError });
|
|
1550
|
+
}
|
|
1551
|
+
return undefined;
|
|
1552
|
+
}
|
|
1553
|
+
if (!isAlreadyExists(error) || !removeDeadLock(claimPath))
|
|
1554
|
+
return undefined;
|
|
1555
|
+
}
|
|
1556
|
+
}
|
|
1557
|
+
if (fd === undefined)
|
|
1558
|
+
return undefined;
|
|
1559
|
+
let result;
|
|
1560
|
+
let actionError;
|
|
1561
|
+
try {
|
|
1562
|
+
result = action();
|
|
1563
|
+
}
|
|
1564
|
+
catch (error) {
|
|
1565
|
+
actionError = error;
|
|
1566
|
+
}
|
|
1567
|
+
finally {
|
|
1568
|
+
fs.closeSync(fd);
|
|
1569
|
+
}
|
|
1570
|
+
try {
|
|
1571
|
+
fs.unlinkSync(claimPath);
|
|
1572
|
+
}
|
|
1573
|
+
catch (error) {
|
|
1574
|
+
throw new Error(`state-store: cannot release ${claimPath}`, { cause: error });
|
|
1575
|
+
}
|
|
1576
|
+
if (actionError !== undefined)
|
|
1577
|
+
throw actionError;
|
|
1578
|
+
return result;
|
|
1579
|
+
}
|
|
914
1580
|
/**
|
|
915
1581
|
* Acquire the same inter-process lock without parking the Node event loop
|
|
916
1582
|
* while another writer owns it. This is deliberately additive: pass-side
|
|
@@ -968,7 +1634,7 @@ function removeDeadLock(lockPath) {
|
|
|
968
1634
|
return isMissing(error);
|
|
969
1635
|
}
|
|
970
1636
|
if (!Number.isSafeInteger(pid) || pid <= 0)
|
|
971
|
-
return
|
|
1637
|
+
return removeMalformedLock(lockPath);
|
|
972
1638
|
try {
|
|
973
1639
|
process.kill(pid, 0);
|
|
974
1640
|
return false;
|
|
@@ -996,7 +1662,7 @@ async function removeDeadLockAsync(lockPath) {
|
|
|
996
1662
|
return false;
|
|
997
1663
|
}
|
|
998
1664
|
if (!Number.isSafeInteger(pid) || pid <= 0)
|
|
999
|
-
return
|
|
1665
|
+
return removeMalformedLockAsync(lockPath);
|
|
1000
1666
|
try {
|
|
1001
1667
|
process.kill(pid, 0);
|
|
1002
1668
|
return false;
|
|
@@ -1015,6 +1681,24 @@ async function removeDeadLockAsync(lockPath) {
|
|
|
1015
1681
|
return false;
|
|
1016
1682
|
}
|
|
1017
1683
|
}
|
|
1684
|
+
function removeMalformedLock(lockPath) {
|
|
1685
|
+
try {
|
|
1686
|
+
fs.unlinkSync(lockPath);
|
|
1687
|
+
return true;
|
|
1688
|
+
}
|
|
1689
|
+
catch (error) {
|
|
1690
|
+
return isMissing(error);
|
|
1691
|
+
}
|
|
1692
|
+
}
|
|
1693
|
+
async function removeMalformedLockAsync(lockPath) {
|
|
1694
|
+
try {
|
|
1695
|
+
await fs.promises.unlink(lockPath);
|
|
1696
|
+
return true;
|
|
1697
|
+
}
|
|
1698
|
+
catch (error) {
|
|
1699
|
+
return isMissing(error);
|
|
1700
|
+
}
|
|
1701
|
+
}
|
|
1018
1702
|
function truncateWal(filePath, boundary) {
|
|
1019
1703
|
let fd;
|
|
1020
1704
|
try {
|
|
@@ -1071,7 +1755,7 @@ function generationFileSize(filePath, kind, generation) {
|
|
|
1071
1755
|
}
|
|
1072
1756
|
catch (error) {
|
|
1073
1757
|
if (isMissing(error)) {
|
|
1074
|
-
throw new
|
|
1758
|
+
throw new StateStoreGenerationMissingError(`${kind} generation ${generation} is missing`, {
|
|
1075
1759
|
cause: error,
|
|
1076
1760
|
});
|
|
1077
1761
|
}
|
|
@@ -1090,6 +1774,9 @@ function hasSnapshotGeneration(dir) {
|
|
|
1090
1774
|
throw new Error(`state-store: cannot inspect ${dir}`, { cause: error });
|
|
1091
1775
|
}
|
|
1092
1776
|
}
|
|
1777
|
+
function hasStateStoreArtifacts(dir) {
|
|
1778
|
+
return fs.readdirSync(dir).some((name) => name === "current" || /^(?:snapshot|wal)-\d+\.bin$/.test(name));
|
|
1779
|
+
}
|
|
1093
1780
|
function snapshotPath(dir, generation) {
|
|
1094
1781
|
return path.join(dir, `snapshot-${generation}.bin`);
|
|
1095
1782
|
}
|
|
@@ -1112,6 +1799,21 @@ function highestSnapshotGenerationOnDisk(dir) {
|
|
|
1112
1799
|
.filter(Number.isSafeInteger);
|
|
1113
1800
|
return generations.length === 0 ? undefined : Math.max(...generations);
|
|
1114
1801
|
}
|
|
1802
|
+
/** The highest published generation: both immutable snapshot and WAL exist. */
|
|
1803
|
+
function highestCompleteGenerationOnDisk(dir) {
|
|
1804
|
+
const snapshots = new Set();
|
|
1805
|
+
const wals = new Set();
|
|
1806
|
+
for (const name of fs.readdirSync(dir)) {
|
|
1807
|
+
const snapshot = /^snapshot-(\d+)\.bin$/.exec(name);
|
|
1808
|
+
if (snapshot)
|
|
1809
|
+
snapshots.add(Number(snapshot[1]));
|
|
1810
|
+
const wal = /^wal-(\d+)\.bin$/.exec(name);
|
|
1811
|
+
if (wal)
|
|
1812
|
+
wals.add(Number(wal[1]));
|
|
1813
|
+
}
|
|
1814
|
+
const generations = [...snapshots].filter((generation) => wals.has(generation));
|
|
1815
|
+
return generations.length === 0 ? undefined : Math.max(...generations);
|
|
1816
|
+
}
|
|
1115
1817
|
function currentGenerationOnDisk(dir) {
|
|
1116
1818
|
try {
|
|
1117
1819
|
const generation = Number(fs.readFileSync(path.join(dir, "current"), "ascii").trim());
|