@indigoai-us/hq-cloud 6.16.34 → 6.16.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +20 -8
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.test.js +1 -2
- package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
- package/dist/bin/sync-runner.test.js +21 -24
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +18 -7
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +30 -3
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +3 -2
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +6 -6
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/company-vault-transport.d.ts +0 -2
- package/dist/company-vault-transport.d.ts.map +1 -1
- package/dist/company-vault-transport.js +0 -2
- package/dist/company-vault-transport.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal-baseline.test.js +18 -15
- package/dist/journal-baseline.test.js.map +1 -1
- package/dist/journal-delta-writes.test.js +2 -2
- package/dist/journal-delta-writes.test.js.map +1 -1
- package/dist/journal-public-boundary.test.d.ts +2 -0
- package/dist/journal-public-boundary.test.d.ts.map +1 -0
- package/dist/journal-public-boundary.test.js +216 -0
- package/dist/journal-public-boundary.test.js.map +1 -0
- package/dist/journal-record-proxy.d.ts +11 -0
- package/dist/journal-record-proxy.d.ts.map +1 -0
- package/dist/journal-record-proxy.js +71 -0
- package/dist/journal-record-proxy.js.map +1 -0
- package/dist/journal-row-store.d.ts +76 -3
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +450 -43
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-row-store.test.js +139 -6
- package/dist/journal-row-store.test.js.map +1 -1
- package/dist/journal-snapshot-decoder.d.ts +18 -0
- package/dist/journal-snapshot-decoder.d.ts.map +1 -0
- package/dist/journal-snapshot-decoder.js +373 -0
- package/dist/journal-snapshot-decoder.js.map +1 -0
- package/dist/journal-snapshot-memory-helpers.d.ts +30 -0
- package/dist/journal-snapshot-memory-helpers.d.ts.map +1 -0
- package/dist/journal-snapshot-memory-helpers.js +203 -0
- package/dist/journal-snapshot-memory-helpers.js.map +1 -0
- package/dist/journal-snapshot-memory.test.d.ts +2 -0
- package/dist/journal-snapshot-memory.test.d.ts.map +1 -0
- package/dist/journal-snapshot-memory.test.js +32 -0
- package/dist/journal-snapshot-memory.test.js.map +1 -0
- package/dist/journal-snapshot-v4-memory.test.d.ts +2 -0
- package/dist/journal-snapshot-v4-memory.test.d.ts.map +1 -0
- package/dist/journal-snapshot-v4-memory.test.js +24 -0
- package/dist/journal-snapshot-v4-memory.test.js.map +1 -0
- package/dist/journal-snapshot-v4.d.ts +52 -0
- package/dist/journal-snapshot-v4.d.ts.map +1 -0
- package/dist/journal-snapshot-v4.js +352 -0
- package/dist/journal-snapshot-v4.js.map +1 -0
- package/dist/journal-snapshot-v4.test.d.ts +2 -0
- package/dist/journal-snapshot-v4.test.d.ts.map +1 -0
- package/dist/journal-snapshot-v4.test.js +280 -0
- package/dist/journal-snapshot-v4.test.js.map +1 -0
- package/dist/journal-store.test.js +335 -17
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +10 -8
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +165 -50
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +1 -1
- package/dist/journal.test.js.map +1 -1
- package/dist/manifest/build-manifest.bench.test.js +165 -58
- package/dist/manifest/build-manifest.bench.test.js.map +1 -1
- package/dist/remote-pull.js +6 -4
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +72 -0
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/session-log-retention.test.js +3 -3
- package/dist/session-log-retention.test.js.map +1 -1
- package/dist/skill-telemetry.d.ts +1 -2
- package/dist/skill-telemetry.d.ts.map +1 -1
- package/dist/skill-telemetry.js +1 -10
- package/dist/skill-telemetry.js.map +1 -1
- package/dist/sync/child-process-sync-worker-env.test.d.ts +2 -0
- package/dist/sync/child-process-sync-worker-env.test.d.ts.map +1 -0
- package/dist/sync/child-process-sync-worker-env.test.js +38 -0
- package/dist/sync/child-process-sync-worker-env.test.js.map +1 -0
- package/dist/sync/feature-flags.d.ts +0 -16
- package/dist/sync/feature-flags.d.ts.map +1 -1
- package/dist/sync/feature-flags.js +0 -11
- package/dist/sync/feature-flags.js.map +1 -1
- package/dist/sync/index.d.ts +2 -2
- package/dist/sync/index.d.ts.map +1 -1
- package/dist/sync/index.js +1 -1
- package/dist/sync/index.js.map +1 -1
- package/dist/sync/logger.test.js +1 -3
- package/dist/sync/logger.test.js.map +1 -1
- package/dist/sync/metrics.test.js +0 -3
- package/dist/sync/metrics.test.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +0 -7
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/state-store.d.ts +23 -0
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +114 -39
- package/dist/sync/state-store.js.map +1 -1
- package/dist/sync/state-store.test.js +82 -0
- package/dist/sync/state-store.test.js.map +1 -1
- package/dist/watcher.test.js +7 -8
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -1
- package/dist/daemon-worker.d.ts +0 -11
- package/dist/daemon-worker.d.ts.map +0 -1
- package/dist/daemon-worker.js +0 -22
- package/dist/daemon-worker.js.map +0 -1
package/dist/sync/state-store.js
CHANGED
|
@@ -8,7 +8,10 @@
|
|
|
8
8
|
import * as crypto from "node:crypto";
|
|
9
9
|
import * as fs from "node:fs";
|
|
10
10
|
import * as path from "node:path";
|
|
11
|
-
|
|
11
|
+
/** The legacy JSON snapshot format emitted by this release's writer. */
|
|
12
|
+
export const SNAPSHOT_V3_MAGIC = Buffer.from("HQSNAP3\0", "ascii");
|
|
13
|
+
/** The packed journal snapshot format accepted by the reader before it is emitted. */
|
|
14
|
+
export const SNAPSHOT_V4_MAGIC = Buffer.from("HQSNAP4\0", "ascii");
|
|
12
15
|
const WAL_MAGIC = Buffer.from("HQWAL3\0\0", "ascii");
|
|
13
16
|
const FRAME_MAGIC = 0x48515733;
|
|
14
17
|
const SNAPSHOT_HEADER_BYTES = 8 + 8 + 4 + 32 + 32;
|
|
@@ -74,6 +77,7 @@ let snapshotMaterializationCount = 0;
|
|
|
74
77
|
let snapshotEncodeCount = 0;
|
|
75
78
|
let snapshotWriteHookForTest;
|
|
76
79
|
let snapshotReadHookForTest;
|
|
80
|
+
let snapshotMemorySampleHookForTest;
|
|
77
81
|
let walReadHookForTest;
|
|
78
82
|
let afterGenerationStageHookForTest;
|
|
79
83
|
let reflinkCopyHookForTest;
|
|
@@ -122,6 +126,10 @@ export function setStateStoreSnapshotWriteHookForTest(hook) {
|
|
|
122
126
|
export function setStateStoreSnapshotReadHookForTest(hook) {
|
|
123
127
|
snapshotReadHookForTest = hook;
|
|
124
128
|
}
|
|
129
|
+
/** Test-only sampling seam before and after decoder work for each bounded snapshot chunk. */
|
|
130
|
+
export function setStateStoreSnapshotMemorySampleHookForTest(hook) {
|
|
131
|
+
snapshotMemorySampleHookForTest = hook;
|
|
132
|
+
}
|
|
125
133
|
/** Test-only seam after the WAL length is sampled during cold recovery. */
|
|
126
134
|
export function setStateStoreWalReadHookForTest(hook) {
|
|
127
135
|
walReadHookForTest = hook;
|
|
@@ -297,7 +305,7 @@ export class StateStore {
|
|
|
297
305
|
const dir = stateDirectory(normalized.rootDir, scopeDigest);
|
|
298
306
|
fs.mkdirSync(dir, { recursive: true });
|
|
299
307
|
while (true) {
|
|
300
|
-
const recovered = readNewestGenerationOptimistically(dir, scopeDigest, normalized.reduce);
|
|
308
|
+
const recovered = readNewestGenerationOptimistically(dir, scopeDigest, normalized.reduce, normalized.snapshotDecoder, normalized.snapshotMagics);
|
|
301
309
|
if (recovered) {
|
|
302
310
|
const store = new StateStore(normalized, recovered, false);
|
|
303
311
|
// A process can die after its durable append but before append() gets
|
|
@@ -337,7 +345,7 @@ export class StateStore {
|
|
|
337
345
|
const dir = stateDirectory(normalized.rootDir, scopeDigest);
|
|
338
346
|
await fs.promises.mkdir(dir, { recursive: true });
|
|
339
347
|
while (true) {
|
|
340
|
-
const recovered = await readNewestGenerationOptimisticallyAsync(dir, scopeDigest, normalized.reduce);
|
|
348
|
+
const recovered = await readNewestGenerationOptimisticallyAsync(dir, scopeDigest, normalized.reduce, normalized.snapshotDecoder, normalized.snapshotMagics);
|
|
341
349
|
if (recovered) {
|
|
342
350
|
const store = new StateStore(normalized, recovered, false);
|
|
343
351
|
// Match open(): a recovered over-budget WAL is compacted before this
|
|
@@ -471,7 +479,7 @@ export class StateStore {
|
|
|
471
479
|
// Preserve recovery's corruption classification for a crashed or
|
|
472
480
|
// malformed store that has snapshots but no complete generation.
|
|
473
481
|
if (hasSnapshotGeneration(dir)) {
|
|
474
|
-
readNewestGeneration(dir, scopeDigest, normalized.reduce, maintenance.onProgress, false);
|
|
482
|
+
readNewestGeneration(dir, scopeDigest, normalized.reduce, normalized.snapshotDecoder, normalized.snapshotMagics, maintenance.onProgress, false);
|
|
475
483
|
continue;
|
|
476
484
|
}
|
|
477
485
|
return { status: "missing" };
|
|
@@ -481,7 +489,7 @@ export class StateStore {
|
|
|
481
489
|
// Snapshot and WAL replay are read-only. Do not make appenders wait for
|
|
482
490
|
// this potentially multi-second materialisation; validate its view before
|
|
483
491
|
// using it to build a replacement generation.
|
|
484
|
-
const recovered = readNewestGeneration(dir, scopeDigest, normalized.reduce, maintenance.onProgress, false);
|
|
492
|
+
const recovered = readNewestGeneration(dir, scopeDigest, normalized.reduce, normalized.snapshotDecoder, normalized.snapshotMagics, maintenance.onProgress, false);
|
|
485
493
|
if (!recovered)
|
|
486
494
|
continue;
|
|
487
495
|
plan = withScopeLock(dir, () => {
|
|
@@ -1058,13 +1066,13 @@ export class StateStore {
|
|
|
1058
1066
|
return undefined;
|
|
1059
1067
|
}
|
|
1060
1068
|
recoverNewestGenerationOutsideLock() {
|
|
1061
|
-
const recovered = readNewestGenerationOptimistically(this.stateDir, this.scopeDigest, this.options.reduce);
|
|
1069
|
+
const recovered = readNewestGenerationOptimistically(this.stateDir, this.scopeDigest, this.options.reduce, this.options.snapshotDecoder, this.options.snapshotMagics);
|
|
1062
1070
|
if (!recovered)
|
|
1063
1071
|
throw new StateStoreCorruptionError("no valid generation remains while appending");
|
|
1064
1072
|
this.installGeneration(recovered);
|
|
1065
1073
|
}
|
|
1066
1074
|
async recoverNewestGenerationOutsideLockAsync() {
|
|
1067
|
-
const recovered = await readNewestGenerationOptimisticallyAsync(this.stateDir, this.scopeDigest, this.options.reduce);
|
|
1075
|
+
const recovered = await readNewestGenerationOptimisticallyAsync(this.stateDir, this.scopeDigest, this.options.reduce, this.options.snapshotDecoder, this.options.snapshotMagics);
|
|
1068
1076
|
if (!recovered)
|
|
1069
1077
|
throw new StateStoreCorruptionError("no valid generation remains while appending");
|
|
1070
1078
|
this.installGeneration(recovered);
|
|
@@ -1121,7 +1129,7 @@ export class StateStore {
|
|
|
1121
1129
|
this.observedHighestSnapshotGeneration = generation.observedHighestSnapshotGeneration ?? generation.generation;
|
|
1122
1130
|
}
|
|
1123
1131
|
}
|
|
1124
|
-
function readNewestGeneration(dir, scopeDigest, reduce, onProgress, repairTornTail = true) {
|
|
1132
|
+
function readNewestGeneration(dir, scopeDigest, reduce, snapshotDecoder, snapshotMagics, onProgress, repairTornTail = true) {
|
|
1125
1133
|
const names = fs.readdirSync(dir);
|
|
1126
1134
|
const generations = names
|
|
1127
1135
|
.map((name) => /^snapshot-(\d+)\.bin$/.exec(name)?.[1])
|
|
@@ -1134,7 +1142,7 @@ function readNewestGeneration(dir, scopeDigest, reduce, onProgress, repairTornTa
|
|
|
1134
1142
|
for (const generation of generations) {
|
|
1135
1143
|
try {
|
|
1136
1144
|
return {
|
|
1137
|
-
...readGeneration(dir, scopeDigest, generation, reduce, onProgress, repairTornTail),
|
|
1145
|
+
...readGeneration(dir, scopeDigest, generation, reduce, snapshotDecoder, snapshotMagics, onProgress, repairTornTail),
|
|
1138
1146
|
observedHighestSnapshotGeneration: generations[0],
|
|
1139
1147
|
};
|
|
1140
1148
|
}
|
|
@@ -1163,10 +1171,10 @@ function readNewestGeneration(dir, scopeDigest, reduce, onProgress, repairTornTa
|
|
|
1163
1171
|
}
|
|
1164
1172
|
return undefined;
|
|
1165
1173
|
}
|
|
1166
|
-
function readGeneration(dir, scopeDigest, generation, reduce, onProgress, repairTornTail = true) {
|
|
1174
|
+
function readGeneration(dir, scopeDigest, generation, reduce, snapshotDecoder, snapshotMagics, onProgress, repairTornTail = true) {
|
|
1167
1175
|
const snapshotBytes = generationFileSize(snapshotPath(dir, generation), "snapshot", generation);
|
|
1168
1176
|
const totalBytes = snapshotBytes + generationFileSize(walPath(dir, generation), "WAL", generation);
|
|
1169
|
-
const snapshot = readSnapshot(snapshotPath(dir, generation), scopeDigest, generation);
|
|
1177
|
+
const snapshot = readSnapshot(snapshotPath(dir, generation), scopeDigest, generation, snapshotDecoder, snapshotMagics);
|
|
1170
1178
|
onProgress?.({ bytesRead: snapshotBytes, totalBytes });
|
|
1171
1179
|
const parsed = readWal(walPath(dir, generation), scopeDigest, generation, snapshot.payload.lastRecordSequence, snapshot.payload.state, reduce, snapshotBytes, totalBytes, onProgress, repairTornTail);
|
|
1172
1180
|
return {
|
|
@@ -1187,7 +1195,7 @@ function readGeneration(dir, scopeDigest, generation, reduce, onProgress, repair
|
|
|
1187
1195
|
* WAL suffix rather than restarting the expensive immutable snapshot read.
|
|
1188
1196
|
* Rotation or a torn suffix remains lock-protected.
|
|
1189
1197
|
*/
|
|
1190
|
-
function readNewestGenerationOptimistically(dir, scopeDigest, reduce, onProgress) {
|
|
1198
|
+
function readNewestGenerationOptimistically(dir, scopeDigest, reduce, snapshotDecoder, snapshotMagics, onProgress) {
|
|
1191
1199
|
let resamples = 0;
|
|
1192
1200
|
while (true) {
|
|
1193
1201
|
const candidate = withScopeLock(dir, () => {
|
|
@@ -1198,7 +1206,7 @@ function readNewestGenerationOptimistically(dir, scopeDigest, reduce, onProgress
|
|
|
1198
1206
|
return undefined;
|
|
1199
1207
|
let recovered;
|
|
1200
1208
|
try {
|
|
1201
|
-
recovered = readNewestGeneration(dir, scopeDigest, reduce, onProgress, false);
|
|
1209
|
+
recovered = readNewestGeneration(dir, scopeDigest, reduce, snapshotDecoder, snapshotMagics, onProgress, false);
|
|
1202
1210
|
}
|
|
1203
1211
|
catch (error) {
|
|
1204
1212
|
if (!(error instanceof StateStoreGenerationMissingError))
|
|
@@ -1218,7 +1226,7 @@ function readNewestGenerationOptimistically(dir, scopeDigest, reduce, onProgress
|
|
|
1218
1226
|
return validated;
|
|
1219
1227
|
}
|
|
1220
1228
|
}
|
|
1221
|
-
async function readNewestGenerationOptimisticallyAsync(dir, scopeDigest, reduce, onProgress) {
|
|
1229
|
+
async function readNewestGenerationOptimisticallyAsync(dir, scopeDigest, reduce, snapshotDecoder, snapshotMagics, onProgress) {
|
|
1222
1230
|
let resamples = 0;
|
|
1223
1231
|
while (true) {
|
|
1224
1232
|
const candidate = await withScopeLockAsync(dir, () => {
|
|
@@ -1229,7 +1237,7 @@ async function readNewestGenerationOptimisticallyAsync(dir, scopeDigest, reduce,
|
|
|
1229
1237
|
return undefined;
|
|
1230
1238
|
let recovered;
|
|
1231
1239
|
try {
|
|
1232
|
-
recovered = readNewestGeneration(dir, scopeDigest, reduce, onProgress, false);
|
|
1240
|
+
recovered = readNewestGeneration(dir, scopeDigest, reduce, snapshotDecoder, snapshotMagics, onProgress, false);
|
|
1233
1241
|
}
|
|
1234
1242
|
catch (error) {
|
|
1235
1243
|
if (!(error instanceof StateStoreGenerationMissingError))
|
|
@@ -1445,7 +1453,85 @@ function discardStagedGeneration(staged) {
|
|
|
1445
1453
|
}
|
|
1446
1454
|
catch { /* staged files are best-effort cleanup */ }
|
|
1447
1455
|
}
|
|
1448
|
-
function
|
|
1456
|
+
function readSnapshotChecksum(fd, payloadOffset, payloadLength, header, generation) {
|
|
1457
|
+
const hash = crypto.createHash("sha256").update(header);
|
|
1458
|
+
let offset = 0;
|
|
1459
|
+
while (offset < payloadLength) {
|
|
1460
|
+
const length = Math.min(SNAPSHOT_CHUNK_BYTES, payloadLength - offset);
|
|
1461
|
+
const chunk = Buffer.allocUnsafe(length);
|
|
1462
|
+
if (readIntoSync(fd, chunk, payloadOffset + offset) !== length) {
|
|
1463
|
+
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn payload`);
|
|
1464
|
+
}
|
|
1465
|
+
hash.update(chunk);
|
|
1466
|
+
snapshotMemorySampleHookForTest?.();
|
|
1467
|
+
offset += length;
|
|
1468
|
+
}
|
|
1469
|
+
const checksum = Buffer.alloc(32);
|
|
1470
|
+
if (readIntoSync(fd, checksum, payloadOffset + payloadLength) !== checksum.length) {
|
|
1471
|
+
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn checksum`);
|
|
1472
|
+
}
|
|
1473
|
+
if (!hash.digest().equals(checksum)) {
|
|
1474
|
+
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a bad checksum`);
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1477
|
+
function readSnapshotPayloadChunks(fd, payloadOffset, payloadLength, generation) {
|
|
1478
|
+
return {
|
|
1479
|
+
byteLength: payloadLength,
|
|
1480
|
+
*[Symbol.iterator]() {
|
|
1481
|
+
let offset = 0;
|
|
1482
|
+
while (offset < payloadLength) {
|
|
1483
|
+
const length = Math.min(SNAPSHOT_CHUNK_BYTES, payloadLength - offset);
|
|
1484
|
+
const chunk = Buffer.allocUnsafe(length);
|
|
1485
|
+
if (readIntoSync(fd, chunk, payloadOffset + offset) !== length) {
|
|
1486
|
+
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn payload`);
|
|
1487
|
+
}
|
|
1488
|
+
snapshotMemorySampleHookForTest?.();
|
|
1489
|
+
offset += length;
|
|
1490
|
+
yield chunk;
|
|
1491
|
+
snapshotMemorySampleHookForTest?.();
|
|
1492
|
+
}
|
|
1493
|
+
},
|
|
1494
|
+
};
|
|
1495
|
+
}
|
|
1496
|
+
function decodeSnapshotWholePayload(fd, header, payloadLength, generation) {
|
|
1497
|
+
const data = Buffer.allocUnsafe(payloadLength);
|
|
1498
|
+
if (readIntoSync(fd, data, SNAPSHOT_FIXED_HEADER_BYTES) !== payloadLength) {
|
|
1499
|
+
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn payload`);
|
|
1500
|
+
}
|
|
1501
|
+
snapshotMemorySampleHookForTest?.();
|
|
1502
|
+
const checksum = Buffer.alloc(32);
|
|
1503
|
+
if (readIntoSync(fd, checksum, SNAPSHOT_FIXED_HEADER_BYTES + payloadLength) !== checksum.length) {
|
|
1504
|
+
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn checksum`);
|
|
1505
|
+
}
|
|
1506
|
+
const expected = crypto.createHash("sha256").update(header).update(data).digest();
|
|
1507
|
+
if (!expected.equals(checksum)) {
|
|
1508
|
+
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a bad checksum`);
|
|
1509
|
+
}
|
|
1510
|
+
try {
|
|
1511
|
+
const text = data.toString("utf8");
|
|
1512
|
+
snapshotMemorySampleHookForTest?.();
|
|
1513
|
+
const decoded = JSON.parse(text);
|
|
1514
|
+
snapshotMemorySampleHookForTest?.();
|
|
1515
|
+
return decoded;
|
|
1516
|
+
}
|
|
1517
|
+
catch (error) {
|
|
1518
|
+
throw new StateStoreCorruptionError(`snapshot generation ${generation} has invalid JSON`, { cause: error });
|
|
1519
|
+
}
|
|
1520
|
+
}
|
|
1521
|
+
function decodeSnapshotIncrementally(fd, header, payloadLength, generation, snapshotDecoder, magic) {
|
|
1522
|
+
readSnapshotChecksum(fd, SNAPSHOT_FIXED_HEADER_BYTES, payloadLength, header, generation);
|
|
1523
|
+
try {
|
|
1524
|
+
const decoded = snapshotDecoder(readSnapshotPayloadChunks(fd, SNAPSHOT_FIXED_HEADER_BYTES, payloadLength, generation), magic);
|
|
1525
|
+
snapshotMemorySampleHookForTest?.();
|
|
1526
|
+
return decoded;
|
|
1527
|
+
}
|
|
1528
|
+
catch (error) {
|
|
1529
|
+
if (error instanceof StateStoreCorruptionError)
|
|
1530
|
+
throw error;
|
|
1531
|
+
throw new StateStoreCorruptionError(`snapshot generation ${generation} has invalid JSON`, { cause: error });
|
|
1532
|
+
}
|
|
1533
|
+
}
|
|
1534
|
+
function readSnapshot(filePath, scopeDigest, generation, snapshotDecoder, snapshotMagics) {
|
|
1449
1535
|
snapshotRecoveryCount++;
|
|
1450
1536
|
let fd;
|
|
1451
1537
|
try {
|
|
@@ -1463,7 +1549,9 @@ function readSnapshot(filePath, scopeDigest, generation) {
|
|
|
1463
1549
|
if (readIntoSync(fd, header, 0) !== header.length) {
|
|
1464
1550
|
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn header`);
|
|
1465
1551
|
}
|
|
1466
|
-
|
|
1552
|
+
const magic = header.subarray(0, 8);
|
|
1553
|
+
const acceptedMagics = [SNAPSHOT_V3_MAGIC, ...(snapshotMagics ?? [])];
|
|
1554
|
+
if (!acceptedMagics.some((accepted) => magic.equals(accepted))) {
|
|
1467
1555
|
throw new StateStoreCorruptionError(`snapshot generation ${generation} has an invalid magic`);
|
|
1468
1556
|
}
|
|
1469
1557
|
if (readU64(header, 8) !== generation) {
|
|
@@ -1477,29 +1565,16 @@ function readSnapshot(filePath, scopeDigest, generation) {
|
|
|
1477
1565
|
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a scope digest mismatch`);
|
|
1478
1566
|
}
|
|
1479
1567
|
snapshotReadHookForTest?.();
|
|
1480
|
-
const
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
}
|
|
1484
|
-
const checksum = Buffer.alloc(32);
|
|
1485
|
-
if (readIntoSync(fd, checksum, SNAPSHOT_FIXED_HEADER_BYTES + payloadLength) !== checksum.length) {
|
|
1486
|
-
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn checksum`);
|
|
1487
|
-
}
|
|
1488
|
-
const expected = crypto.createHash("sha256").update(header).update(data).digest();
|
|
1489
|
-
if (!expected.equals(checksum)) {
|
|
1490
|
-
throw new StateStoreCorruptionError(`snapshot generation ${generation} has a bad checksum`);
|
|
1491
|
-
}
|
|
1492
|
-
let decoded;
|
|
1493
|
-
try {
|
|
1494
|
-
decoded = JSON.parse(data.toString("utf8"));
|
|
1495
|
-
}
|
|
1496
|
-
catch (error) {
|
|
1497
|
-
throw new StateStoreCorruptionError(`snapshot generation ${generation} has invalid JSON`, { cause: error });
|
|
1498
|
-
}
|
|
1568
|
+
const decoded = snapshotDecoder === undefined
|
|
1569
|
+
? decodeSnapshotWholePayload(fd, header, payloadLength, generation)
|
|
1570
|
+
: decodeSnapshotIncrementally(fd, header, payloadLength, generation, snapshotDecoder, magic);
|
|
1499
1571
|
if (!isSnapshotPayload(decoded)) {
|
|
1500
1572
|
throw new StateStoreCorruptionError(`snapshot generation ${generation} has an invalid state payload`);
|
|
1501
1573
|
}
|
|
1502
|
-
return {
|
|
1574
|
+
return {
|
|
1575
|
+
payload: decoded,
|
|
1576
|
+
maxReadBytes: snapshotDecoder === undefined ? payloadLength : Math.min(payloadLength, SNAPSHOT_CHUNK_BYTES),
|
|
1577
|
+
};
|
|
1503
1578
|
}
|
|
1504
1579
|
catch (error) {
|
|
1505
1580
|
if (error instanceof StateStoreCorruptionError)
|
|
@@ -1814,7 +1889,7 @@ function encodeSnapshotChunks(generation, scopeDigest, payload) {
|
|
|
1814
1889
|
writeCanonicalJson(payload, sink);
|
|
1815
1890
|
sink.flush();
|
|
1816
1891
|
const header = Buffer.alloc(SNAPSHOT_FIXED_HEADER_BYTES);
|
|
1817
|
-
|
|
1892
|
+
SNAPSHOT_V3_MAGIC.copy(header, 0);
|
|
1818
1893
|
writeU64(header, generation, 8);
|
|
1819
1894
|
header.writeUInt32BE(sink.byteLength, 16);
|
|
1820
1895
|
scopeDigest.copy(header, 20);
|
|
@@ -1834,7 +1909,7 @@ function encodeSnapshotChunks(generation, scopeDigest, payload) {
|
|
|
1834
1909
|
function encodeSnapshotWholeBuffer(generation, scopeDigest, payload) {
|
|
1835
1910
|
const data = Buffer.from(canonicalJson(payload), "utf8");
|
|
1836
1911
|
const header = Buffer.alloc(SNAPSHOT_FIXED_HEADER_BYTES);
|
|
1837
|
-
|
|
1912
|
+
SNAPSHOT_V3_MAGIC.copy(header, 0);
|
|
1838
1913
|
writeU64(header, generation, 8);
|
|
1839
1914
|
header.writeUInt32BE(data.length, 16);
|
|
1840
1915
|
scopeDigest.copy(header, 20);
|