@indigoai-us/hq-cloud 6.16.34 → 6.16.36

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  2. package/dist/bin/sync-runner-watch-loop.js +20 -8
  3. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  4. package/dist/bin/sync-runner-watch-loop.test.js +1 -2
  5. package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
  6. package/dist/bin/sync-runner.test.js +21 -24
  7. package/dist/bin/sync-runner.test.js.map +1 -1
  8. package/dist/cli/share.d.ts.map +1 -1
  9. package/dist/cli/share.js +18 -7
  10. package/dist/cli/share.js.map +1 -1
  11. package/dist/cli/share.test.js +30 -3
  12. package/dist/cli/share.test.js.map +1 -1
  13. package/dist/cli/sync.d.ts.map +1 -1
  14. package/dist/cli/sync.js +3 -2
  15. package/dist/cli/sync.js.map +1 -1
  16. package/dist/cli/sync.test.js +6 -6
  17. package/dist/cli/sync.test.js.map +1 -1
  18. package/dist/company-vault-transport.d.ts +0 -2
  19. package/dist/company-vault-transport.d.ts.map +1 -1
  20. package/dist/company-vault-transport.js +0 -2
  21. package/dist/company-vault-transport.js.map +1 -1
  22. package/dist/index.d.ts +2 -2
  23. package/dist/index.d.ts.map +1 -1
  24. package/dist/index.js +1 -1
  25. package/dist/index.js.map +1 -1
  26. package/dist/journal-baseline.test.js +18 -15
  27. package/dist/journal-baseline.test.js.map +1 -1
  28. package/dist/journal-delta-writes.test.js +2 -2
  29. package/dist/journal-delta-writes.test.js.map +1 -1
  30. package/dist/journal-public-boundary.test.d.ts +2 -0
  31. package/dist/journal-public-boundary.test.d.ts.map +1 -0
  32. package/dist/journal-public-boundary.test.js +216 -0
  33. package/dist/journal-public-boundary.test.js.map +1 -0
  34. package/dist/journal-record-proxy.d.ts +11 -0
  35. package/dist/journal-record-proxy.d.ts.map +1 -0
  36. package/dist/journal-record-proxy.js +71 -0
  37. package/dist/journal-record-proxy.js.map +1 -0
  38. package/dist/journal-row-store.d.ts +76 -3
  39. package/dist/journal-row-store.d.ts.map +1 -1
  40. package/dist/journal-row-store.js +450 -43
  41. package/dist/journal-row-store.js.map +1 -1
  42. package/dist/journal-row-store.test.js +139 -6
  43. package/dist/journal-row-store.test.js.map +1 -1
  44. package/dist/journal-snapshot-decoder.d.ts +18 -0
  45. package/dist/journal-snapshot-decoder.d.ts.map +1 -0
  46. package/dist/journal-snapshot-decoder.js +373 -0
  47. package/dist/journal-snapshot-decoder.js.map +1 -0
  48. package/dist/journal-snapshot-memory-helpers.d.ts +30 -0
  49. package/dist/journal-snapshot-memory-helpers.d.ts.map +1 -0
  50. package/dist/journal-snapshot-memory-helpers.js +203 -0
  51. package/dist/journal-snapshot-memory-helpers.js.map +1 -0
  52. package/dist/journal-snapshot-memory.test.d.ts +2 -0
  53. package/dist/journal-snapshot-memory.test.d.ts.map +1 -0
  54. package/dist/journal-snapshot-memory.test.js +32 -0
  55. package/dist/journal-snapshot-memory.test.js.map +1 -0
  56. package/dist/journal-snapshot-v4-memory.test.d.ts +2 -0
  57. package/dist/journal-snapshot-v4-memory.test.d.ts.map +1 -0
  58. package/dist/journal-snapshot-v4-memory.test.js +24 -0
  59. package/dist/journal-snapshot-v4-memory.test.js.map +1 -0
  60. package/dist/journal-snapshot-v4.d.ts +52 -0
  61. package/dist/journal-snapshot-v4.d.ts.map +1 -0
  62. package/dist/journal-snapshot-v4.js +352 -0
  63. package/dist/journal-snapshot-v4.js.map +1 -0
  64. package/dist/journal-snapshot-v4.test.d.ts +2 -0
  65. package/dist/journal-snapshot-v4.test.d.ts.map +1 -0
  66. package/dist/journal-snapshot-v4.test.js +280 -0
  67. package/dist/journal-snapshot-v4.test.js.map +1 -0
  68. package/dist/journal-store.test.js +335 -17
  69. package/dist/journal-store.test.js.map +1 -1
  70. package/dist/journal.d.ts +10 -8
  71. package/dist/journal.d.ts.map +1 -1
  72. package/dist/journal.js +165 -50
  73. package/dist/journal.js.map +1 -1
  74. package/dist/journal.test.js +1 -1
  75. package/dist/journal.test.js.map +1 -1
  76. package/dist/manifest/build-manifest.bench.test.js +165 -58
  77. package/dist/manifest/build-manifest.bench.test.js.map +1 -1
  78. package/dist/remote-pull.js +6 -4
  79. package/dist/remote-pull.js.map +1 -1
  80. package/dist/remote-pull.test.js +72 -0
  81. package/dist/remote-pull.test.js.map +1 -1
  82. package/dist/session-log-retention.test.js +3 -3
  83. package/dist/session-log-retention.test.js.map +1 -1
  84. package/dist/skill-telemetry.d.ts +1 -2
  85. package/dist/skill-telemetry.d.ts.map +1 -1
  86. package/dist/skill-telemetry.js +1 -10
  87. package/dist/skill-telemetry.js.map +1 -1
  88. package/dist/sync/child-process-sync-worker-env.test.d.ts +2 -0
  89. package/dist/sync/child-process-sync-worker-env.test.d.ts.map +1 -0
  90. package/dist/sync/child-process-sync-worker-env.test.js +38 -0
  91. package/dist/sync/child-process-sync-worker-env.test.js.map +1 -0
  92. package/dist/sync/feature-flags.d.ts +0 -16
  93. package/dist/sync/feature-flags.d.ts.map +1 -1
  94. package/dist/sync/feature-flags.js +0 -11
  95. package/dist/sync/feature-flags.js.map +1 -1
  96. package/dist/sync/index.d.ts +2 -2
  97. package/dist/sync/index.d.ts.map +1 -1
  98. package/dist/sync/index.js +1 -1
  99. package/dist/sync/index.js.map +1 -1
  100. package/dist/sync/logger.test.js +1 -3
  101. package/dist/sync/logger.test.js.map +1 -1
  102. package/dist/sync/metrics.test.js +0 -3
  103. package/dist/sync/metrics.test.js.map +1 -1
  104. package/dist/sync/push-receiver.d.ts +0 -7
  105. package/dist/sync/push-receiver.d.ts.map +1 -1
  106. package/dist/sync/push-receiver.js.map +1 -1
  107. package/dist/sync/state-store.d.ts +23 -0
  108. package/dist/sync/state-store.d.ts.map +1 -1
  109. package/dist/sync/state-store.js +114 -39
  110. package/dist/sync/state-store.js.map +1 -1
  111. package/dist/sync/state-store.test.js +82 -0
  112. package/dist/sync/state-store.test.js.map +1 -1
  113. package/dist/watcher.test.js +7 -8
  114. package/dist/watcher.test.js.map +1 -1
  115. package/package.json +1 -1
  116. package/dist/daemon-worker.d.ts +0 -11
  117. package/dist/daemon-worker.d.ts.map +0 -1
  118. package/dist/daemon-worker.js +0 -22
  119. package/dist/daemon-worker.js.map +0 -1
@@ -8,7 +8,10 @@
8
8
  import * as crypto from "node:crypto";
9
9
  import * as fs from "node:fs";
10
10
  import * as path from "node:path";
11
- const SNAPSHOT_MAGIC = Buffer.from("HQSNAP3\0", "ascii");
11
+ /** The legacy JSON snapshot format emitted by this release's writer. */
12
+ export const SNAPSHOT_V3_MAGIC = Buffer.from("HQSNAP3\0", "ascii");
13
+ /** The packed journal snapshot format accepted by the reader before it is emitted. */
14
+ export const SNAPSHOT_V4_MAGIC = Buffer.from("HQSNAP4\0", "ascii");
12
15
  const WAL_MAGIC = Buffer.from("HQWAL3\0\0", "ascii");
13
16
  const FRAME_MAGIC = 0x48515733;
14
17
  const SNAPSHOT_HEADER_BYTES = 8 + 8 + 4 + 32 + 32;
@@ -74,6 +77,7 @@ let snapshotMaterializationCount = 0;
74
77
  let snapshotEncodeCount = 0;
75
78
  let snapshotWriteHookForTest;
76
79
  let snapshotReadHookForTest;
80
+ let snapshotMemorySampleHookForTest;
77
81
  let walReadHookForTest;
78
82
  let afterGenerationStageHookForTest;
79
83
  let reflinkCopyHookForTest;
@@ -122,6 +126,10 @@ export function setStateStoreSnapshotWriteHookForTest(hook) {
122
126
  export function setStateStoreSnapshotReadHookForTest(hook) {
123
127
  snapshotReadHookForTest = hook;
124
128
  }
129
+ /** Test-only sampling seam before and after decoder work for each bounded snapshot chunk. */
130
+ export function setStateStoreSnapshotMemorySampleHookForTest(hook) {
131
+ snapshotMemorySampleHookForTest = hook;
132
+ }
125
133
  /** Test-only seam after the WAL length is sampled during cold recovery. */
126
134
  export function setStateStoreWalReadHookForTest(hook) {
127
135
  walReadHookForTest = hook;
@@ -297,7 +305,7 @@ export class StateStore {
297
305
  const dir = stateDirectory(normalized.rootDir, scopeDigest);
298
306
  fs.mkdirSync(dir, { recursive: true });
299
307
  while (true) {
300
- const recovered = readNewestGenerationOptimistically(dir, scopeDigest, normalized.reduce);
308
+ const recovered = readNewestGenerationOptimistically(dir, scopeDigest, normalized.reduce, normalized.snapshotDecoder, normalized.snapshotMagics);
301
309
  if (recovered) {
302
310
  const store = new StateStore(normalized, recovered, false);
303
311
  // A process can die after its durable append but before append() gets
@@ -337,7 +345,7 @@ export class StateStore {
337
345
  const dir = stateDirectory(normalized.rootDir, scopeDigest);
338
346
  await fs.promises.mkdir(dir, { recursive: true });
339
347
  while (true) {
340
- const recovered = await readNewestGenerationOptimisticallyAsync(dir, scopeDigest, normalized.reduce);
348
+ const recovered = await readNewestGenerationOptimisticallyAsync(dir, scopeDigest, normalized.reduce, normalized.snapshotDecoder, normalized.snapshotMagics);
341
349
  if (recovered) {
342
350
  const store = new StateStore(normalized, recovered, false);
343
351
  // Match open(): a recovered over-budget WAL is compacted before this
@@ -471,7 +479,7 @@ export class StateStore {
471
479
  // Preserve recovery's corruption classification for a crashed or
472
480
  // malformed store that has snapshots but no complete generation.
473
481
  if (hasSnapshotGeneration(dir)) {
474
- readNewestGeneration(dir, scopeDigest, normalized.reduce, maintenance.onProgress, false);
482
+ readNewestGeneration(dir, scopeDigest, normalized.reduce, normalized.snapshotDecoder, normalized.snapshotMagics, maintenance.onProgress, false);
475
483
  continue;
476
484
  }
477
485
  return { status: "missing" };
@@ -481,7 +489,7 @@ export class StateStore {
481
489
  // Snapshot and WAL replay are read-only. Do not make appenders wait for
482
490
  // this potentially multi-second materialisation; validate its view before
483
491
  // using it to build a replacement generation.
484
- const recovered = readNewestGeneration(dir, scopeDigest, normalized.reduce, maintenance.onProgress, false);
492
+ const recovered = readNewestGeneration(dir, scopeDigest, normalized.reduce, normalized.snapshotDecoder, normalized.snapshotMagics, maintenance.onProgress, false);
485
493
  if (!recovered)
486
494
  continue;
487
495
  plan = withScopeLock(dir, () => {
@@ -1058,13 +1066,13 @@ export class StateStore {
1058
1066
  return undefined;
1059
1067
  }
1060
1068
  recoverNewestGenerationOutsideLock() {
1061
- const recovered = readNewestGenerationOptimistically(this.stateDir, this.scopeDigest, this.options.reduce);
1069
+ const recovered = readNewestGenerationOptimistically(this.stateDir, this.scopeDigest, this.options.reduce, this.options.snapshotDecoder, this.options.snapshotMagics);
1062
1070
  if (!recovered)
1063
1071
  throw new StateStoreCorruptionError("no valid generation remains while appending");
1064
1072
  this.installGeneration(recovered);
1065
1073
  }
1066
1074
  async recoverNewestGenerationOutsideLockAsync() {
1067
- const recovered = await readNewestGenerationOptimisticallyAsync(this.stateDir, this.scopeDigest, this.options.reduce);
1075
+ const recovered = await readNewestGenerationOptimisticallyAsync(this.stateDir, this.scopeDigest, this.options.reduce, this.options.snapshotDecoder, this.options.snapshotMagics);
1068
1076
  if (!recovered)
1069
1077
  throw new StateStoreCorruptionError("no valid generation remains while appending");
1070
1078
  this.installGeneration(recovered);
@@ -1121,7 +1129,7 @@ export class StateStore {
1121
1129
  this.observedHighestSnapshotGeneration = generation.observedHighestSnapshotGeneration ?? generation.generation;
1122
1130
  }
1123
1131
  }
1124
- function readNewestGeneration(dir, scopeDigest, reduce, onProgress, repairTornTail = true) {
1132
+ function readNewestGeneration(dir, scopeDigest, reduce, snapshotDecoder, snapshotMagics, onProgress, repairTornTail = true) {
1125
1133
  const names = fs.readdirSync(dir);
1126
1134
  const generations = names
1127
1135
  .map((name) => /^snapshot-(\d+)\.bin$/.exec(name)?.[1])
@@ -1134,7 +1142,7 @@ function readNewestGeneration(dir, scopeDigest, reduce, onProgress, repairTornTa
1134
1142
  for (const generation of generations) {
1135
1143
  try {
1136
1144
  return {
1137
- ...readGeneration(dir, scopeDigest, generation, reduce, onProgress, repairTornTail),
1145
+ ...readGeneration(dir, scopeDigest, generation, reduce, snapshotDecoder, snapshotMagics, onProgress, repairTornTail),
1138
1146
  observedHighestSnapshotGeneration: generations[0],
1139
1147
  };
1140
1148
  }
@@ -1163,10 +1171,10 @@ function readNewestGeneration(dir, scopeDigest, reduce, onProgress, repairTornTa
1163
1171
  }
1164
1172
  return undefined;
1165
1173
  }
1166
- function readGeneration(dir, scopeDigest, generation, reduce, onProgress, repairTornTail = true) {
1174
+ function readGeneration(dir, scopeDigest, generation, reduce, snapshotDecoder, snapshotMagics, onProgress, repairTornTail = true) {
1167
1175
  const snapshotBytes = generationFileSize(snapshotPath(dir, generation), "snapshot", generation);
1168
1176
  const totalBytes = snapshotBytes + generationFileSize(walPath(dir, generation), "WAL", generation);
1169
- const snapshot = readSnapshot(snapshotPath(dir, generation), scopeDigest, generation);
1177
+ const snapshot = readSnapshot(snapshotPath(dir, generation), scopeDigest, generation, snapshotDecoder, snapshotMagics);
1170
1178
  onProgress?.({ bytesRead: snapshotBytes, totalBytes });
1171
1179
  const parsed = readWal(walPath(dir, generation), scopeDigest, generation, snapshot.payload.lastRecordSequence, snapshot.payload.state, reduce, snapshotBytes, totalBytes, onProgress, repairTornTail);
1172
1180
  return {
@@ -1187,7 +1195,7 @@ function readGeneration(dir, scopeDigest, generation, reduce, onProgress, repair
1187
1195
  * WAL suffix rather than restarting the expensive immutable snapshot read.
1188
1196
  * Rotation or a torn suffix remains lock-protected.
1189
1197
  */
1190
- function readNewestGenerationOptimistically(dir, scopeDigest, reduce, onProgress) {
1198
+ function readNewestGenerationOptimistically(dir, scopeDigest, reduce, snapshotDecoder, snapshotMagics, onProgress) {
1191
1199
  let resamples = 0;
1192
1200
  while (true) {
1193
1201
  const candidate = withScopeLock(dir, () => {
@@ -1198,7 +1206,7 @@ function readNewestGenerationOptimistically(dir, scopeDigest, reduce, onProgress
1198
1206
  return undefined;
1199
1207
  let recovered;
1200
1208
  try {
1201
- recovered = readNewestGeneration(dir, scopeDigest, reduce, onProgress, false);
1209
+ recovered = readNewestGeneration(dir, scopeDigest, reduce, snapshotDecoder, snapshotMagics, onProgress, false);
1202
1210
  }
1203
1211
  catch (error) {
1204
1212
  if (!(error instanceof StateStoreGenerationMissingError))
@@ -1218,7 +1226,7 @@ function readNewestGenerationOptimistically(dir, scopeDigest, reduce, onProgress
1218
1226
  return validated;
1219
1227
  }
1220
1228
  }
1221
- async function readNewestGenerationOptimisticallyAsync(dir, scopeDigest, reduce, onProgress) {
1229
+ async function readNewestGenerationOptimisticallyAsync(dir, scopeDigest, reduce, snapshotDecoder, snapshotMagics, onProgress) {
1222
1230
  let resamples = 0;
1223
1231
  while (true) {
1224
1232
  const candidate = await withScopeLockAsync(dir, () => {
@@ -1229,7 +1237,7 @@ async function readNewestGenerationOptimisticallyAsync(dir, scopeDigest, reduce,
1229
1237
  return undefined;
1230
1238
  let recovered;
1231
1239
  try {
1232
- recovered = readNewestGeneration(dir, scopeDigest, reduce, onProgress, false);
1240
+ recovered = readNewestGeneration(dir, scopeDigest, reduce, snapshotDecoder, snapshotMagics, onProgress, false);
1233
1241
  }
1234
1242
  catch (error) {
1235
1243
  if (!(error instanceof StateStoreGenerationMissingError))
@@ -1445,7 +1453,85 @@ function discardStagedGeneration(staged) {
1445
1453
  }
1446
1454
  catch { /* staged files are best-effort cleanup */ }
1447
1455
  }
1448
- function readSnapshot(filePath, scopeDigest, generation) {
1456
+ function readSnapshotChecksum(fd, payloadOffset, payloadLength, header, generation) {
1457
+ const hash = crypto.createHash("sha256").update(header);
1458
+ let offset = 0;
1459
+ while (offset < payloadLength) {
1460
+ const length = Math.min(SNAPSHOT_CHUNK_BYTES, payloadLength - offset);
1461
+ const chunk = Buffer.allocUnsafe(length);
1462
+ if (readIntoSync(fd, chunk, payloadOffset + offset) !== length) {
1463
+ throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn payload`);
1464
+ }
1465
+ hash.update(chunk);
1466
+ snapshotMemorySampleHookForTest?.();
1467
+ offset += length;
1468
+ }
1469
+ const checksum = Buffer.alloc(32);
1470
+ if (readIntoSync(fd, checksum, payloadOffset + payloadLength) !== checksum.length) {
1471
+ throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn checksum`);
1472
+ }
1473
+ if (!hash.digest().equals(checksum)) {
1474
+ throw new StateStoreCorruptionError(`snapshot generation ${generation} has a bad checksum`);
1475
+ }
1476
+ }
1477
+ function readSnapshotPayloadChunks(fd, payloadOffset, payloadLength, generation) {
1478
+ return {
1479
+ byteLength: payloadLength,
1480
+ *[Symbol.iterator]() {
1481
+ let offset = 0;
1482
+ while (offset < payloadLength) {
1483
+ const length = Math.min(SNAPSHOT_CHUNK_BYTES, payloadLength - offset);
1484
+ const chunk = Buffer.allocUnsafe(length);
1485
+ if (readIntoSync(fd, chunk, payloadOffset + offset) !== length) {
1486
+ throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn payload`);
1487
+ }
1488
+ snapshotMemorySampleHookForTest?.();
1489
+ offset += length;
1490
+ yield chunk;
1491
+ snapshotMemorySampleHookForTest?.();
1492
+ }
1493
+ },
1494
+ };
1495
+ }
1496
+ function decodeSnapshotWholePayload(fd, header, payloadLength, generation) {
1497
+ const data = Buffer.allocUnsafe(payloadLength);
1498
+ if (readIntoSync(fd, data, SNAPSHOT_FIXED_HEADER_BYTES) !== payloadLength) {
1499
+ throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn payload`);
1500
+ }
1501
+ snapshotMemorySampleHookForTest?.();
1502
+ const checksum = Buffer.alloc(32);
1503
+ if (readIntoSync(fd, checksum, SNAPSHOT_FIXED_HEADER_BYTES + payloadLength) !== checksum.length) {
1504
+ throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn checksum`);
1505
+ }
1506
+ const expected = crypto.createHash("sha256").update(header).update(data).digest();
1507
+ if (!expected.equals(checksum)) {
1508
+ throw new StateStoreCorruptionError(`snapshot generation ${generation} has a bad checksum`);
1509
+ }
1510
+ try {
1511
+ const text = data.toString("utf8");
1512
+ snapshotMemorySampleHookForTest?.();
1513
+ const decoded = JSON.parse(text);
1514
+ snapshotMemorySampleHookForTest?.();
1515
+ return decoded;
1516
+ }
1517
+ catch (error) {
1518
+ throw new StateStoreCorruptionError(`snapshot generation ${generation} has invalid JSON`, { cause: error });
1519
+ }
1520
+ }
1521
+ function decodeSnapshotIncrementally(fd, header, payloadLength, generation, snapshotDecoder, magic) {
1522
+ readSnapshotChecksum(fd, SNAPSHOT_FIXED_HEADER_BYTES, payloadLength, header, generation);
1523
+ try {
1524
+ const decoded = snapshotDecoder(readSnapshotPayloadChunks(fd, SNAPSHOT_FIXED_HEADER_BYTES, payloadLength, generation), magic);
1525
+ snapshotMemorySampleHookForTest?.();
1526
+ return decoded;
1527
+ }
1528
+ catch (error) {
1529
+ if (error instanceof StateStoreCorruptionError)
1530
+ throw error;
1531
+ throw new StateStoreCorruptionError(`snapshot generation ${generation} has invalid JSON`, { cause: error });
1532
+ }
1533
+ }
1534
+ function readSnapshot(filePath, scopeDigest, generation, snapshotDecoder, snapshotMagics) {
1449
1535
  snapshotRecoveryCount++;
1450
1536
  let fd;
1451
1537
  try {
@@ -1463,7 +1549,9 @@ function readSnapshot(filePath, scopeDigest, generation) {
1463
1549
  if (readIntoSync(fd, header, 0) !== header.length) {
1464
1550
  throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn header`);
1465
1551
  }
1466
- if (!header.subarray(0, 8).equals(SNAPSHOT_MAGIC)) {
1552
+ const magic = header.subarray(0, 8);
1553
+ const acceptedMagics = [SNAPSHOT_V3_MAGIC, ...(snapshotMagics ?? [])];
1554
+ if (!acceptedMagics.some((accepted) => magic.equals(accepted))) {
1467
1555
  throw new StateStoreCorruptionError(`snapshot generation ${generation} has an invalid magic`);
1468
1556
  }
1469
1557
  if (readU64(header, 8) !== generation) {
@@ -1477,29 +1565,16 @@ function readSnapshot(filePath, scopeDigest, generation) {
1477
1565
  throw new StateStoreCorruptionError(`snapshot generation ${generation} has a scope digest mismatch`);
1478
1566
  }
1479
1567
  snapshotReadHookForTest?.();
1480
- const data = Buffer.allocUnsafe(payloadLength);
1481
- if (readIntoSync(fd, data, SNAPSHOT_FIXED_HEADER_BYTES) !== payloadLength) {
1482
- throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn payload`);
1483
- }
1484
- const checksum = Buffer.alloc(32);
1485
- if (readIntoSync(fd, checksum, SNAPSHOT_FIXED_HEADER_BYTES + payloadLength) !== checksum.length) {
1486
- throw new StateStoreCorruptionError(`snapshot generation ${generation} has a torn checksum`);
1487
- }
1488
- const expected = crypto.createHash("sha256").update(header).update(data).digest();
1489
- if (!expected.equals(checksum)) {
1490
- throw new StateStoreCorruptionError(`snapshot generation ${generation} has a bad checksum`);
1491
- }
1492
- let decoded;
1493
- try {
1494
- decoded = JSON.parse(data.toString("utf8"));
1495
- }
1496
- catch (error) {
1497
- throw new StateStoreCorruptionError(`snapshot generation ${generation} has invalid JSON`, { cause: error });
1498
- }
1568
+ const decoded = snapshotDecoder === undefined
1569
+ ? decodeSnapshotWholePayload(fd, header, payloadLength, generation)
1570
+ : decodeSnapshotIncrementally(fd, header, payloadLength, generation, snapshotDecoder, magic);
1499
1571
  if (!isSnapshotPayload(decoded)) {
1500
1572
  throw new StateStoreCorruptionError(`snapshot generation ${generation} has an invalid state payload`);
1501
1573
  }
1502
- return { payload: decoded, maxReadBytes: payloadLength };
1574
+ return {
1575
+ payload: decoded,
1576
+ maxReadBytes: snapshotDecoder === undefined ? payloadLength : Math.min(payloadLength, SNAPSHOT_CHUNK_BYTES),
1577
+ };
1503
1578
  }
1504
1579
  catch (error) {
1505
1580
  if (error instanceof StateStoreCorruptionError)
@@ -1814,7 +1889,7 @@ function encodeSnapshotChunks(generation, scopeDigest, payload) {
1814
1889
  writeCanonicalJson(payload, sink);
1815
1890
  sink.flush();
1816
1891
  const header = Buffer.alloc(SNAPSHOT_FIXED_HEADER_BYTES);
1817
- SNAPSHOT_MAGIC.copy(header, 0);
1892
+ SNAPSHOT_V3_MAGIC.copy(header, 0);
1818
1893
  writeU64(header, generation, 8);
1819
1894
  header.writeUInt32BE(sink.byteLength, 16);
1820
1895
  scopeDigest.copy(header, 20);
@@ -1834,7 +1909,7 @@ function encodeSnapshotChunks(generation, scopeDigest, payload) {
1834
1909
  function encodeSnapshotWholeBuffer(generation, scopeDigest, payload) {
1835
1910
  const data = Buffer.from(canonicalJson(payload), "utf8");
1836
1911
  const header = Buffer.alloc(SNAPSHOT_FIXED_HEADER_BYTES);
1837
- SNAPSHOT_MAGIC.copy(header, 0);
1912
+ SNAPSHOT_V3_MAGIC.copy(header, 0);
1838
1913
  writeU64(header, generation, 8);
1839
1914
  header.writeUInt32BE(data.length, 16);
1840
1915
  scopeDigest.copy(header, 20);