@brunobracaioli/aibrain 0.3.0-beta.16 → 0.3.0-beta.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -16,7 +16,7 @@ import {
16
16
  import { constants as fsConstants } from 'node:fs';
17
17
  import { spawn } from 'node:child_process';
18
18
  import { createHash, randomUUID } from 'node:crypto';
19
- import { dirname, relative, resolve } from 'node:path';
19
+ import { basename, dirname, relative, resolve } from 'node:path';
20
20
  import { StringDecoder } from 'node:string_decoder';
21
21
  import { pathToFileURL } from 'node:url';
22
22
  import { createTelemetryReporter, TELEMETRY_TIMEOUT_MS } from './context-telemetry-reporter.mjs';
@@ -47,6 +47,8 @@ const MAX_CAPTURE_ITEMS = 65_536;
47
47
  const MAX_CAPTURE_DIRECTORY_BYTES = 256 * 1024 * 1024;
48
48
  const MAX_CAPTURE_STAGING_HEADROOM_BYTES = 16 * 1024 * 1024;
49
49
  const MAX_CAPTURE_FILES = 128;
50
+ const MAX_CAPTURE_PUBLICATION_FILE_HEADROOM = MAX_CAPTURE_PAGES + 2;
51
+ const MAX_CAPTURE_PUBLICATION_FILES = MAX_CAPTURE_FILES + MAX_CAPTURE_PUBLICATION_FILE_HEADROOM;
50
52
  const CAPTURE_LOCK_WAIT_MS = 1_000;
51
53
  const MAX_OWNED_REQUESTS = 1_000;
52
54
  const MAX_RETIRED_REQUESTS = 2_000;
@@ -619,6 +621,25 @@ async function ensurePrivateDirectory(directory, fsOps = captureFs) {
619
621
  await fsOps.chmod(target, 0o700);
620
622
  return target;
621
623
  }
624
+ async function validatePrivateDirectoryPath(directory, fsOps = captureFs) {
625
+ const target = resolve(directory);
626
+ const parts = target.split('/');
627
+ let current = parts[0] === '' ? '/' : parts[0];
628
+ for (const part of parts.slice(1)) {
629
+ if (!part) continue;
630
+ current = current === '/' ? `/${part}` : `${current}/${part}`;
631
+ const info = await fsOps.lstat(current);
632
+ if (info.isSymbolicLink() || !info.isDirectory())
633
+ throw new Error('checkpoint directory is unsafe');
634
+ if (
635
+ current === target &&
636
+ ((typeof process.getuid === 'function' && info.uid !== process.getuid()) ||
637
+ (info.mode & 0o077) !== 0)
638
+ )
639
+ throw new Error('capture directory is not private');
640
+ }
641
+ return target;
642
+ }
622
643
  async function assertPrivateFile(file, label, fsOps = captureFs) {
623
644
  const info = await fsOps.lstat(file);
624
645
  if (
@@ -641,6 +662,17 @@ function captureCapacityError(state) {
641
662
  error.diagnosticReason = 'storage_capacity_exceeded';
642
663
  error.captureDirectoryBytes = state?.totalBytes ?? MAX_CAPTURE_DIRECTORY_BYTES;
643
664
  error.captureDirectoryLimitBytes = MAX_CAPTURE_DIRECTORY_BYTES;
665
+ error.captureFileCount =
666
+ state?.managedFileCount ?? state?.managedFiles?.length ?? MAX_CAPTURE_FILES;
667
+ error.captureFileLimit = MAX_CAPTURE_FILES;
668
+ return error;
669
+ }
670
+
671
+ function captureStorageCleanupError(state) {
672
+ const error = new Error('capture storage cleanup remains incomplete');
673
+ error.diagnosticReason = 'private_storage_failed';
674
+ error.captureDirectoryBytes = state?.totalBytes ?? MAX_CAPTURE_DIRECTORY_BYTES;
675
+ error.captureDirectoryLimitBytes = MAX_CAPTURE_DIRECTORY_BYTES;
644
676
  error.captureFileCount = state?.managedFiles?.length ?? MAX_CAPTURE_FILES;
645
677
  error.captureFileLimit = MAX_CAPTURE_FILES;
646
678
  return error;
@@ -719,10 +751,20 @@ function captureNameForSnapshot(value, suffix = '') {
719
751
  return `${strictThreadName(value.threadId)}${suffix}.json`;
720
752
  }
721
753
 
722
- async function readCaptureDirectoryState(directory, fsOps = captureFs) {
754
+ async function readCaptureDirectoryState(
755
+ directory,
756
+ fsOps = captureFs,
757
+ { replacedCurrentName = null } = {},
758
+ ) {
723
759
  const entries = await fsOps.readdir(directory, { withFileTypes: true });
724
760
  if (entries.length > MAX_CAPTURE_FILES * 4)
725
761
  throw captureCapacityError({ totalBytes: MAX_CAPTURE_DIRECTORY_BYTES, managedFiles: [] });
762
+ let activeSentinelInfo = null;
763
+ try {
764
+ activeSentinelInfo = await fsOps.lstat(resolve(directory, '.capture.lock'));
765
+ } catch (error) {
766
+ if (error?.code !== 'ENOENT') throw error;
767
+ }
726
768
  const state = {
727
769
  entryCount: entries.length,
728
770
  totalBytes: 0,
@@ -737,6 +779,14 @@ async function readCaptureDirectoryState(directory, fsOps = captureFs) {
737
779
  if (entry.name === '.capture.lock') continue;
738
780
  const file = resolve(directory, entry.name);
739
781
  const info = await fsOps.lstat(file);
782
+ if (
783
+ activeSentinelInfo !== null &&
784
+ Number.isInteger(activeSentinelInfo.dev) &&
785
+ Number.isInteger(activeSentinelInfo.ino) &&
786
+ info.dev === activeSentinelInfo.dev &&
787
+ info.ino === activeSentinelInfo.ino
788
+ )
789
+ continue;
740
790
  state.totalBytes += info.size;
741
791
  if (
742
792
  info.isSymbolicLink() ||
@@ -829,9 +879,10 @@ async function readCaptureDirectoryState(directory, fsOps = captureFs) {
829
879
  immutable: entry.name === sourceName,
830
880
  group: `${value.threadId}.${value.sourceRevision}`,
831
881
  });
832
- if (!entry.name.endsWith('.' + value.sourceRevision + '.json'))
882
+ const replacedCurrent = entry.name === replacedCurrentName;
883
+ if (!replacedCurrent && !entry.name.endsWith('.' + value.sourceRevision + '.json'))
833
884
  state.referencedImmutableNames.add(sourceName);
834
- if (value.version === 2 && entry.name !== sourceName) {
885
+ if (value.version === 2 && entry.name !== sourceName && !replacedCurrent) {
835
886
  for (const segment of value.captureManifest.segments) {
836
887
  if (typeof segment === 'string') state.referencedImmutableNames.add(segment);
837
888
  }
@@ -871,6 +922,46 @@ function captureLockError(reason) {
871
922
 
872
923
  const CAPTURE_LOCK_MARKER = `${JSON.stringify({ backend: 'flock', schema: 2 })}\n`;
873
924
  const CAPTURE_GUARD_NAME = '.capture.guard';
925
+ const CAPTURE_PAGE_STAGE_NAME =
926
+ /^\.capture-(?:segment-)?stage\.[A-Za-z0-9._-]{1,128}\.[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\.(?:[0-9]|[12][0-9]|3[01])\.tmp$/iu;
927
+ const CAPTURE_SOURCE_STAGE_NAME =
928
+ /^\.capture-stage\.[A-Za-z0-9._-]{1,128}\.[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\.(?:[0-9]|[12][0-9]|3[01])\.tmp$/iu;
929
+
930
+ async function removePrivateCaptureTemps(directory, fsOps, pattern) {
931
+ const entries = await fsOps.readdir(directory, { withFileTypes: true });
932
+ let removedFiles = 0;
933
+ let removedBytes = 0;
934
+ for (const entry of entries) {
935
+ if (!pattern.test(entry.name)) continue;
936
+ const file = resolve(directory, entry.name);
937
+ let info;
938
+ try {
939
+ info = await fsOps.lstat(file);
940
+ } catch (error) {
941
+ if (error?.code === 'ENOENT') continue;
942
+ throw error;
943
+ }
944
+ if (
945
+ info.isSymbolicLink() ||
946
+ !info.isFile() ||
947
+ (typeof process.getuid === 'function' && info.uid !== process.getuid()) ||
948
+ (info.mode & 0o077) !== 0
949
+ )
950
+ continue;
951
+ try {
952
+ await fsOps.unlink(file);
953
+ removedFiles += 1;
954
+ removedBytes += info.size;
955
+ } catch (error) {
956
+ if (error?.code !== 'ENOENT') throw error;
957
+ }
958
+ }
959
+ return { removedFiles, removedBytes };
960
+ }
961
+
962
+ async function removeAbandonedCapturePageStages(directory, fsOps) {
963
+ return removePrivateCaptureTemps(directory, fsOps, CAPTURE_PAGE_STAGE_NAME);
964
+ }
874
965
 
875
966
  async function observerKernelIdentity() {
876
967
  try {
@@ -1136,8 +1227,12 @@ async function withCaptureStoreLock(
1136
1227
  const nativeFs =
1137
1228
  fsOps === captureFs ||
1138
1229
  (fsOps.open === open && fsOps.lstat === lstat && fsOps.writeFile === writeFile);
1139
- if (nativeFs) return withAdvisoryCaptureLock(directory, task);
1140
- return withInjectedCaptureLock(directory, task, fsOps, { now, sleep });
1230
+ const taskWithStageCleanup = async () => {
1231
+ const reclaimed = await removeAbandonedCapturePageStages(directory, fsOps);
1232
+ return task(reclaimed);
1233
+ };
1234
+ if (nativeFs) return withAdvisoryCaptureLock(directory, taskWithStageCleanup);
1235
+ return withInjectedCaptureLock(directory, taskWithStageCleanup, fsOps, { now, sleep });
1141
1236
  }
1142
1237
 
1143
1238
  async function writePrivateFile(file, serialized, fsOps = captureFs) {
@@ -1168,9 +1263,30 @@ async function writeBoundedCapture(file, serialized, value, fsOps = captureFs, l
1168
1263
  file.startsWith(directory) ? file.slice(directory.length + 1) : file,
1169
1264
  );
1170
1265
  const targetName = target.slice(directory.length + 1);
1266
+ const pairedSourceTarget =
1267
+ lockOptions.pairedSourcePath === undefined
1268
+ ? null
1269
+ : resolve(
1270
+ directory,
1271
+ lockOptions.pairedSourcePath.startsWith(directory)
1272
+ ? lockOptions.pairedSourcePath.slice(directory.length + 1)
1273
+ : lockOptions.pairedSourcePath,
1274
+ );
1275
+ const pairedSourceName = pairedSourceTarget?.slice(directory.length + 1) ?? null;
1276
+ const sourceName = `${strictThreadName(value.threadId)}.${value.sourceRevision}.json`;
1277
+ if (
1278
+ pairedSourceTarget !== null &&
1279
+ (dirname(pairedSourceTarget) !== directory ||
1280
+ pairedSourceName !== sourceName ||
1281
+ targetName !== captureNameForSnapshot(value))
1282
+ )
1283
+ throw new Error('capture publication target identity validation failed');
1171
1284
  const operation = async () => {
1172
1285
  const temporary = `${target}.${process.pid}.${randomUUID()}.tmp`;
1173
- const sourceName = `${strictThreadName(value.threadId)}.${value.sourceRevision}.json`;
1286
+ const sourceTemporary =
1287
+ pairedSourceTarget === null
1288
+ ? null
1289
+ : `${pairedSourceTarget}.${process.pid}.${randomUUID()}.tmp`;
1174
1290
  const expectedNames = new Set([
1175
1291
  captureNameForSnapshot(value),
1176
1292
  captureNameForSnapshot(value, '.completed'),
@@ -1189,7 +1305,41 @@ async function writeBoundedCapture(file, serialized, value, fsOps = captureFs, l
1189
1305
  } catch (error) {
1190
1306
  if (error?.code !== 'ENOENT') throw error;
1191
1307
  }
1308
+ let existingSource = false;
1309
+ if (pairedSourceTarget !== null) {
1310
+ try {
1311
+ await assertPrivateFile(pairedSourceTarget, 'immutable capture source', fsOps);
1312
+ const immutable = JSON.parse(
1313
+ await readPrivateFile(
1314
+ pairedSourceTarget,
1315
+ 'immutable capture source',
1316
+ MAX_CAPTURE_BYTES,
1317
+ fsOps,
1318
+ ),
1319
+ );
1320
+ validateCaptureSnapshot(immutable, value.sourceRevision);
1321
+ if (immutable.threadId !== value.threadId)
1322
+ throw new Error('immutable capture source identity validation failed');
1323
+ existingSource = true;
1324
+ } catch (error) {
1325
+ if (error?.code !== 'ENOENT') throw error;
1326
+ }
1327
+ }
1192
1328
  try {
1329
+ if (sourceTemporary !== null && !existingSource) {
1330
+ await fsOps.writeFile(sourceTemporary, serialized, { flag: 'wx', mode: 0o600 });
1331
+ const writtenSource = await assertPrivateFile(
1332
+ sourceTemporary,
1333
+ 'immutable capture candidate',
1334
+ fsOps,
1335
+ );
1336
+ if (writtenSource.size !== Buffer.byteLength(serialized, 'utf8'))
1337
+ throw new Error('immutable capture candidate read-back size mismatch');
1338
+ validateCaptureSnapshot(
1339
+ JSON.parse(await fsOps.readFile(sourceTemporary, 'utf8')),
1340
+ value.sourceRevision,
1341
+ );
1342
+ }
1193
1343
  await fsOps.writeFile(temporary, serialized, { flag: 'wx', mode: 0o600 });
1194
1344
  const written = await assertPrivateFile(temporary, 'capture candidate', fsOps);
1195
1345
  if (written.size !== Buffer.byteLength(serialized, 'utf8'))
@@ -1198,7 +1348,9 @@ async function writeBoundedCapture(file, serialized, value, fsOps = captureFs, l
1198
1348
  JSON.parse(await fsOps.readFile(temporary, 'utf8')),
1199
1349
  value.sourceRevision,
1200
1350
  );
1201
- const state = await readCaptureDirectoryState(directory, fsOps);
1351
+ const state = await readCaptureDirectoryState(directory, fsOps, {
1352
+ ...(pairedSourceTarget === null ? {} : { replacedCurrentName: targetName }),
1353
+ });
1202
1354
  if (state.unsafeRecovery || state.blockedEntries) throw captureCapacityError(state);
1203
1355
  const protectedNames = new Set([targetName, sourceName]);
1204
1356
  if (value.version === 2 && Array.isArray(value.captureManifest?.segments)) {
@@ -1231,7 +1383,10 @@ async function writeBoundedCapture(file, serialized, value, fsOps = captureFs, l
1231
1383
  const temporaryHeadroom = 2 * MAX_CAPTURE_BYTES;
1232
1384
  let projectedPeakBytes = state.totalBytes;
1233
1385
  let projectedSteadyBytes = state.totalBytes - replacedBytes;
1234
- let projectedFiles = state.managedFiles.length + (targetWasManaged ? 0 : 1);
1386
+ let projectedFiles =
1387
+ state.managedFiles.length +
1388
+ (sourceTemporary !== null && !existingSource ? 1 : 0) +
1389
+ (targetWasManaged ? 0 : 1);
1235
1390
  const retire = [];
1236
1391
  for (const group of eligible) {
1237
1392
  if (
@@ -1251,14 +1406,25 @@ async function writeBoundedCapture(file, serialized, value, fsOps = captureFs, l
1251
1406
  projectedSteadyBytes > MAX_CAPTURE_DIRECTORY_BYTES ||
1252
1407
  projectedFiles > MAX_CAPTURE_FILES
1253
1408
  )
1254
- throw captureCapacityError(state);
1409
+ throw captureCapacityError({
1410
+ ...state,
1411
+ totalBytes: projectedSteadyBytes,
1412
+ managedFileCount: projectedFiles,
1413
+ });
1255
1414
  const backup = existingTarget
1256
1415
  ? `${target}.${process.pid}.${randomUUID()}.capture-backup`
1257
1416
  : null;
1417
+ const retiredBytes = retire.reduce((total, entry) => total + entry.size, 0);
1258
1418
  const staged = [];
1259
1419
  let backupCreated = false;
1260
1420
  let candidatePublished = false;
1421
+ let sourcePublished = false;
1422
+ let committed = false;
1261
1423
  try {
1424
+ if (sourceTemporary !== null && !existingSource) {
1425
+ await fsOps.rename(sourceTemporary, pairedSourceTarget);
1426
+ sourcePublished = true;
1427
+ }
1262
1428
  if (backup !== null) {
1263
1429
  await fsOps.rename(target, backup);
1264
1430
  backupCreated = true;
@@ -1270,18 +1436,72 @@ async function writeBoundedCapture(file, serialized, value, fsOps = captureFs, l
1270
1436
  await fsOps.rename(candidate.file, stagedPath);
1271
1437
  staged.push({ original: candidate.file, staged: stagedPath });
1272
1438
  }
1273
- if (backup !== null) await fsOps.unlink(backup).catch(() => undefined);
1274
- for (const item of staged) await fsOps.unlink(item.staged).catch(() => undefined);
1439
+ const publishedSourceTarget =
1440
+ pairedSourceTarget ?? (targetName === sourceName ? target : null);
1441
+ if (publishedSourceTarget !== null) {
1442
+ const sourceValue = JSON.parse(
1443
+ await readPrivateFile(
1444
+ publishedSourceTarget,
1445
+ 'published immutable capture source',
1446
+ MAX_CAPTURE_BYTES,
1447
+ fsOps,
1448
+ ),
1449
+ );
1450
+ validateCaptureSnapshot(sourceValue, value.sourceRevision);
1451
+ if (sourceValue.threadId !== value.threadId)
1452
+ throw new Error('published capture source identity validation failed');
1453
+ }
1454
+ const publishedCurrent = JSON.parse(
1455
+ await readPrivateFile(target, 'published current capture', MAX_CAPTURE_BYTES, fsOps),
1456
+ );
1457
+ validateCaptureSnapshot(publishedCurrent, value.sourceRevision);
1458
+ if (publishedCurrent.threadId !== value.threadId)
1459
+ throw new Error('published capture group identity validation failed');
1460
+ const postState = await readCaptureDirectoryState(directory, fsOps);
1461
+ const postSteadyBytes =
1462
+ postState.totalBytes - (backupCreated ? replacedBytes : 0) - retiredBytes;
1463
+ if (
1464
+ postState.unsafeRecovery ||
1465
+ postState.blockedEntries ||
1466
+ postState.managedFiles.length > MAX_CAPTURE_FILES ||
1467
+ postSteadyBytes > MAX_CAPTURE_DIRECTORY_BYTES
1468
+ )
1469
+ throw captureCapacityError({
1470
+ totalBytes: postSteadyBytes,
1471
+ managedFiles: postState.managedFiles,
1472
+ });
1473
+ committed = true;
1474
+ let cleanupFailed = false;
1475
+ const removeCommittedPath = async (path) => {
1476
+ try {
1477
+ await fsOps.unlink(path);
1478
+ } catch (error) {
1479
+ if (error?.code !== 'ENOENT') cleanupFailed = true;
1480
+ }
1481
+ };
1482
+ if (backup !== null) await removeCommittedPath(backup);
1483
+ for (const item of staged) await removeCommittedPath(item.staged);
1484
+ if (cleanupFailed) {
1485
+ let actualState = postState;
1486
+ try {
1487
+ actualState = await readCaptureDirectoryState(directory, fsOps);
1488
+ } catch {}
1489
+ throw captureStorageCleanupError(actualState);
1490
+ }
1275
1491
  } catch (error) {
1492
+ if (committed) throw error;
1276
1493
  if (candidatePublished) await fsOps.unlink(target).catch(() => undefined);
1277
1494
  for (const item of staged)
1278
1495
  await fsOps.rename(item.staged, item.original).catch(() => undefined);
1279
1496
  if (backupCreated && backup !== null)
1280
1497
  await fsOps.rename(backup, target).catch(() => undefined);
1498
+ if (sourcePublished && pairedSourceTarget !== null)
1499
+ await fsOps.unlink(pairedSourceTarget).catch(() => undefined);
1281
1500
  throw error;
1282
1501
  }
1283
1502
  } catch (error) {
1284
1503
  await fsOps.unlink(temporary).catch(() => undefined);
1504
+ if (sourceTemporary !== null) await fsOps.unlink(sourceTemporary).catch(() => undefined);
1285
1505
  throw error;
1286
1506
  }
1287
1507
  };
@@ -1290,6 +1510,38 @@ async function writeBoundedCapture(file, serialized, value, fsOps = captureFs, l
1290
1510
  : withCaptureStoreLock(directory, operation, fsOps, lockOptions);
1291
1511
  }
1292
1512
 
1513
+ async function retireUnreferencedCaptureGroup(
1514
+ directory,
1515
+ groupName,
1516
+ fsOps = captureFs,
1517
+ lockHeld = false,
1518
+ ) {
1519
+ const operation = async () => {
1520
+ const state = await readCaptureDirectoryState(directory, fsOps);
1521
+ if (state.unsafeRecovery || state.blockedEntries || state.protectedGroups.has(groupName))
1522
+ return false;
1523
+ const members = state.managedFiles.filter(
1524
+ (entry) => entry.group === groupName && entry.immutable,
1525
+ );
1526
+ if (members.length === 0) return false;
1527
+ const staged = [];
1528
+ try {
1529
+ for (const entry of members) {
1530
+ const stagedPath = `${entry.file}.${process.pid}.${randomUUID()}.capture-retire`;
1531
+ await fsOps.rename(entry.file, stagedPath);
1532
+ staged.push({ original: entry.file, staged: stagedPath });
1533
+ }
1534
+ } catch (error) {
1535
+ for (const item of staged)
1536
+ await fsOps.rename(item.staged, item.original).catch(() => undefined);
1537
+ throw error;
1538
+ }
1539
+ for (const item of staged) await fsOps.unlink(item.staged).catch(() => undefined);
1540
+ return true;
1541
+ };
1542
+ return lockHeld ? operation() : withCaptureStoreLock(directory, operation, fsOps);
1543
+ }
1544
+
1293
1545
  async function atomicWritePrivate(
1294
1546
  file,
1295
1547
  value,
@@ -1299,6 +1551,7 @@ async function atomicWritePrivate(
1299
1551
  capture = false,
1300
1552
  captureLock = false,
1301
1553
  captureLockHeld = false,
1554
+ pairedSourcePath,
1302
1555
  } = {},
1303
1556
  ) {
1304
1557
  const serialized = `${compact ? JSON.stringify(value) : JSON.stringify(value, null, 2)}\n`;
@@ -1310,36 +1563,49 @@ async function atomicWritePrivate(
1310
1563
  throw error;
1311
1564
  }
1312
1565
  if (capture) {
1313
- const result = await writeBoundedCapture(file, serialized, value, captureFs, {
1566
+ const lockOptions = {
1314
1567
  lockHeld: captureLockHeld,
1315
- });
1568
+ ...(pairedSourcePath === undefined ? {} : { pairedSourcePath }),
1569
+ };
1570
+ const result = await writeBoundedCapture(file, serialized, value, captureFs, lockOptions);
1316
1571
  return result;
1317
1572
  }
1318
1573
  const directory = await ensurePrivateDirectory(dirname(file));
1319
1574
  if (captureLock) return withCaptureStoreLock(directory, () => writePrivateFile(file, serialized));
1320
1575
  return writePrivateFile(file, serialized);
1321
1576
  }
1322
- async function atomicWriteImmutablePrivate(file, value, sourceRevision, captureLockHeld = false) {
1323
- validateCaptureSnapshot(value, sourceRevision);
1324
- await atomicWritePrivate(file, value, {
1325
- maxBytes: MAX_CAPTURE_BYTES,
1326
- compact: true,
1327
- capture: true,
1328
- captureLockHeld,
1329
- });
1330
- }
1331
-
1332
1577
  async function writeSegmentedCapture(
1333
1578
  directory,
1334
1579
  { threadId, sourceRevision, metadata, pages, lockHeld = false, fs = captureFs },
1335
1580
  ) {
1581
+ if (!lockHeld)
1582
+ return withCaptureStoreLock(
1583
+ directory,
1584
+ () =>
1585
+ writeSegmentedCapture(directory, {
1586
+ threadId,
1587
+ sourceRevision,
1588
+ metadata,
1589
+ pages,
1590
+ lockHeld: true,
1591
+ fs,
1592
+ }),
1593
+ fs,
1594
+ );
1336
1595
  if (!Array.isArray(pages) || pages.length < 1 || pages.length > MAX_CAPTURE_PAGES)
1337
1596
  throw new Error('capture page count exceeds bound');
1338
1597
  const segments = [];
1339
- const prepared = [];
1598
+ const ownedStagePaths = new Set();
1340
1599
  const cleanupSource = async () => {
1341
- for (const descriptor of pages) {
1342
- if (descriptor?.stagedPath) await fs.unlink(descriptor.stagedPath).catch(() => undefined);
1600
+ for (const file of ownedStagePaths) {
1601
+ try {
1602
+ await assertPrivateFile(file, 'capture staged page cleanup', fs);
1603
+ } catch {
1604
+ continue;
1605
+ }
1606
+ await fs.unlink(file).catch((error) => {
1607
+ if (error?.code !== 'ENOENT') throw error;
1608
+ });
1343
1609
  }
1344
1610
  };
1345
1611
  try {
@@ -1353,13 +1619,16 @@ async function writeSegmentedCapture(
1353
1619
  !/^[a-f0-9]{64}$/u.test(descriptor.pageHash)
1354
1620
  )
1355
1621
  throw new Error('capture staged page identity validation failed');
1622
+ const stagedPath = resolve(descriptor.stagedPath);
1623
+ if (
1624
+ dirname(stagedPath) !== resolve(directory) ||
1625
+ !CAPTURE_SOURCE_STAGE_NAME.test(basename(stagedPath))
1626
+ )
1627
+ throw new Error('capture staged page path validation failed');
1628
+ const stagedInfo = await assertPrivateFile(stagedPath, 'capture staged page', fs);
1629
+ ownedStagePaths.add(stagedPath);
1356
1630
  const page = JSON.parse(
1357
- await readPrivateFile(
1358
- descriptor.stagedPath,
1359
- 'capture staged page',
1360
- MAX_CAPTURE_PAGE_BYTES,
1361
- fs,
1362
- ),
1631
+ await readPrivateFile(stagedPath, 'capture staged page', MAX_CAPTURE_PAGE_BYTES, fs),
1363
1632
  );
1364
1633
  if (
1365
1634
  !isRecord(page) ||
@@ -1386,27 +1655,28 @@ async function writeSegmentedCapture(
1386
1655
  throw new Error('capture page exceeds bound');
1387
1656
  const stagedState = await readCaptureDirectoryState(directory, fs);
1388
1657
  if (
1389
- stagedState.totalBytes + Buffer.byteLength(serialized, 'utf8') >
1658
+ stagedState.totalBytes - stagedInfo.size + Buffer.byteLength(serialized, 'utf8') >
1390
1659
  MAX_CAPTURE_DIRECTORY_BYTES + MAX_CAPTURE_STAGING_HEADROOM_BYTES
1391
1660
  )
1392
1661
  throw captureCapacityError(stagedState);
1393
- const stagedFinalPath = resolve(
1394
- directory,
1395
- `.capture-segment-stage.${safeThreadName(threadId)}.${randomUUID()}.${descriptor.pageIndex}.tmp`,
1662
+ await fs.writeFile(stagedPath, serialized, { flag: 'w', mode: 0o600 });
1663
+ const rewritten = await assertPrivateFile(stagedPath, 'capture staged segment', fs);
1664
+ if (rewritten.size !== Buffer.byteLength(serialized, 'utf8'))
1665
+ throw new Error('capture staged segment read-back size mismatch');
1666
+ validateCaptureSegment(
1667
+ JSON.parse(await fs.readFile(stagedPath, 'utf8')),
1668
+ threadId,
1669
+ sourceRevision,
1396
1670
  );
1397
- await fs.writeFile(stagedFinalPath, serialized, { flag: 'wx', mode: 0o600 });
1398
- prepared.push(stagedFinalPath);
1399
1671
  segments.push({
1400
1672
  name: captureSegmentName(threadId, sourceRevision, descriptor.pageIndex),
1401
1673
  pageHash: descriptor.pageHash,
1402
1674
  itemCount: payload.itemCount,
1403
1675
  bytes: Buffer.byteLength(serialized, 'utf8'),
1404
- stagedPath: descriptor.stagedPath,
1405
- stagedFinalPath,
1676
+ stagedPath,
1406
1677
  });
1407
1678
  }
1408
1679
  } catch (error) {
1409
- for (const file of prepared) await fs.unlink(file).catch(() => undefined);
1410
1680
  await cleanupSource();
1411
1681
  throw error;
1412
1682
  }
@@ -1425,19 +1695,16 @@ async function writeSegmentedCapture(
1425
1695
  captureManifestRevision(metadata.thread, manifest.pageHashes) !== sourceRevision
1426
1696
  ) {
1427
1697
  await cleanupSource();
1428
- for (const file of prepared) await fs.unlink(file).catch(() => undefined);
1429
1698
  throw new Error('capture manifest revision does not match page hashes');
1430
1699
  }
1431
1700
  if (manifest.totalBytes > MAX_CAPTURE_TOTAL_BYTES || manifest.itemCount > MAX_CAPTURE_ITEMS) {
1432
1701
  await cleanupSource();
1433
- for (const file of prepared) await fs.unlink(file).catch(() => undefined);
1434
1702
  throw new Error('capture exceeds segmented bound');
1435
1703
  }
1436
1704
  const operation = async () => {
1437
1705
  const state = await readCaptureDirectoryState(directory, fs);
1438
1706
  if (state.unsafeRecovery || state.blockedEntries) {
1439
1707
  await cleanupSource();
1440
- for (const file of prepared) await fs.unlink(file).catch(() => undefined);
1441
1708
  throw captureCapacityError(state);
1442
1709
  }
1443
1710
  const staged = [];
@@ -1452,12 +1719,11 @@ async function writeSegmentedCapture(
1452
1719
  );
1453
1720
  validateCaptureSegment(existing, threadId, sourceRevision);
1454
1721
  if (existing.pageHash !== segment.pageHash) throw new Error('capture segment collision');
1455
- await fs.unlink(segment.stagedFinalPath).catch(() => undefined);
1456
1722
  continue;
1457
1723
  } catch (error) {
1458
1724
  if (error?.code !== 'ENOENT') throw error;
1459
1725
  }
1460
- staged.push({ temporary: segment.stagedFinalPath, target, bytes: segment.bytes });
1726
+ staged.push({ temporary: segment.stagedPath, target, bytes: segment.bytes });
1461
1727
  }
1462
1728
  const protectedNames = new Set(segments.map((segment) => segment.name));
1463
1729
  const groups = new Map();
@@ -1487,8 +1753,12 @@ async function writeSegmentedCapture(
1487
1753
  projectedFiles -= members.length;
1488
1754
  retired.push(...members);
1489
1755
  }
1490
- if (projected > MAX_CAPTURE_DIRECTORY_BYTES || projectedFiles > MAX_CAPTURE_FILES)
1491
- throw captureCapacityError({ totalBytes: projected, managedFiles: state.managedFiles });
1756
+ if (projected > MAX_CAPTURE_DIRECTORY_BYTES || projectedFiles > MAX_CAPTURE_PUBLICATION_FILES)
1757
+ throw captureCapacityError({
1758
+ totalBytes: projected,
1759
+ managedFileCount: projectedFiles,
1760
+ managedFiles: state.managedFiles,
1761
+ });
1492
1762
  const retiredPaths = [];
1493
1763
  try {
1494
1764
  for (const entry of retired) {
@@ -1509,8 +1779,7 @@ async function writeSegmentedCapture(
1509
1779
  );
1510
1780
  validateCaptureSegment(verify, threadId, sourceRevision);
1511
1781
  }
1512
- for (const segment of segments) await fs.unlink(segment.stagedPath).catch(() => undefined);
1513
- for (const file of prepared) await fs.unlink(file).catch(() => undefined);
1782
+ await cleanupSource();
1514
1783
  for (const item of retiredPaths) await fs.unlink(item.staged).catch(() => undefined);
1515
1784
  return manifest;
1516
1785
  } catch (error) {
@@ -1525,12 +1794,11 @@ async function writeSegmentedCapture(
1525
1794
  await fs.unlink(item.target).catch(() => undefined);
1526
1795
  }
1527
1796
  for (const target of publishedTargets) await fs.unlink(target).catch(() => undefined);
1528
- for (const segment of segments) await fs.unlink(segment.stagedPath).catch(() => undefined);
1529
- for (const file of prepared) await fs.unlink(file).catch(() => undefined);
1797
+ await cleanupSource();
1530
1798
  throw error;
1531
1799
  }
1532
1800
  };
1533
- return lockHeld ? operation() : withCaptureStoreLock(directory, operation, fs);
1801
+ return operation();
1534
1802
  }
1535
1803
 
1536
1804
  /**
@@ -1543,36 +1811,145 @@ export async function __testWriteSegmentedCapture(
1543
1811
  { fs = captureFs } = {},
1544
1812
  ) {
1545
1813
  await ensurePrivateDirectory(directory, fs);
1546
- const staged = [];
1547
- try {
1548
- for (const [pageIndex, page] of pages.entries()) {
1549
- const serialized = `${JSON.stringify(page)}\n`;
1550
- const state = await readCaptureDirectoryState(directory, fs);
1551
- if (state.totalBytes + Buffer.byteLength(serialized, 'utf8') > MAX_CAPTURE_DIRECTORY_BYTES)
1552
- throw captureCapacityError(state);
1553
- const stagedPath = resolve(
1554
- directory,
1555
- `.capture-stage.test.${safeThreadName(threadId)}.${randomUUID()}.${pageIndex}.tmp`,
1556
- );
1557
- await fs.writeFile(stagedPath, serialized, { flag: 'wx', mode: 0o600 });
1558
- staged.push({
1559
- pageIndex,
1560
- stagedPath,
1561
- pageHash: hashJson(page),
1562
- bytes: Buffer.byteLength(serialized, 'utf8'),
1563
- });
1564
- }
1565
- return await writeSegmentedCapture(directory, {
1566
- threadId,
1567
- sourceRevision,
1568
- metadata,
1569
- pages: staged,
1570
- fs,
1571
- });
1572
- } catch (error) {
1573
- for (const page of staged) await fs.unlink(page.stagedPath).catch(() => undefined);
1574
- throw error;
1575
- }
1814
+ return withCaptureStoreLock(
1815
+ directory,
1816
+ async () => {
1817
+ const staged = [];
1818
+ try {
1819
+ for (const [pageIndex, page] of pages.entries()) {
1820
+ const serialized = `${JSON.stringify(page)}\n`;
1821
+ const state = await readCaptureDirectoryState(directory, fs);
1822
+ if (
1823
+ state.totalBytes + Buffer.byteLength(serialized, 'utf8') >
1824
+ MAX_CAPTURE_DIRECTORY_BYTES + MAX_CAPTURE_STAGING_HEADROOM_BYTES
1825
+ )
1826
+ throw captureCapacityError(state);
1827
+ const stagedPath = resolve(
1828
+ directory,
1829
+ `.capture-stage.${safeThreadName(threadId)}.${randomUUID()}.${pageIndex}.tmp`,
1830
+ );
1831
+ await fs.writeFile(stagedPath, serialized, { flag: 'wx', mode: 0o600 });
1832
+ staged.push({
1833
+ pageIndex,
1834
+ stagedPath,
1835
+ pageHash: hashJson(page),
1836
+ bytes: Buffer.byteLength(serialized, 'utf8'),
1837
+ });
1838
+ }
1839
+ return await writeSegmentedCapture(directory, {
1840
+ threadId,
1841
+ sourceRevision,
1842
+ metadata,
1843
+ pages: staged,
1844
+ lockHeld: true,
1845
+ fs,
1846
+ });
1847
+ } catch (error) {
1848
+ for (const page of staged) await fs.unlink(page.stagedPath).catch(() => undefined);
1849
+ throw error;
1850
+ }
1851
+ },
1852
+ fs,
1853
+ );
1854
+ }
1855
+
1856
+ export async function __testWriteSegmentedCaptureFromStages(
1857
+ directory,
1858
+ { threadId, sourceRevision, metadata = { thread: { id: threadId } }, pages },
1859
+ { fs = captureFs } = {},
1860
+ ) {
1861
+ await ensurePrivateDirectory(directory, fs);
1862
+ return withCaptureStoreLock(
1863
+ directory,
1864
+ () =>
1865
+ writeSegmentedCapture(directory, {
1866
+ threadId,
1867
+ sourceRevision,
1868
+ metadata,
1869
+ pages,
1870
+ lockHeld: true,
1871
+ fs,
1872
+ }),
1873
+ fs,
1874
+ );
1875
+ }
1876
+
1877
+ export async function __testPublishSegmentedCapture(
1878
+ directory,
1879
+ { threadId, sourceRevision, metadata = { thread: { id: threadId } }, pages },
1880
+ { fs = captureFs } = {},
1881
+ ) {
1882
+ await ensurePrivateDirectory(directory, fs);
1883
+ return withCaptureStoreLock(
1884
+ directory,
1885
+ async () => {
1886
+ const staged = [];
1887
+ let segmentsPublished = false;
1888
+ try {
1889
+ for (const [pageIndex, page] of pages.entries()) {
1890
+ const serialized = `${JSON.stringify(page)}\n`;
1891
+ const state = await readCaptureDirectoryState(directory, fs);
1892
+ if (
1893
+ state.totalBytes + Buffer.byteLength(serialized, 'utf8') >
1894
+ MAX_CAPTURE_DIRECTORY_BYTES + MAX_CAPTURE_STAGING_HEADROOM_BYTES
1895
+ )
1896
+ throw captureCapacityError(state);
1897
+ const stagedPath = resolve(
1898
+ directory,
1899
+ `.capture-stage.${safeThreadName(threadId)}.${randomUUID()}.${pageIndex}.tmp`,
1900
+ );
1901
+ await fs.writeFile(stagedPath, serialized, { flag: 'wx', mode: 0o600 });
1902
+ staged.push({
1903
+ pageIndex,
1904
+ stagedPath,
1905
+ pageHash: hashJson(page),
1906
+ bytes: Buffer.byteLength(serialized, 'utf8'),
1907
+ });
1908
+ }
1909
+ const manifest = await writeSegmentedCapture(directory, {
1910
+ threadId,
1911
+ sourceRevision,
1912
+ metadata,
1913
+ pages: staged,
1914
+ lockHeld: true,
1915
+ fs,
1916
+ });
1917
+ segmentsPublished = true;
1918
+ const value = {
1919
+ version: 2,
1920
+ threadId,
1921
+ sourceRevision,
1922
+ threadRead: metadata,
1923
+ captureManifest: manifest,
1924
+ };
1925
+ await writeBoundedCapture(
1926
+ resolve(directory, `${strictThreadName(threadId)}.json`),
1927
+ `${JSON.stringify(value)}\n`,
1928
+ value,
1929
+ fs,
1930
+ {
1931
+ lockHeld: true,
1932
+ pairedSourcePath: resolve(
1933
+ directory,
1934
+ `${strictThreadName(threadId)}.${sourceRevision}.json`,
1935
+ ),
1936
+ },
1937
+ );
1938
+ return { manifest, value };
1939
+ } catch (error) {
1940
+ for (const page of staged) await fs.unlink(page.stagedPath).catch(() => undefined);
1941
+ if (segmentsPublished)
1942
+ await retireUnreferencedCaptureGroup(
1943
+ directory,
1944
+ `${threadId}.${sourceRevision}`,
1945
+ fs,
1946
+ true,
1947
+ ).catch(() => undefined);
1948
+ throw error;
1949
+ }
1950
+ },
1951
+ fs,
1952
+ );
1576
1953
  }
1577
1954
 
1578
1955
  // Test-only seam for deterministic capture-store fault injection. Production
@@ -1589,6 +1966,26 @@ export async function __testWriteBoundedCapture(
1589
1966
  return writeBoundedCapture(file, serialized, value, fs, { now, sleep });
1590
1967
  }
1591
1968
 
1969
+ export async function maintainCaptureStorage(directory, { fs = captureFs } = {}) {
1970
+ const target = await validatePrivateDirectoryPath(directory, fs);
1971
+ return withCaptureStoreLock(
1972
+ target,
1973
+ async ({ removedFiles = 0, removedBytes = 0 } = {}) => {
1974
+ const state = await readCaptureDirectoryState(target, fs);
1975
+ if (state.unsafeRecovery || state.blockedEntries) throw captureCapacityError(state);
1976
+ return {
1977
+ reclaimedFiles: removedFiles,
1978
+ reclaimedBytes: removedBytes,
1979
+ managedFiles: state.managedFiles.length,
1980
+ directoryBytes: state.totalBytes,
1981
+ fileLimit: MAX_CAPTURE_FILES,
1982
+ directoryByteLimit: MAX_CAPTURE_DIRECTORY_BYTES,
1983
+ };
1984
+ },
1985
+ fs,
1986
+ );
1987
+ }
1988
+
1592
1989
  export async function __testHoldCaptureStoreLock(directory, readyPath) {
1593
1990
  await ensurePrivateDirectory(directory);
1594
1991
  return withCaptureStoreLock(directory, async () => {
@@ -3268,25 +3665,31 @@ export function createCodexContextProxy({
3268
3665
  } else if (Buffer.byteLength(JSON.stringify(compactSnapshot), 'utf8') > MAX_CAPTURE_BYTES) {
3269
3666
  throw new Error('capture snapshot exceeds bound');
3270
3667
  }
3271
- const manifestForRollback = snapshot.version === 2 ? snapshot.captureManifest : null;
3272
3668
  const immutableCapturePath = resolve(
3273
3669
  checkpointDir,
3274
3670
  `${safeThreadName(threadId)}.${sourceRevision}.json`,
3275
3671
  );
3276
3672
  try {
3277
- await atomicWriteImmutablePrivate(immutableCapturePath, snapshot, sourceRevision, true);
3673
+ await atomicWritePrivate(
3674
+ resolve(checkpointDir, `${safeThreadName(threadId)}.json`),
3675
+ snapshot,
3676
+ {
3677
+ maxBytes: MAX_CAPTURE_BYTES,
3678
+ compact: true,
3679
+ capture: true,
3680
+ captureLockHeld: true,
3681
+ pairedSourcePath: immutableCapturePath,
3682
+ },
3683
+ );
3278
3684
  } catch (error) {
3279
- if (manifestForRollback?.segments) {
3280
- for (const segment of manifestForRollback.segments)
3281
- await unlink(resolve(checkpointDir, segment)).catch(() => undefined);
3282
- }
3685
+ await retireUnreferencedCaptureGroup(
3686
+ checkpointDir,
3687
+ `${threadId}.${sourceRevision}`,
3688
+ captureFs,
3689
+ true,
3690
+ ).catch(() => undefined);
3283
3691
  throw error;
3284
3692
  }
3285
- await atomicWritePrivate(
3286
- resolve(checkpointDir, `${safeThreadName(threadId)}.json`),
3287
- snapshot,
3288
- { maxBytes: MAX_CAPTURE_BYTES, compact: true, capture: true, captureLockHeld: true },
3289
- );
3290
3693
  return { capture, read, sourceRevision, snapshot, immutableCapturePath };
3291
3694
  });
3292
3695
  const { capture, read, sourceRevision, snapshot, immutableCapturePath } = publication;