@brunobracaioli/aibrain 0.3.0-beta.7 → 0.4.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +80 -15
- package/dist/assets/claude/.claude-plugin/plugin.json +1 -1
- package/dist/assets/claude/hooks/context-compaction.ts +1100 -77
- package/dist/assets/claude/hooks/native-host-contract-runtime.ts +18 -0
- package/dist/assets/claude/hooks/native-host-contract.test.ts +51 -0
- package/dist/assets/codex-context-compaction.mjs +2179 -194
- package/dist/assets/jev-paid-pilot-terminal.mjs +334 -35
- package/dist/claude-compatibility.mjs +506 -0
- package/dist/claude-observation.mjs +117 -3
- package/dist/cli.mjs +105 -28
- package/dist/codex-compatibility.mjs +481 -0
- package/dist/codex-diagnostics.mjs +39 -3
- package/dist/codex-protocol-fingerprint.json +131 -0
- package/dist/engineering-pack-public-key.json +10 -0
- package/dist/engineering-pack.mjs +722 -0
- package/dist/engineering.mjs +194 -0
- package/dist/launch.mjs +71 -14
- package/dist/mcp.mjs +8 -8
- package/dist/package.json +1 -1
- package/dist/platform.mjs +47 -3
- package/package.json +3 -2
|
@@ -1,29 +1,68 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
chmod,
|
|
5
|
+
link as hardlink,
|
|
6
|
+
lstat,
|
|
7
|
+
mkdir,
|
|
8
|
+
open,
|
|
9
|
+
readFile,
|
|
10
|
+
readlink,
|
|
11
|
+
readdir,
|
|
12
|
+
rename,
|
|
13
|
+
unlink,
|
|
14
|
+
writeFile,
|
|
15
|
+
} from 'node:fs/promises';
|
|
16
|
+
import { constants as fsConstants } from 'node:fs';
|
|
4
17
|
import { spawn } from 'node:child_process';
|
|
5
18
|
import { createHash, randomUUID } from 'node:crypto';
|
|
6
|
-
import { dirname, relative, resolve } from 'node:path';
|
|
19
|
+
import { basename, dirname, relative, resolve } from 'node:path';
|
|
7
20
|
import { StringDecoder } from 'node:string_decoder';
|
|
8
21
|
import { pathToFileURL } from 'node:url';
|
|
9
22
|
import { createTelemetryReporter, TELEMETRY_TIMEOUT_MS } from './context-telemetry-reporter.mjs';
|
|
10
23
|
import { preflightSelectionState } from './context-selection-fitting.mjs';
|
|
11
24
|
|
|
25
|
+
const captureFs = {
|
|
26
|
+
chmod,
|
|
27
|
+
hardlink,
|
|
28
|
+
lstat,
|
|
29
|
+
mkdir,
|
|
30
|
+
open,
|
|
31
|
+
readFile,
|
|
32
|
+
readdir,
|
|
33
|
+
rename,
|
|
34
|
+
unlink,
|
|
35
|
+
writeFile,
|
|
36
|
+
};
|
|
37
|
+
|
|
12
38
|
const DEFAULT_THRESHOLD = 85;
|
|
13
39
|
const COOLDOWN_MS = 60_000;
|
|
14
40
|
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
15
|
-
const MAX_LINE_BYTES =
|
|
16
|
-
const MAX_CAPTURE_BYTES =
|
|
41
|
+
const MAX_LINE_BYTES = 8 * 1024 * 1024;
|
|
42
|
+
const MAX_CAPTURE_BYTES = 8 * 1024 * 1024;
|
|
43
|
+
const MAX_CAPTURE_PAGE_BYTES = 8 * 1024 * 1024;
|
|
44
|
+
const MAX_CAPTURE_TOTAL_BYTES = 128 * 1024 * 1024;
|
|
45
|
+
const MAX_CAPTURE_PAGES = 32;
|
|
46
|
+
const MAX_CAPTURE_ITEMS = 65_536;
|
|
47
|
+
const MAX_CAPTURE_DIRECTORY_BYTES = 256 * 1024 * 1024;
|
|
48
|
+
const MAX_CAPTURE_STAGING_HEADROOM_BYTES = 16 * 1024 * 1024;
|
|
49
|
+
const MAX_CAPTURE_FILES = 128;
|
|
50
|
+
const MAX_CAPTURE_PUBLICATION_FILE_HEADROOM = MAX_CAPTURE_PAGES + 2;
|
|
51
|
+
const MAX_CAPTURE_PUBLICATION_FILES = MAX_CAPTURE_FILES + MAX_CAPTURE_PUBLICATION_FILE_HEADROOM;
|
|
52
|
+
const CAPTURE_LOCK_WAIT_MS = 1_000;
|
|
17
53
|
const MAX_OWNED_REQUESTS = 1_000;
|
|
18
54
|
const MAX_RETIRED_REQUESTS = 2_000;
|
|
55
|
+
const MAX_CLIENT_REQUEST_BINDINGS = 128;
|
|
56
|
+
const MAX_OWNED_THREAD_IDS = 256;
|
|
19
57
|
const MAX_RECOVERY_ITEMS = 128;
|
|
20
|
-
const MAX_RECOVERY_SOURCE_INDEX =
|
|
58
|
+
const MAX_RECOVERY_SOURCE_INDEX = 65_536;
|
|
21
59
|
const MAX_NOTE_BYTES = 128 * 1024;
|
|
22
60
|
const MAX_SELECTION_BYTES = 512 * 1024;
|
|
23
61
|
const MAX_POLICY_BYTES = 8 * 1024;
|
|
24
62
|
const MAX_NOTE_ITEM_CHARS = 16_384;
|
|
25
63
|
const USAGE_FRESHNESS_MS = 60_000;
|
|
26
64
|
const MAX_DIAGNOSTIC_STRING = 128;
|
|
65
|
+
const MAX_DIAGNOSTIC_BYTES = MAX_CAPTURE_DIRECTORY_BYTES;
|
|
27
66
|
const MAX_BUDGET_SNAPSHOT_AGE_MS = 15_000;
|
|
28
67
|
const CONTEXT_ERROR_CODES = new Set([
|
|
29
68
|
'INVALID_INPUT',
|
|
@@ -79,6 +118,14 @@ const DIAGNOSTIC_REASONS = new Set([
|
|
|
79
118
|
'timeout',
|
|
80
119
|
'thread_closed',
|
|
81
120
|
'protocol_error',
|
|
121
|
+
'capture_size_exceeded',
|
|
122
|
+
'capture_item_count_exceeded',
|
|
123
|
+
'private_storage_failed',
|
|
124
|
+
'storage_capacity_exceeded',
|
|
125
|
+
'capture_lock_timeout',
|
|
126
|
+
'capture_lock_legacy',
|
|
127
|
+
'capture_lock_owner_unverifiable',
|
|
128
|
+
'capture_lock_backend_unavailable',
|
|
82
129
|
'attached',
|
|
83
130
|
'uncertain',
|
|
84
131
|
'disconnected',
|
|
@@ -335,6 +382,16 @@ function boundedCorrelationValue(value) {
|
|
|
335
382
|
return typeof value === 'string' && value.length > 0 && value.length <= 256 ? value : null;
|
|
336
383
|
}
|
|
337
384
|
|
|
385
|
+
function safeRpcCode(value) {
|
|
386
|
+
return Number.isSafeInteger(value) && value >= -32_768 && value <= 32_767 ? value : null;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
function clientRequestKey(value) {
|
|
390
|
+
if (typeof value === 'string' && value.length > 0 && value.length <= 128) return `s:${value}`;
|
|
391
|
+
if (Number.isSafeInteger(value)) return `n:${value}`;
|
|
392
|
+
return null;
|
|
393
|
+
}
|
|
394
|
+
|
|
338
395
|
/**
|
|
339
396
|
* Closed private diagnostic contract consumed by the client sink. It contains
|
|
340
397
|
* stage/reason enums, pseudonymous session identity, operation/correlation
|
|
@@ -355,12 +412,14 @@ export function normalizeCodexDiagnostic(event, { threadId = null } = {}) {
|
|
|
355
412
|
: null;
|
|
356
413
|
if (stage === null || reason === null) return null;
|
|
357
414
|
const operationId = isUuid(event.operationId) ? event.operationId : undefined;
|
|
358
|
-
const correlationId =
|
|
415
|
+
const correlationId = isSafeCorrelationId(event.correlationId) ? event.correlationId : null;
|
|
416
|
+
const rpcCode = safeRpcCode(event.rpcCode ?? event.rpc_code);
|
|
359
417
|
const counts = isRecord(event.counts)
|
|
360
418
|
? Object.fromEntries(
|
|
361
419
|
Object.entries(event.counts).filter(
|
|
362
420
|
([key, value]) =>
|
|
363
|
-
/^[a-z_]{1,48}$/u.test(key) &&
|
|
421
|
+
/^[a-z_]{1,48}$/u.test(key) &&
|
|
422
|
+
isSafeBoundedInteger(value, 0, /_bytes$/u.test(key) ? MAX_DIAGNOSTIC_BYTES : 1_000_000),
|
|
364
423
|
),
|
|
365
424
|
)
|
|
366
425
|
: undefined;
|
|
@@ -371,6 +430,7 @@ export function normalizeCodexDiagnostic(event, { threadId = null } = {}) {
|
|
|
371
430
|
reason,
|
|
372
431
|
...(operationId === undefined ? {} : { operation_id: operationId }),
|
|
373
432
|
...(correlationId === null ? {} : { correlation_id: correlationId }),
|
|
433
|
+
...(rpcCode === null ? {} : { rpc_code: rpcCode }),
|
|
374
434
|
...(typeof threadId === 'string' && threadId.length > 0
|
|
375
435
|
? { session_key: hashText(threadId) }
|
|
376
436
|
: {}),
|
|
@@ -477,87 +537,1461 @@ export function parseBudgetSnapshot(value) {
|
|
|
477
537
|
return value;
|
|
478
538
|
}
|
|
479
539
|
|
|
480
|
-
function budgetSnapshotFresh(snapshot, receivedAt, currentTime) {
|
|
481
|
-
if (!isRecord(snapshot) || !Number.isSafeInteger(receivedAt) || currentTime < receivedAt)
|
|
482
|
-
return false;
|
|
483
|
-
const lifetime = Date.parse(snapshot.expires_at) - Date.parse(snapshot.server_time);
|
|
484
|
-
return (
|
|
485
|
-
lifetime > 0 && lifetime <= MAX_BUDGET_SNAPSHOT_AGE_MS && currentTime - receivedAt <= lifetime
|
|
540
|
+
function budgetSnapshotFresh(snapshot, receivedAt, currentTime) {
|
|
541
|
+
if (!isRecord(snapshot) || !Number.isSafeInteger(receivedAt) || currentTime < receivedAt)
|
|
542
|
+
return false;
|
|
543
|
+
const lifetime = Date.parse(snapshot.expires_at) - Date.parse(snapshot.server_time);
|
|
544
|
+
return (
|
|
545
|
+
lifetime > 0 && lifetime <= MAX_BUDGET_SNAPSHOT_AGE_MS && currentTime - receivedAt <= lifetime
|
|
546
|
+
);
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
function diagnosticReasonForError(error) {
|
|
550
|
+
if (DIAGNOSTIC_REASONS.has(error?.diagnosticReason)) return error.diagnosticReason;
|
|
551
|
+
if (safeRpcCode(error?.rpcCode) !== null) return 'protocol_error';
|
|
552
|
+
const message = error instanceof Error ? error.message : String(error ?? '');
|
|
553
|
+
if (
|
|
554
|
+
/snapshot exceeds bound|capture exceeds|capture size|oversized internal response/iu.test(
|
|
555
|
+
message,
|
|
556
|
+
)
|
|
557
|
+
)
|
|
558
|
+
return 'capture_size_exceeded';
|
|
559
|
+
if (/capture item count|source index exceeds bound/iu.test(message))
|
|
560
|
+
return 'capture_item_count_exceeded';
|
|
561
|
+
if (/capture storage capacity|storage capacity|capture directory/iu.test(message))
|
|
562
|
+
return 'storage_capacity_exceeded';
|
|
563
|
+
if (/capture source identity validation/iu.test(message)) return 'source_changed';
|
|
564
|
+
if (/capture candidate|checkpoint target/iu.test(message)) return 'private_storage_failed';
|
|
565
|
+
if (/thread\/(?:read|turns\/list)|capture history|page unavailable/iu.test(message))
|
|
566
|
+
return 'history_unavailable';
|
|
567
|
+
if (/checkpoint|recovery notes|private (?:file|storage)|storage|EACCES|ENOSPC/iu.test(message))
|
|
568
|
+
return 'private_storage_failed';
|
|
569
|
+
if (/tim(?:e|ed)[ -]?out/iu.test(message)) return 'timeout';
|
|
570
|
+
if (/closed|deleted/iu.test(message)) return 'thread_closed';
|
|
571
|
+
if (/source changed|state changed|stale/iu.test(message)) return 'source_changed';
|
|
572
|
+
if (/candidate|history|thread\/read/iu.test(message)) return 'history_unavailable';
|
|
573
|
+
return 'request_failed';
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
function rpcFailure(kind, error) {
|
|
577
|
+
const failure = new Error(`${kind} request rejected by the app-server`);
|
|
578
|
+
failure.name = 'CodexProtocolError';
|
|
579
|
+
failure.diagnosticReason = 'protocol_error';
|
|
580
|
+
failure.rpcCode = safeRpcCode(error?.code);
|
|
581
|
+
return failure;
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
function boundedFailureCounts(error) {
|
|
585
|
+
if (!isRecord(error)) return undefined;
|
|
586
|
+
const counts = {};
|
|
587
|
+
for (const [source, target] of [
|
|
588
|
+
['captureItemCount', 'capture_item_count'],
|
|
589
|
+
['captureItemLimit', 'capture_item_limit'],
|
|
590
|
+
['captureBytes', 'capture_bytes'],
|
|
591
|
+
['captureLimitBytes', 'capture_limit_bytes'],
|
|
592
|
+
['responseBytes', 'response_bytes'],
|
|
593
|
+
['responseLimitBytes', 'response_limit_bytes'],
|
|
594
|
+
['captureDirectoryBytes', 'capture_directory_bytes'],
|
|
595
|
+
['captureDirectoryLimitBytes', 'capture_directory_limit_bytes'],
|
|
596
|
+
['captureFileCount', 'capture_file_count'],
|
|
597
|
+
['captureFileLimit', 'capture_file_limit'],
|
|
598
|
+
]) {
|
|
599
|
+
const maximum = target.includes('directory_bytes')
|
|
600
|
+
? MAX_CAPTURE_DIRECTORY_BYTES
|
|
601
|
+
: target.includes('bytes')
|
|
602
|
+
? MAX_DIAGNOSTIC_BYTES
|
|
603
|
+
: 1_000_000;
|
|
604
|
+
if (isSafeBoundedInteger(error[source], 0, maximum)) counts[target] = error[source];
|
|
605
|
+
}
|
|
606
|
+
return Object.keys(counts).length > 0 ? counts : undefined;
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
async function ensurePrivateDirectory(directory, fsOps = captureFs) {
|
|
610
|
+
const target = resolve(directory);
|
|
611
|
+
await fsOps.mkdir(target, { recursive: true, mode: 0o700 });
|
|
612
|
+
const parts = target.split('/');
|
|
613
|
+
let current = parts[0] === '' ? '/' : parts[0];
|
|
614
|
+
for (const part of parts.slice(parts[0] === '' ? 1 : 1)) {
|
|
615
|
+
if (!part) continue;
|
|
616
|
+
current = current === '/' ? `/${part}` : `${current}/${part}`;
|
|
617
|
+
const info = await fsOps.lstat(current);
|
|
618
|
+
if (info.isSymbolicLink() || !info.isDirectory())
|
|
619
|
+
throw new Error('checkpoint directory is unsafe');
|
|
620
|
+
}
|
|
621
|
+
await fsOps.chmod(target, 0o700);
|
|
622
|
+
return target;
|
|
623
|
+
}
|
|
624
|
+
async function validatePrivateDirectoryPath(directory, fsOps = captureFs) {
|
|
625
|
+
const target = resolve(directory);
|
|
626
|
+
const parts = target.split('/');
|
|
627
|
+
let current = parts[0] === '' ? '/' : parts[0];
|
|
628
|
+
for (const part of parts.slice(1)) {
|
|
629
|
+
if (!part) continue;
|
|
630
|
+
current = current === '/' ? `/${part}` : `${current}/${part}`;
|
|
631
|
+
const info = await fsOps.lstat(current);
|
|
632
|
+
if (info.isSymbolicLink() || !info.isDirectory())
|
|
633
|
+
throw new Error('checkpoint directory is unsafe');
|
|
634
|
+
if (
|
|
635
|
+
current === target &&
|
|
636
|
+
((typeof process.getuid === 'function' && info.uid !== process.getuid()) ||
|
|
637
|
+
(info.mode & 0o077) !== 0)
|
|
638
|
+
)
|
|
639
|
+
throw new Error('capture directory is not private');
|
|
640
|
+
}
|
|
641
|
+
return target;
|
|
642
|
+
}
|
|
643
|
+
async function assertPrivateFile(file, label, fsOps = captureFs) {
|
|
644
|
+
const info = await fsOps.lstat(file);
|
|
645
|
+
if (
|
|
646
|
+
info.isSymbolicLink() ||
|
|
647
|
+
!info.isFile() ||
|
|
648
|
+
(info.mode & 0o077) !== 0 ||
|
|
649
|
+
(typeof process.getuid === 'function' && info.uid !== process.getuid())
|
|
650
|
+
)
|
|
651
|
+
throw new Error(`${label} is not a private regular file`);
|
|
652
|
+
return info;
|
|
653
|
+
}
|
|
654
|
+
async function readPrivateFile(file, label, maxBytes = null, fsOps = captureFs) {
|
|
655
|
+
const info = await assertPrivateFile(file, label, fsOps);
|
|
656
|
+
if (maxBytes !== null && info.size > maxBytes) throw new Error(`${label} exceeds its bound`);
|
|
657
|
+
return fsOps.readFile(file, 'utf8');
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
function captureCapacityError(state) {
|
|
661
|
+
const error = new Error('capture storage capacity exceeded');
|
|
662
|
+
error.diagnosticReason = 'storage_capacity_exceeded';
|
|
663
|
+
error.captureDirectoryBytes = state?.totalBytes ?? MAX_CAPTURE_DIRECTORY_BYTES;
|
|
664
|
+
error.captureDirectoryLimitBytes = MAX_CAPTURE_DIRECTORY_BYTES;
|
|
665
|
+
error.captureFileCount =
|
|
666
|
+
state?.managedFileCount ?? state?.managedFiles?.length ?? MAX_CAPTURE_FILES;
|
|
667
|
+
error.captureFileLimit = MAX_CAPTURE_FILES;
|
|
668
|
+
return error;
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
function captureStorageCleanupError(state) {
|
|
672
|
+
const error = new Error('capture storage cleanup remains incomplete');
|
|
673
|
+
error.diagnosticReason = 'private_storage_failed';
|
|
674
|
+
error.captureDirectoryBytes = state?.totalBytes ?? MAX_CAPTURE_DIRECTORY_BYTES;
|
|
675
|
+
error.captureDirectoryLimitBytes = MAX_CAPTURE_DIRECTORY_BYTES;
|
|
676
|
+
error.captureFileCount = state?.managedFiles?.length ?? MAX_CAPTURE_FILES;
|
|
677
|
+
error.captureFileLimit = MAX_CAPTURE_FILES;
|
|
678
|
+
return error;
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
function validateCaptureSnapshot(value, expectedSourceRevision = null) {
|
|
682
|
+
if (
|
|
683
|
+
!isRecord(value) ||
|
|
684
|
+
![1, 2].includes(value.version) ||
|
|
685
|
+
typeof value.threadId !== 'string' ||
|
|
686
|
+
typeof value.sourceRevision !== 'string' ||
|
|
687
|
+
!/^[a-f0-9]{64}$/u.test(value.sourceRevision) ||
|
|
688
|
+
!isRecord(value.threadRead) ||
|
|
689
|
+
value.threadRead.thread?.id !== value.threadId ||
|
|
690
|
+
(value.version === 1 && hashJson(value.threadRead) !== value.sourceRevision) ||
|
|
691
|
+
(value.version === 2 &&
|
|
692
|
+
(!isRecord(value.captureManifest) ||
|
|
693
|
+
value.captureManifest.version !== 'codex-capture-manifest-v1' ||
|
|
694
|
+
value.captureManifest.sourceRevision !== value.sourceRevision ||
|
|
695
|
+
value.captureManifest.threadId !== value.threadId ||
|
|
696
|
+
!isSafeBoundedInteger(value.captureManifest.pageCount, 1, MAX_CAPTURE_PAGES) ||
|
|
697
|
+
!isSafeBoundedInteger(value.captureManifest.itemCount, 0, MAX_CAPTURE_ITEMS) ||
|
|
698
|
+
!isSafeBoundedInteger(value.captureManifest.totalBytes, 1, MAX_CAPTURE_TOTAL_BYTES) ||
|
|
699
|
+
!Array.isArray(value.captureManifest.segments) ||
|
|
700
|
+
value.captureManifest.segments.length !== value.captureManifest.pageCount ||
|
|
701
|
+
!Array.isArray(value.captureManifest.pageHashes) ||
|
|
702
|
+
value.captureManifest.pageHashes.length !== value.captureManifest.pageCount ||
|
|
703
|
+
value.captureManifest.pageHashes.some((hash) => !/^[a-f0-9]{64}$/u.test(hash)) ||
|
|
704
|
+
captureManifestRevision(value.threadRead.thread, value.captureManifest.pageHashes) !==
|
|
705
|
+
value.sourceRevision ||
|
|
706
|
+
value.captureManifest.segments.some(
|
|
707
|
+
(segment, index) =>
|
|
708
|
+
segment !== captureSegmentName(value.threadId, value.sourceRevision, index),
|
|
709
|
+
))) ||
|
|
710
|
+
(expectedSourceRevision !== null && value.sourceRevision !== expectedSourceRevision)
|
|
711
|
+
)
|
|
712
|
+
throw new Error('capture source identity validation failed');
|
|
713
|
+
strictThreadName(value.threadId);
|
|
714
|
+
return value;
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
function validateCaptureSegment(value, expectedThreadId = null, expectedSourceRevision = null) {
|
|
718
|
+
if (
|
|
719
|
+
!isRecord(value) ||
|
|
720
|
+
value.version !== 'codex-capture-segment-v1' ||
|
|
721
|
+
typeof value.threadId !== 'string' ||
|
|
722
|
+
(expectedThreadId !== null && value.threadId !== expectedThreadId) ||
|
|
723
|
+
typeof value.sourceRevision !== 'string' ||
|
|
724
|
+
!/^[a-f0-9]{64}$/u.test(value.sourceRevision) ||
|
|
725
|
+
(expectedSourceRevision !== null && value.sourceRevision !== expectedSourceRevision) ||
|
|
726
|
+
!isSafeBoundedInteger(value.pageIndex, 0, MAX_CAPTURE_PAGES - 1) ||
|
|
727
|
+
!isRecord(value.page) ||
|
|
728
|
+
value.page.threadId !== value.threadId ||
|
|
729
|
+
!Array.isArray(value.page.turns) ||
|
|
730
|
+
typeof value.pageHash !== 'string' ||
|
|
731
|
+
!/^[a-f0-9]{64}$/u.test(value.pageHash) ||
|
|
732
|
+
hashJson(value.page) !== value.pageHash ||
|
|
733
|
+
!isSafeBoundedInteger(value.itemCount, 0, MAX_CAPTURE_ITEMS)
|
|
734
|
+
)
|
|
735
|
+
throw new Error('capture segment identity validation failed');
|
|
736
|
+
return value;
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
function captureSegmentName(threadId, sourceRevision, pageIndex) {
|
|
740
|
+
return `${strictThreadName(threadId)}.${sourceRevision}.segment-${String(pageIndex).padStart(4, '0')}.json`;
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
function captureManifestRevision(metadataThread, pageHashes) {
|
|
744
|
+
const thread = Object.fromEntries(
|
|
745
|
+
Object.entries(metadataThread).filter(([key]) => key !== 'turns'),
|
|
746
|
+
);
|
|
747
|
+
return hashJson({ thread, pages: pageHashes });
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
function captureNameForSnapshot(value, suffix = '') {
|
|
751
|
+
return `${strictThreadName(value.threadId)}${suffix}.json`;
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
async function readCaptureDirectoryState(
|
|
755
|
+
directory,
|
|
756
|
+
fsOps = captureFs,
|
|
757
|
+
{ replacedCurrentName = null } = {},
|
|
758
|
+
) {
|
|
759
|
+
const entries = await fsOps.readdir(directory, { withFileTypes: true });
|
|
760
|
+
if (entries.length > MAX_CAPTURE_FILES * 4)
|
|
761
|
+
throw captureCapacityError({ totalBytes: MAX_CAPTURE_DIRECTORY_BYTES, managedFiles: [] });
|
|
762
|
+
let activeSentinelInfo = null;
|
|
763
|
+
try {
|
|
764
|
+
activeSentinelInfo = await fsOps.lstat(resolve(directory, '.capture.lock'));
|
|
765
|
+
} catch (error) {
|
|
766
|
+
if (error?.code !== 'ENOENT') throw error;
|
|
767
|
+
}
|
|
768
|
+
const state = {
|
|
769
|
+
entryCount: entries.length,
|
|
770
|
+
totalBytes: 0,
|
|
771
|
+
managedFiles: [],
|
|
772
|
+
referencedImmutableNames: new Set(),
|
|
773
|
+
protectedGroups: new Set(),
|
|
774
|
+
manifestRefs: [],
|
|
775
|
+
blockedEntries: false,
|
|
776
|
+
unsafeRecovery: false,
|
|
777
|
+
};
|
|
778
|
+
for (const entry of entries) {
|
|
779
|
+
if (entry.name === '.capture.lock') continue;
|
|
780
|
+
const file = resolve(directory, entry.name);
|
|
781
|
+
const info = await fsOps.lstat(file);
|
|
782
|
+
if (
|
|
783
|
+
activeSentinelInfo !== null &&
|
|
784
|
+
Number.isInteger(activeSentinelInfo.dev) &&
|
|
785
|
+
Number.isInteger(activeSentinelInfo.ino) &&
|
|
786
|
+
info.dev === activeSentinelInfo.dev &&
|
|
787
|
+
info.ino === activeSentinelInfo.ino
|
|
788
|
+
)
|
|
789
|
+
continue;
|
|
790
|
+
state.totalBytes += info.size;
|
|
791
|
+
if (
|
|
792
|
+
info.isSymbolicLink() ||
|
|
793
|
+
!info.isFile() ||
|
|
794
|
+
(typeof process.getuid === 'function' && info.uid !== process.getuid()) ||
|
|
795
|
+
(info.mode & 0o077) !== 0
|
|
796
|
+
) {
|
|
797
|
+
state.blockedEntries = true;
|
|
798
|
+
continue;
|
|
799
|
+
}
|
|
800
|
+
if (!entry.name.endsWith('.json')) continue;
|
|
801
|
+
let value;
|
|
802
|
+
try {
|
|
803
|
+
if (info.size > MAX_CAPTURE_BYTES) {
|
|
804
|
+
if (entry.name.endsWith('.recovery.json')) state.unsafeRecovery = true;
|
|
805
|
+
continue;
|
|
806
|
+
}
|
|
807
|
+
value = JSON.parse(await fsOps.readFile(file, 'utf8'));
|
|
808
|
+
} catch {
|
|
809
|
+
if (entry.name.endsWith('.recovery.json')) state.unsafeRecovery = true;
|
|
810
|
+
continue;
|
|
811
|
+
}
|
|
812
|
+
if (entry.name.endsWith('.recovery.json')) {
|
|
813
|
+
if (
|
|
814
|
+
isRecord(value) &&
|
|
815
|
+
typeof value.threadId === 'string' &&
|
|
816
|
+
/^[a-f0-9]{64}$/u.test(value.sourceRevision) &&
|
|
817
|
+
typeof value.notes === 'string' &&
|
|
818
|
+
typeof value.notesHash === 'string' &&
|
|
819
|
+
value.notesHash === hashText(value.notes)
|
|
820
|
+
) {
|
|
821
|
+
try {
|
|
822
|
+
state.referencedImmutableNames.add(
|
|
823
|
+
`${strictThreadName(value.threadId)}.${value.sourceRevision}.json`,
|
|
824
|
+
);
|
|
825
|
+
state.protectedGroups.add(`${strictThreadName(value.threadId)}.${value.sourceRevision}`);
|
|
826
|
+
} catch {
|
|
827
|
+
state.unsafeRecovery = true;
|
|
828
|
+
}
|
|
829
|
+
} else state.unsafeRecovery = true;
|
|
830
|
+
if (isRecord(value) && typeof value.notes === 'string') {
|
|
831
|
+
const references = [
|
|
832
|
+
...value.notes.matchAll(/capture_file:\s+.*?([A-Za-z0-9._-]+\.[a-f0-9]{64}\.json)/gu),
|
|
833
|
+
];
|
|
834
|
+
for (const match of references) state.referencedImmutableNames.add(match[1]);
|
|
835
|
+
}
|
|
836
|
+
continue;
|
|
837
|
+
}
|
|
838
|
+
if (isRecord(value) && value.version === 'codex-capture-segment-v1') {
|
|
839
|
+
try {
|
|
840
|
+
validateCaptureSegment(value);
|
|
841
|
+
const expectedName = captureSegmentName(
|
|
842
|
+
value.threadId,
|
|
843
|
+
value.sourceRevision,
|
|
844
|
+
value.pageIndex,
|
|
845
|
+
);
|
|
846
|
+
if (entry.name !== expectedName) continue;
|
|
847
|
+
state.managedFiles.push({
|
|
848
|
+
file,
|
|
849
|
+
name: entry.name,
|
|
850
|
+
size: info.size,
|
|
851
|
+
mtimeMs: info.mtimeMs,
|
|
852
|
+
immutable: true,
|
|
853
|
+
group: `${value.threadId}.${value.sourceRevision}`,
|
|
854
|
+
});
|
|
855
|
+
} catch {
|
|
856
|
+
state.blockedEntries = true;
|
|
857
|
+
}
|
|
858
|
+
continue;
|
|
859
|
+
}
|
|
860
|
+
try {
|
|
861
|
+
validateCaptureSnapshot(value);
|
|
862
|
+
} catch {
|
|
863
|
+
continue;
|
|
864
|
+
}
|
|
865
|
+
const sourceName = `${strictThreadName(value.threadId)}.${value.sourceRevision}.json`;
|
|
866
|
+
if (
|
|
867
|
+
!new Set([
|
|
868
|
+
captureNameForSnapshot(value),
|
|
869
|
+
captureNameForSnapshot(value, '.completed'),
|
|
870
|
+
sourceName,
|
|
871
|
+
]).has(entry.name)
|
|
872
|
+
)
|
|
873
|
+
continue;
|
|
874
|
+
state.managedFiles.push({
|
|
875
|
+
file,
|
|
876
|
+
name: entry.name,
|
|
877
|
+
size: info.size,
|
|
878
|
+
mtimeMs: info.mtimeMs,
|
|
879
|
+
immutable: entry.name === sourceName,
|
|
880
|
+
group: `${value.threadId}.${value.sourceRevision}`,
|
|
881
|
+
});
|
|
882
|
+
const replacedCurrent = entry.name === replacedCurrentName;
|
|
883
|
+
if (!replacedCurrent && !entry.name.endsWith('.' + value.sourceRevision + '.json'))
|
|
884
|
+
state.referencedImmutableNames.add(sourceName);
|
|
885
|
+
if (value.version === 2 && entry.name !== sourceName && !replacedCurrent) {
|
|
886
|
+
for (const segment of value.captureManifest.segments) {
|
|
887
|
+
if (typeof segment === 'string') state.referencedImmutableNames.add(segment);
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
if (value.version === 2) state.manifestRefs.push(value.captureManifest);
|
|
891
|
+
if (
|
|
892
|
+
value.version === 2 &&
|
|
893
|
+
state.protectedGroups.has(`${value.threadId}.${value.sourceRevision}`)
|
|
894
|
+
) {
|
|
895
|
+
for (const segment of value.captureManifest.segments)
|
|
896
|
+
state.referencedImmutableNames.add(segment);
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
const managedNames = new Set(state.managedFiles.map((entry) => entry.name));
|
|
900
|
+
for (const manifest of state.manifestRefs) {
|
|
901
|
+
if (manifest.segments.some((segment) => !managedNames.has(segment)))
|
|
902
|
+
state.blockedEntries = true;
|
|
903
|
+
}
|
|
904
|
+
for (const entry of state.managedFiles) {
|
|
905
|
+
if (entry.group && state.referencedImmutableNames.has(entry.name))
|
|
906
|
+
state.protectedGroups.add(entry.group);
|
|
907
|
+
}
|
|
908
|
+
return state;
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
function captureLockError(reason) {
|
|
912
|
+
const messages = {
|
|
913
|
+
capture_lock_timeout: 'capture lock is held by a live or unverified owner',
|
|
914
|
+
capture_lock_legacy: 'capture lock uses an unidentifiable legacy sentinel',
|
|
915
|
+
capture_lock_owner_unverifiable: 'capture lock owner identity is unavailable',
|
|
916
|
+
capture_lock_backend_unavailable: 'capture lock backend is unavailable',
|
|
917
|
+
};
|
|
918
|
+
const error = new Error(messages[reason] ?? 'capture lock unavailable');
|
|
919
|
+
error.diagnosticReason = reason;
|
|
920
|
+
return error;
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
const CAPTURE_LOCK_MARKER = `${JSON.stringify({ backend: 'flock', schema: 2 })}\n`;
|
|
924
|
+
const CAPTURE_GUARD_NAME = '.capture.guard';
|
|
925
|
+
const CAPTURE_PAGE_STAGE_NAME =
|
|
926
|
+
/^\.capture-(?:segment-)?stage\.[A-Za-z0-9._-]{1,128}\.[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\.(?:[0-9]|[12][0-9]|3[01])\.tmp$/iu;
|
|
927
|
+
const CAPTURE_SOURCE_STAGE_NAME =
|
|
928
|
+
/^\.capture-stage\.[A-Za-z0-9._-]{1,128}\.[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\.(?:[0-9]|[12][0-9]|3[01])\.tmp$/iu;
|
|
929
|
+
|
|
930
|
+
async function removePrivateCaptureTemps(directory, fsOps, pattern) {
|
|
931
|
+
const entries = await fsOps.readdir(directory, { withFileTypes: true });
|
|
932
|
+
let removedFiles = 0;
|
|
933
|
+
let removedBytes = 0;
|
|
934
|
+
for (const entry of entries) {
|
|
935
|
+
if (!pattern.test(entry.name)) continue;
|
|
936
|
+
const file = resolve(directory, entry.name);
|
|
937
|
+
let info;
|
|
938
|
+
try {
|
|
939
|
+
info = await fsOps.lstat(file);
|
|
940
|
+
} catch (error) {
|
|
941
|
+
if (error?.code === 'ENOENT') continue;
|
|
942
|
+
throw error;
|
|
943
|
+
}
|
|
944
|
+
if (
|
|
945
|
+
info.isSymbolicLink() ||
|
|
946
|
+
!info.isFile() ||
|
|
947
|
+
(typeof process.getuid === 'function' && info.uid !== process.getuid()) ||
|
|
948
|
+
(info.mode & 0o077) !== 0
|
|
949
|
+
)
|
|
950
|
+
continue;
|
|
951
|
+
try {
|
|
952
|
+
await fsOps.unlink(file);
|
|
953
|
+
removedFiles += 1;
|
|
954
|
+
removedBytes += info.size;
|
|
955
|
+
} catch (error) {
|
|
956
|
+
if (error?.code !== 'ENOENT') throw error;
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
return { removedFiles, removedBytes };
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
async function removeAbandonedCapturePageStages(directory, fsOps) {
|
|
963
|
+
return removePrivateCaptureTemps(directory, fsOps, CAPTURE_PAGE_STAGE_NAME);
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
async function observerKernelIdentity() {
|
|
967
|
+
try {
|
|
968
|
+
return {
|
|
969
|
+
bootId: (await readFile('/proc/sys/kernel/random/boot_id', 'utf8')).trim(),
|
|
970
|
+
pidNamespace: await readlink('/proc/self/ns/pid'),
|
|
971
|
+
};
|
|
972
|
+
} catch {
|
|
973
|
+
return null;
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
|
|
977
|
+
async function processIdentity(pid) {
|
|
978
|
+
if (!Number.isSafeInteger(pid) || pid <= 0) return { status: 'invalid' };
|
|
979
|
+
const observer = await observerKernelIdentity();
|
|
980
|
+
if (observer === null) return { status: 'unverifiable' };
|
|
981
|
+
try {
|
|
982
|
+
const ownerNamespace = await readlink(`/proc/${pid}/ns/pid`);
|
|
983
|
+
if (ownerNamespace !== observer.pidNamespace || observer.bootId.length === 0)
|
|
984
|
+
return { status: 'foreign' };
|
|
985
|
+
const stat = await readFile(`/proc/${pid}/stat`, 'utf8');
|
|
986
|
+
const closing = stat.lastIndexOf(')');
|
|
987
|
+
const fields =
|
|
988
|
+
closing < 0
|
|
989
|
+
? []
|
|
990
|
+
: stat
|
|
991
|
+
.slice(closing + 2)
|
|
992
|
+
.trim()
|
|
993
|
+
.split(/\s+/u);
|
|
994
|
+
const startTime = fields[19];
|
|
995
|
+
if (typeof startTime !== 'string' || !/^[0-9]+$/u.test(startTime))
|
|
996
|
+
return { status: 'unverifiable' };
|
|
997
|
+
return { status: 'live', startTime, bootId: observer.bootId, pidNamespace: ownerNamespace };
|
|
998
|
+
} catch (error) {
|
|
999
|
+
if (error?.code === 'ENOENT') return { status: 'dead' };
|
|
1000
|
+
return { status: 'unverifiable' };
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
1003
|
+
|
|
1004
|
+
async function captureSentinelOwner(lockPath, fsOps) {
|
|
1005
|
+
let info;
|
|
1006
|
+
try {
|
|
1007
|
+
info = await fsOps.lstat(lockPath);
|
|
1008
|
+
} catch (error) {
|
|
1009
|
+
if (error?.code === 'ENOENT') return { status: 'absent' };
|
|
1010
|
+
throw error;
|
|
1011
|
+
}
|
|
1012
|
+
if (info.isSymbolicLink() || !info.isFile() || (info.mode & 0o077) !== 0)
|
|
1013
|
+
throw new Error('capture lock is not a private regular file');
|
|
1014
|
+
if (info.size === 0) return { status: 'legacy' };
|
|
1015
|
+
if (info.size > 4096) return { status: 'unverifiable' };
|
|
1016
|
+
let value;
|
|
1017
|
+
try {
|
|
1018
|
+
value = JSON.parse(await fsOps.readFile(lockPath, 'utf8'));
|
|
1019
|
+
} catch {
|
|
1020
|
+
return { status: 'unverifiable' };
|
|
1021
|
+
}
|
|
1022
|
+
if (
|
|
1023
|
+
!isRecord(value) ||
|
|
1024
|
+
!hasOnlyKeys(value, [
|
|
1025
|
+
'schema',
|
|
1026
|
+
'backend',
|
|
1027
|
+
'pid',
|
|
1028
|
+
'startTime',
|
|
1029
|
+
'bootId',
|
|
1030
|
+
'pidNamespace',
|
|
1031
|
+
'nonce',
|
|
1032
|
+
]) ||
|
|
1033
|
+
value.schema !== 1 ||
|
|
1034
|
+
value.backend !== 'flock' ||
|
|
1035
|
+
!isSafeBoundedInteger(value.pid, 1, 4_000_000_000) ||
|
|
1036
|
+
typeof value.startTime !== 'string' ||
|
|
1037
|
+
!/^[0-9]+$/u.test(value.startTime) ||
|
|
1038
|
+
typeof value.bootId !== 'string' ||
|
|
1039
|
+
!/^[a-f0-9-]{36}$/iu.test(value.bootId) ||
|
|
1040
|
+
typeof value.pidNamespace !== 'string' ||
|
|
1041
|
+
!/^pid:\[\d+\]$/u.test(value.pidNamespace) ||
|
|
1042
|
+
typeof value.nonce !== 'string' ||
|
|
1043
|
+
!isUuid(value.nonce)
|
|
1044
|
+
)
|
|
1045
|
+
return { status: 'unverifiable' };
|
|
1046
|
+
const observer = await observerKernelIdentity();
|
|
1047
|
+
if (observer === null) return { status: 'unverifiable' };
|
|
1048
|
+
if (observer.bootId !== value.bootId) return { status: 'dead' };
|
|
1049
|
+
if (observer.pidNamespace !== value.pidNamespace) return { status: 'foreign' };
|
|
1050
|
+
const current = await processIdentity(value.pid);
|
|
1051
|
+
if (current.status !== 'live') return { status: current.status };
|
|
1052
|
+
if (current.bootId !== value.bootId || current.pidNamespace !== value.pidNamespace)
|
|
1053
|
+
return { status: 'foreign' };
|
|
1054
|
+
return current.startTime === value.startTime ? { status: 'live' } : { status: 'dead' };
|
|
1055
|
+
}
|
|
1056
|
+
|
|
1057
|
+
async function ensureCaptureSentinel(lockPath, task) {
|
|
1058
|
+
const identity = await processIdentity(process.pid);
|
|
1059
|
+
if (identity.status !== 'live') throw captureLockError('capture_lock_owner_unverifiable');
|
|
1060
|
+
const metadata = `${JSON.stringify({
|
|
1061
|
+
schema: 1,
|
|
1062
|
+
backend: 'flock',
|
|
1063
|
+
pid: process.pid,
|
|
1064
|
+
startTime: identity.startTime,
|
|
1065
|
+
bootId: identity.bootId,
|
|
1066
|
+
pidNamespace: identity.pidNamespace,
|
|
1067
|
+
nonce: randomUUID(),
|
|
1068
|
+
})}\n`;
|
|
1069
|
+
while (true) {
|
|
1070
|
+
const temporary = `${lockPath}.${process.pid}.${randomUUID()}.tmp`;
|
|
1071
|
+
let contention = false;
|
|
1072
|
+
let published = false;
|
|
1073
|
+
try {
|
|
1074
|
+
let handle;
|
|
1075
|
+
try {
|
|
1076
|
+
const flags =
|
|
1077
|
+
fsConstants.O_WRONLY |
|
|
1078
|
+
fsConstants.O_CREAT |
|
|
1079
|
+
fsConstants.O_EXCL |
|
|
1080
|
+
(fsConstants.O_NOFOLLOW ?? 0);
|
|
1081
|
+
handle = await captureFs.open(temporary, flags, 0o600);
|
|
1082
|
+
await handle.writeFile(metadata, 'utf8');
|
|
1083
|
+
await handle.sync?.();
|
|
1084
|
+
} finally {
|
|
1085
|
+
await handle?.close().catch(() => undefined);
|
|
1086
|
+
}
|
|
1087
|
+
await captureFs.hardlink(temporary, lockPath);
|
|
1088
|
+
published = true;
|
|
1089
|
+
const publishedInfo = await captureFs.lstat(lockPath);
|
|
1090
|
+
try {
|
|
1091
|
+
return await task();
|
|
1092
|
+
} finally {
|
|
1093
|
+
const current = await captureFs.lstat(lockPath).catch(() => null);
|
|
1094
|
+
if (
|
|
1095
|
+
current !== null &&
|
|
1096
|
+
current.dev === publishedInfo.dev &&
|
|
1097
|
+
current.ino === publishedInfo.ino
|
|
1098
|
+
)
|
|
1099
|
+
await captureFs.unlink(lockPath).catch(() => undefined);
|
|
1100
|
+
}
|
|
1101
|
+
} catch (error) {
|
|
1102
|
+
if (!published && error?.code === 'EEXIST') contention = true;
|
|
1103
|
+
else throw error;
|
|
1104
|
+
} finally {
|
|
1105
|
+
await captureFs.unlink(temporary).catch(() => undefined);
|
|
1106
|
+
}
|
|
1107
|
+
if (!contention) throw captureLockError('capture_lock_owner_unverifiable');
|
|
1108
|
+
const owner = await captureSentinelOwner(lockPath, captureFs);
|
|
1109
|
+
if (owner.status === 'dead') {
|
|
1110
|
+
await captureFs.unlink(lockPath).catch((unlinkError) => {
|
|
1111
|
+
if (unlinkError?.code !== 'ENOENT') throw unlinkError;
|
|
1112
|
+
});
|
|
1113
|
+
continue;
|
|
1114
|
+
}
|
|
1115
|
+
if (owner.status === 'legacy') throw captureLockError('capture_lock_legacy');
|
|
1116
|
+
throw captureLockError('capture_lock_owner_unverifiable');
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
async function withAdvisoryCaptureLock(directory, task) {
|
|
1121
|
+
const guardPath = resolve(directory, CAPTURE_GUARD_NAME);
|
|
1122
|
+
const lockPath = resolve(directory, '.capture.lock');
|
|
1123
|
+
let guardHandle;
|
|
1124
|
+
try {
|
|
1125
|
+
const flags = fsConstants.O_RDWR | fsConstants.O_CREAT | (fsConstants.O_NOFOLLOW ?? 0);
|
|
1126
|
+
guardHandle = await captureFs.open(guardPath, flags, 0o600);
|
|
1127
|
+
const info = await guardHandle.stat();
|
|
1128
|
+
if (
|
|
1129
|
+
!info.isFile() ||
|
|
1130
|
+
info.isSymbolicLink?.() ||
|
|
1131
|
+
info.nlink !== 1 ||
|
|
1132
|
+
(typeof process.getuid === 'function' && info.uid !== process.getuid()) ||
|
|
1133
|
+
(info.mode & 0o077) !== 0 ||
|
|
1134
|
+
info.size > 4096
|
|
1135
|
+
)
|
|
1136
|
+
throw captureLockError('capture_lock_owner_unverifiable');
|
|
1137
|
+
} catch (error) {
|
|
1138
|
+
await guardHandle?.close().catch(() => undefined);
|
|
1139
|
+
if (error?.code === 'ENOENT') throw captureLockError('capture_lock_backend_unavailable');
|
|
1140
|
+
throw error;
|
|
1141
|
+
}
|
|
1142
|
+
const started = Date.now();
|
|
1143
|
+
while (true) {
|
|
1144
|
+
let helper;
|
|
1145
|
+
try {
|
|
1146
|
+
helper = spawn('flock', ['-x', '-n', '3'], {
|
|
1147
|
+
stdio: ['ignore', 'ignore', 'ignore', guardHandle.fd],
|
|
1148
|
+
});
|
|
1149
|
+
const acquired = await new Promise((resolvePromise, rejectPromise) => {
|
|
1150
|
+
const timer = setTimeout(() => {
|
|
1151
|
+
helper.kill('SIGKILL');
|
|
1152
|
+
rejectPromise(captureLockError('capture_lock_backend_unavailable'));
|
|
1153
|
+
}, 2_000);
|
|
1154
|
+
helper.once('error', (error) => {
|
|
1155
|
+
clearTimeout(timer);
|
|
1156
|
+
if (error?.code === 'ENOENT')
|
|
1157
|
+
rejectPromise(captureLockError('capture_lock_backend_unavailable'));
|
|
1158
|
+
else rejectPromise(error);
|
|
1159
|
+
});
|
|
1160
|
+
helper.once('exit', (code) => {
|
|
1161
|
+
clearTimeout(timer);
|
|
1162
|
+
if (code === 0) resolvePromise(true);
|
|
1163
|
+
else if (code === 1) resolvePromise(false);
|
|
1164
|
+
else rejectPromise(captureLockError('capture_lock_backend_unavailable'));
|
|
1165
|
+
});
|
|
1166
|
+
});
|
|
1167
|
+
if (acquired) {
|
|
1168
|
+
let marker = await guardHandle.readFile('utf8');
|
|
1169
|
+
if (marker.length === 0 || marker !== CAPTURE_LOCK_MARKER) {
|
|
1170
|
+
await guardHandle.truncate(0);
|
|
1171
|
+
await guardHandle.write(CAPTURE_LOCK_MARKER, 0, 'utf8');
|
|
1172
|
+
await guardHandle.sync?.();
|
|
1173
|
+
}
|
|
1174
|
+
try {
|
|
1175
|
+
return await ensureCaptureSentinel(lockPath, task);
|
|
1176
|
+
} finally {
|
|
1177
|
+
await guardHandle.close().catch(() => undefined);
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
1180
|
+
} catch (error) {
|
|
1181
|
+
if (helper && helper.exitCode === null) helper.kill('SIGTERM');
|
|
1182
|
+
await guardHandle.close().catch(() => undefined);
|
|
1183
|
+
throw error;
|
|
1184
|
+
}
|
|
1185
|
+
if (helper && helper.exitCode === null) helper.kill('SIGTERM');
|
|
1186
|
+
if (Date.now() - started >= CAPTURE_LOCK_WAIT_MS) {
|
|
1187
|
+
await guardHandle.close().catch(() => undefined);
|
|
1188
|
+
throw captureLockError('capture_lock_timeout');
|
|
1189
|
+
}
|
|
1190
|
+
await new Promise((resolvePromise) => setTimeout(resolvePromise, 10));
|
|
1191
|
+
}
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
async function withInjectedCaptureLock(directory, task, fsOps, { now, sleep }) {
|
|
1195
|
+
const lockPath = resolve(directory, '.capture.lock');
|
|
1196
|
+
const started = now();
|
|
1197
|
+
while (true) {
|
|
1198
|
+
let handle;
|
|
1199
|
+
try {
|
|
1200
|
+
handle = await fsOps.open(lockPath, 'wx', 0o600);
|
|
1201
|
+
try {
|
|
1202
|
+
return await task();
|
|
1203
|
+
} finally {
|
|
1204
|
+
await handle.close();
|
|
1205
|
+
await fsOps.unlink(lockPath).catch(() => undefined);
|
|
1206
|
+
}
|
|
1207
|
+
} catch (error) {
|
|
1208
|
+
if (handle) throw error;
|
|
1209
|
+
if (error?.code !== 'EEXIST') throw error;
|
|
1210
|
+
if (now() - started >= CAPTURE_LOCK_WAIT_MS)
|
|
1211
|
+
throw captureLockError('capture_lock_owner_unverifiable');
|
|
1212
|
+
await sleep(10);
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
}
|
|
1216
|
+
|
|
1217
|
+
async function withCaptureStoreLock(
|
|
1218
|
+
directory,
|
|
1219
|
+
task,
|
|
1220
|
+
fsOps = captureFs,
|
|
1221
|
+
{
|
|
1222
|
+
now = Date.now,
|
|
1223
|
+
sleep = (milliseconds) =>
|
|
1224
|
+
new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds)),
|
|
1225
|
+
} = {},
|
|
1226
|
+
) {
|
|
1227
|
+
const nativeFs =
|
|
1228
|
+
fsOps === captureFs ||
|
|
1229
|
+
(fsOps.open === open && fsOps.lstat === lstat && fsOps.writeFile === writeFile);
|
|
1230
|
+
const taskWithStageCleanup = async () => {
|
|
1231
|
+
const reclaimed = await removeAbandonedCapturePageStages(directory, fsOps);
|
|
1232
|
+
return task(reclaimed);
|
|
1233
|
+
};
|
|
1234
|
+
if (nativeFs) return withAdvisoryCaptureLock(directory, taskWithStageCleanup);
|
|
1235
|
+
return withInjectedCaptureLock(directory, taskWithStageCleanup, fsOps, { now, sleep });
|
|
1236
|
+
}
|
|
1237
|
+
|
|
1238
|
+
async function writePrivateFile(file, serialized, fsOps = captureFs) {
|
|
1239
|
+
const directory = await ensurePrivateDirectory(dirname(file), fsOps);
|
|
1240
|
+
const target = resolve(
|
|
1241
|
+
directory,
|
|
1242
|
+
file.startsWith(directory) ? file.slice(directory.length + 1) : file,
|
|
1243
|
+
);
|
|
1244
|
+
try {
|
|
1245
|
+
await assertPrivateFile(target, 'checkpoint target', fsOps);
|
|
1246
|
+
} catch (error) {
|
|
1247
|
+
if (error?.code !== 'ENOENT') throw error;
|
|
1248
|
+
}
|
|
1249
|
+
const temporary = `${target}.${process.pid}.${randomUUID()}.tmp`;
|
|
1250
|
+
try {
|
|
1251
|
+
await fsOps.writeFile(temporary, serialized, { flag: 'wx', mode: 0o600 });
|
|
1252
|
+
await fsOps.rename(temporary, target);
|
|
1253
|
+
} catch (error) {
|
|
1254
|
+
await fsOps.unlink(temporary).catch(() => undefined);
|
|
1255
|
+
throw error;
|
|
1256
|
+
}
|
|
1257
|
+
}
|
|
1258
|
+
|
|
1259
|
+
async function writeBoundedCapture(file, serialized, value, fsOps = captureFs, lockOptions = {}) {
|
|
1260
|
+
const directory = await ensurePrivateDirectory(dirname(file), fsOps);
|
|
1261
|
+
const target = resolve(
|
|
1262
|
+
directory,
|
|
1263
|
+
file.startsWith(directory) ? file.slice(directory.length + 1) : file,
|
|
1264
|
+
);
|
|
1265
|
+
const targetName = target.slice(directory.length + 1);
|
|
1266
|
+
const pairedSourceTarget =
|
|
1267
|
+
lockOptions.pairedSourcePath === undefined
|
|
1268
|
+
? null
|
|
1269
|
+
: resolve(
|
|
1270
|
+
directory,
|
|
1271
|
+
lockOptions.pairedSourcePath.startsWith(directory)
|
|
1272
|
+
? lockOptions.pairedSourcePath.slice(directory.length + 1)
|
|
1273
|
+
: lockOptions.pairedSourcePath,
|
|
1274
|
+
);
|
|
1275
|
+
const pairedSourceName = pairedSourceTarget?.slice(directory.length + 1) ?? null;
|
|
1276
|
+
const sourceName = `${strictThreadName(value.threadId)}.${value.sourceRevision}.json`;
|
|
1277
|
+
if (
|
|
1278
|
+
pairedSourceTarget !== null &&
|
|
1279
|
+
(dirname(pairedSourceTarget) !== directory ||
|
|
1280
|
+
pairedSourceName !== sourceName ||
|
|
1281
|
+
targetName !== captureNameForSnapshot(value))
|
|
1282
|
+
)
|
|
1283
|
+
throw new Error('capture publication target identity validation failed');
|
|
1284
|
+
const operation = async () => {
|
|
1285
|
+
const temporary = `${target}.${process.pid}.${randomUUID()}.tmp`;
|
|
1286
|
+
const sourceTemporary =
|
|
1287
|
+
pairedSourceTarget === null
|
|
1288
|
+
? null
|
|
1289
|
+
: `${pairedSourceTarget}.${process.pid}.${randomUUID()}.tmp`;
|
|
1290
|
+
const expectedNames = new Set([
|
|
1291
|
+
captureNameForSnapshot(value),
|
|
1292
|
+
captureNameForSnapshot(value, '.completed'),
|
|
1293
|
+
sourceName,
|
|
1294
|
+
]);
|
|
1295
|
+
let existingTarget = null;
|
|
1296
|
+
try {
|
|
1297
|
+
existingTarget = await assertPrivateFile(target, 'capture target', fsOps);
|
|
1298
|
+
const existing = JSON.parse(
|
|
1299
|
+
await readPrivateFile(target, 'capture target', MAX_CAPTURE_BYTES, fsOps),
|
|
1300
|
+
);
|
|
1301
|
+
validateCaptureSnapshot(existing);
|
|
1302
|
+
if (!expectedNames.has(targetName) || existing.threadId !== value.threadId)
|
|
1303
|
+
throw new Error('capture target identity validation failed');
|
|
1304
|
+
if (targetName === sourceName && existing.sourceRevision === value.sourceRevision) return;
|
|
1305
|
+
} catch (error) {
|
|
1306
|
+
if (error?.code !== 'ENOENT') throw error;
|
|
1307
|
+
}
|
|
1308
|
+
let existingSource = false;
|
|
1309
|
+
if (pairedSourceTarget !== null) {
|
|
1310
|
+
try {
|
|
1311
|
+
await assertPrivateFile(pairedSourceTarget, 'immutable capture source', fsOps);
|
|
1312
|
+
const immutable = JSON.parse(
|
|
1313
|
+
await readPrivateFile(
|
|
1314
|
+
pairedSourceTarget,
|
|
1315
|
+
'immutable capture source',
|
|
1316
|
+
MAX_CAPTURE_BYTES,
|
|
1317
|
+
fsOps,
|
|
1318
|
+
),
|
|
1319
|
+
);
|
|
1320
|
+
validateCaptureSnapshot(immutable, value.sourceRevision);
|
|
1321
|
+
if (immutable.threadId !== value.threadId)
|
|
1322
|
+
throw new Error('immutable capture source identity validation failed');
|
|
1323
|
+
existingSource = true;
|
|
1324
|
+
} catch (error) {
|
|
1325
|
+
if (error?.code !== 'ENOENT') throw error;
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1328
|
+
try {
|
|
1329
|
+
if (sourceTemporary !== null && !existingSource) {
|
|
1330
|
+
await fsOps.writeFile(sourceTemporary, serialized, { flag: 'wx', mode: 0o600 });
|
|
1331
|
+
const writtenSource = await assertPrivateFile(
|
|
1332
|
+
sourceTemporary,
|
|
1333
|
+
'immutable capture candidate',
|
|
1334
|
+
fsOps,
|
|
1335
|
+
);
|
|
1336
|
+
if (writtenSource.size !== Buffer.byteLength(serialized, 'utf8'))
|
|
1337
|
+
throw new Error('immutable capture candidate read-back size mismatch');
|
|
1338
|
+
validateCaptureSnapshot(
|
|
1339
|
+
JSON.parse(await fsOps.readFile(sourceTemporary, 'utf8')),
|
|
1340
|
+
value.sourceRevision,
|
|
1341
|
+
);
|
|
1342
|
+
}
|
|
1343
|
+
await fsOps.writeFile(temporary, serialized, { flag: 'wx', mode: 0o600 });
|
|
1344
|
+
const written = await assertPrivateFile(temporary, 'capture candidate', fsOps);
|
|
1345
|
+
if (written.size !== Buffer.byteLength(serialized, 'utf8'))
|
|
1346
|
+
throw new Error('capture candidate read-back size mismatch');
|
|
1347
|
+
validateCaptureSnapshot(
|
|
1348
|
+
JSON.parse(await fsOps.readFile(temporary, 'utf8')),
|
|
1349
|
+
value.sourceRevision,
|
|
1350
|
+
);
|
|
1351
|
+
const state = await readCaptureDirectoryState(directory, fsOps, {
|
|
1352
|
+
...(pairedSourceTarget === null ? {} : { replacedCurrentName: targetName }),
|
|
1353
|
+
});
|
|
1354
|
+
if (state.unsafeRecovery || state.blockedEntries) throw captureCapacityError(state);
|
|
1355
|
+
const protectedNames = new Set([targetName, sourceName]);
|
|
1356
|
+
if (value.version === 2 && Array.isArray(value.captureManifest?.segments)) {
|
|
1357
|
+
for (const segment of value.captureManifest.segments) protectedNames.add(segment);
|
|
1358
|
+
}
|
|
1359
|
+
const targetWasManaged = state.managedFiles.some((entry) => entry.name === targetName);
|
|
1360
|
+
const eligibleGroups = new Map();
|
|
1361
|
+
if (!state.blockedEntries) {
|
|
1362
|
+
for (const entry of state.managedFiles) {
|
|
1363
|
+
if (
|
|
1364
|
+
!entry.immutable ||
|
|
1365
|
+
protectedNames.has(entry.name) ||
|
|
1366
|
+
state.referencedImmutableNames.has(entry.name) ||
|
|
1367
|
+
state.protectedGroups.has(entry.group)
|
|
1368
|
+
)
|
|
1369
|
+
continue;
|
|
1370
|
+
const group = entry.group ?? entry.name;
|
|
1371
|
+
const members = eligibleGroups.get(group) ?? [];
|
|
1372
|
+
members.push(entry);
|
|
1373
|
+
eligibleGroups.set(group, members);
|
|
1374
|
+
}
|
|
1375
|
+
}
|
|
1376
|
+
const eligible = [...eligibleGroups.values()].sort(
|
|
1377
|
+
(left, right) =>
|
|
1378
|
+
Math.min(...left.map((entry) => entry.mtimeMs)) -
|
|
1379
|
+
Math.min(...right.map((entry) => entry.mtimeMs)),
|
|
1380
|
+
);
|
|
1381
|
+
const targetEntry = state.managedFiles.find((entry) => entry.name === targetName);
|
|
1382
|
+
const replacedBytes = targetEntry?.size ?? 0;
|
|
1383
|
+
const temporaryHeadroom = 2 * MAX_CAPTURE_BYTES;
|
|
1384
|
+
let projectedPeakBytes = state.totalBytes;
|
|
1385
|
+
let projectedSteadyBytes = state.totalBytes - replacedBytes;
|
|
1386
|
+
let projectedFiles =
|
|
1387
|
+
state.managedFiles.length +
|
|
1388
|
+
(sourceTemporary !== null && !existingSource ? 1 : 0) +
|
|
1389
|
+
(targetWasManaged ? 0 : 1);
|
|
1390
|
+
const retire = [];
|
|
1391
|
+
for (const group of eligible) {
|
|
1392
|
+
if (
|
|
1393
|
+
projectedPeakBytes <= MAX_CAPTURE_DIRECTORY_BYTES + temporaryHeadroom &&
|
|
1394
|
+
projectedSteadyBytes <= MAX_CAPTURE_DIRECTORY_BYTES &&
|
|
1395
|
+
projectedFiles <= MAX_CAPTURE_FILES
|
|
1396
|
+
)
|
|
1397
|
+
break;
|
|
1398
|
+
const groupBytes = group.reduce((sum, entry) => sum + entry.size, 0);
|
|
1399
|
+
projectedPeakBytes -= groupBytes;
|
|
1400
|
+
projectedSteadyBytes -= groupBytes;
|
|
1401
|
+
projectedFiles -= group.length;
|
|
1402
|
+
retire.push(...group);
|
|
1403
|
+
}
|
|
1404
|
+
if (
|
|
1405
|
+
projectedPeakBytes > MAX_CAPTURE_DIRECTORY_BYTES + temporaryHeadroom ||
|
|
1406
|
+
projectedSteadyBytes > MAX_CAPTURE_DIRECTORY_BYTES ||
|
|
1407
|
+
projectedFiles > MAX_CAPTURE_FILES
|
|
1408
|
+
)
|
|
1409
|
+
throw captureCapacityError({
|
|
1410
|
+
...state,
|
|
1411
|
+
totalBytes: projectedSteadyBytes,
|
|
1412
|
+
managedFileCount: projectedFiles,
|
|
1413
|
+
});
|
|
1414
|
+
const backup = existingTarget
|
|
1415
|
+
? `${target}.${process.pid}.${randomUUID()}.capture-backup`
|
|
1416
|
+
: null;
|
|
1417
|
+
const retiredBytes = retire.reduce((total, entry) => total + entry.size, 0);
|
|
1418
|
+
const staged = [];
|
|
1419
|
+
let backupCreated = false;
|
|
1420
|
+
let candidatePublished = false;
|
|
1421
|
+
let sourcePublished = false;
|
|
1422
|
+
let committed = false;
|
|
1423
|
+
try {
|
|
1424
|
+
if (sourceTemporary !== null && !existingSource) {
|
|
1425
|
+
await fsOps.rename(sourceTemporary, pairedSourceTarget);
|
|
1426
|
+
sourcePublished = true;
|
|
1427
|
+
}
|
|
1428
|
+
if (backup !== null) {
|
|
1429
|
+
await fsOps.rename(target, backup);
|
|
1430
|
+
backupCreated = true;
|
|
1431
|
+
}
|
|
1432
|
+
await fsOps.rename(temporary, target);
|
|
1433
|
+
candidatePublished = true;
|
|
1434
|
+
for (const candidate of retire) {
|
|
1435
|
+
const stagedPath = `${candidate.file}.${process.pid}.${randomUUID()}.capture-retire`;
|
|
1436
|
+
await fsOps.rename(candidate.file, stagedPath);
|
|
1437
|
+
staged.push({ original: candidate.file, staged: stagedPath });
|
|
1438
|
+
}
|
|
1439
|
+
const publishedSourceTarget =
|
|
1440
|
+
pairedSourceTarget ?? (targetName === sourceName ? target : null);
|
|
1441
|
+
if (publishedSourceTarget !== null) {
|
|
1442
|
+
const sourceValue = JSON.parse(
|
|
1443
|
+
await readPrivateFile(
|
|
1444
|
+
publishedSourceTarget,
|
|
1445
|
+
'published immutable capture source',
|
|
1446
|
+
MAX_CAPTURE_BYTES,
|
|
1447
|
+
fsOps,
|
|
1448
|
+
),
|
|
1449
|
+
);
|
|
1450
|
+
validateCaptureSnapshot(sourceValue, value.sourceRevision);
|
|
1451
|
+
if (sourceValue.threadId !== value.threadId)
|
|
1452
|
+
throw new Error('published capture source identity validation failed');
|
|
1453
|
+
}
|
|
1454
|
+
const publishedCurrent = JSON.parse(
|
|
1455
|
+
await readPrivateFile(target, 'published current capture', MAX_CAPTURE_BYTES, fsOps),
|
|
1456
|
+
);
|
|
1457
|
+
validateCaptureSnapshot(publishedCurrent, value.sourceRevision);
|
|
1458
|
+
if (publishedCurrent.threadId !== value.threadId)
|
|
1459
|
+
throw new Error('published capture group identity validation failed');
|
|
1460
|
+
const postState = await readCaptureDirectoryState(directory, fsOps);
|
|
1461
|
+
const postSteadyBytes =
|
|
1462
|
+
postState.totalBytes - (backupCreated ? replacedBytes : 0) - retiredBytes;
|
|
1463
|
+
if (
|
|
1464
|
+
postState.unsafeRecovery ||
|
|
1465
|
+
postState.blockedEntries ||
|
|
1466
|
+
postState.managedFiles.length > MAX_CAPTURE_FILES ||
|
|
1467
|
+
postSteadyBytes > MAX_CAPTURE_DIRECTORY_BYTES
|
|
1468
|
+
)
|
|
1469
|
+
throw captureCapacityError({
|
|
1470
|
+
totalBytes: postSteadyBytes,
|
|
1471
|
+
managedFiles: postState.managedFiles,
|
|
1472
|
+
});
|
|
1473
|
+
committed = true;
|
|
1474
|
+
let cleanupFailed = false;
|
|
1475
|
+
const removeCommittedPath = async (path) => {
|
|
1476
|
+
try {
|
|
1477
|
+
await fsOps.unlink(path);
|
|
1478
|
+
} catch (error) {
|
|
1479
|
+
if (error?.code !== 'ENOENT') cleanupFailed = true;
|
|
1480
|
+
}
|
|
1481
|
+
};
|
|
1482
|
+
if (backup !== null) await removeCommittedPath(backup);
|
|
1483
|
+
for (const item of staged) await removeCommittedPath(item.staged);
|
|
1484
|
+
if (cleanupFailed) {
|
|
1485
|
+
let actualState = postState;
|
|
1486
|
+
try {
|
|
1487
|
+
actualState = await readCaptureDirectoryState(directory, fsOps);
|
|
1488
|
+
} catch {}
|
|
1489
|
+
throw captureStorageCleanupError(actualState);
|
|
1490
|
+
}
|
|
1491
|
+
} catch (error) {
|
|
1492
|
+
if (committed) throw error;
|
|
1493
|
+
if (candidatePublished) await fsOps.unlink(target).catch(() => undefined);
|
|
1494
|
+
for (const item of staged)
|
|
1495
|
+
await fsOps.rename(item.staged, item.original).catch(() => undefined);
|
|
1496
|
+
if (backupCreated && backup !== null)
|
|
1497
|
+
await fsOps.rename(backup, target).catch(() => undefined);
|
|
1498
|
+
if (sourcePublished && pairedSourceTarget !== null)
|
|
1499
|
+
await fsOps.unlink(pairedSourceTarget).catch(() => undefined);
|
|
1500
|
+
throw error;
|
|
1501
|
+
}
|
|
1502
|
+
} catch (error) {
|
|
1503
|
+
await fsOps.unlink(temporary).catch(() => undefined);
|
|
1504
|
+
if (sourceTemporary !== null) await fsOps.unlink(sourceTemporary).catch(() => undefined);
|
|
1505
|
+
throw error;
|
|
1506
|
+
}
|
|
1507
|
+
};
|
|
1508
|
+
return lockOptions.lockHeld
|
|
1509
|
+
? operation()
|
|
1510
|
+
: withCaptureStoreLock(directory, operation, fsOps, lockOptions);
|
|
1511
|
+
}
|
|
1512
|
+
|
|
1513
|
+
async function retireUnreferencedCaptureGroup(
|
|
1514
|
+
directory,
|
|
1515
|
+
groupName,
|
|
1516
|
+
fsOps = captureFs,
|
|
1517
|
+
lockHeld = false,
|
|
1518
|
+
) {
|
|
1519
|
+
const operation = async () => {
|
|
1520
|
+
const state = await readCaptureDirectoryState(directory, fsOps);
|
|
1521
|
+
if (state.unsafeRecovery || state.blockedEntries || state.protectedGroups.has(groupName))
|
|
1522
|
+
return false;
|
|
1523
|
+
const members = state.managedFiles.filter(
|
|
1524
|
+
(entry) => entry.group === groupName && entry.immutable,
|
|
1525
|
+
);
|
|
1526
|
+
if (members.length === 0) return false;
|
|
1527
|
+
const staged = [];
|
|
1528
|
+
try {
|
|
1529
|
+
for (const entry of members) {
|
|
1530
|
+
const stagedPath = `${entry.file}.${process.pid}.${randomUUID()}.capture-retire`;
|
|
1531
|
+
await fsOps.rename(entry.file, stagedPath);
|
|
1532
|
+
staged.push({ original: entry.file, staged: stagedPath });
|
|
1533
|
+
}
|
|
1534
|
+
} catch (error) {
|
|
1535
|
+
for (const item of staged)
|
|
1536
|
+
await fsOps.rename(item.staged, item.original).catch(() => undefined);
|
|
1537
|
+
throw error;
|
|
1538
|
+
}
|
|
1539
|
+
for (const item of staged) await fsOps.unlink(item.staged).catch(() => undefined);
|
|
1540
|
+
return true;
|
|
1541
|
+
};
|
|
1542
|
+
return lockHeld ? operation() : withCaptureStoreLock(directory, operation, fsOps);
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1545
|
+
async function atomicWritePrivate(
|
|
1546
|
+
file,
|
|
1547
|
+
value,
|
|
1548
|
+
{
|
|
1549
|
+
maxBytes = null,
|
|
1550
|
+
compact = false,
|
|
1551
|
+
capture = false,
|
|
1552
|
+
captureLock = false,
|
|
1553
|
+
captureLockHeld = false,
|
|
1554
|
+
pairedSourcePath,
|
|
1555
|
+
} = {},
|
|
1556
|
+
) {
|
|
1557
|
+
const serialized = `${compact ? JSON.stringify(value) : JSON.stringify(value, null, 2)}\n`;
|
|
1558
|
+
if (maxBytes !== null && Buffer.byteLength(serialized, 'utf8') > maxBytes) {
|
|
1559
|
+
const error = new Error('capture snapshot exceeds bound');
|
|
1560
|
+
error.diagnosticReason = 'capture_size_exceeded';
|
|
1561
|
+
error.captureBytes = Buffer.byteLength(serialized, 'utf8');
|
|
1562
|
+
error.captureLimitBytes = maxBytes;
|
|
1563
|
+
throw error;
|
|
1564
|
+
}
|
|
1565
|
+
if (capture) {
|
|
1566
|
+
const lockOptions = {
|
|
1567
|
+
lockHeld: captureLockHeld,
|
|
1568
|
+
...(pairedSourcePath === undefined ? {} : { pairedSourcePath }),
|
|
1569
|
+
};
|
|
1570
|
+
const result = await writeBoundedCapture(file, serialized, value, captureFs, lockOptions);
|
|
1571
|
+
return result;
|
|
1572
|
+
}
|
|
1573
|
+
const directory = await ensurePrivateDirectory(dirname(file));
|
|
1574
|
+
if (captureLock) return withCaptureStoreLock(directory, () => writePrivateFile(file, serialized));
|
|
1575
|
+
return writePrivateFile(file, serialized);
|
|
1576
|
+
}
|
|
1577
|
+
async function writeSegmentedCapture(
|
|
1578
|
+
directory,
|
|
1579
|
+
{ threadId, sourceRevision, metadata, pages, lockHeld = false, fs = captureFs },
|
|
1580
|
+
) {
|
|
1581
|
+
if (!lockHeld)
|
|
1582
|
+
return withCaptureStoreLock(
|
|
1583
|
+
directory,
|
|
1584
|
+
() =>
|
|
1585
|
+
writeSegmentedCapture(directory, {
|
|
1586
|
+
threadId,
|
|
1587
|
+
sourceRevision,
|
|
1588
|
+
metadata,
|
|
1589
|
+
pages,
|
|
1590
|
+
lockHeld: true,
|
|
1591
|
+
fs,
|
|
1592
|
+
}),
|
|
1593
|
+
fs,
|
|
1594
|
+
);
|
|
1595
|
+
if (!Array.isArray(pages) || pages.length < 1 || pages.length > MAX_CAPTURE_PAGES)
|
|
1596
|
+
throw new Error('capture page count exceeds bound');
|
|
1597
|
+
const segments = [];
|
|
1598
|
+
const ownedStagePaths = new Set();
|
|
1599
|
+
const cleanupSource = async () => {
|
|
1600
|
+
for (const file of ownedStagePaths) {
|
|
1601
|
+
try {
|
|
1602
|
+
await assertPrivateFile(file, 'capture staged page cleanup', fs);
|
|
1603
|
+
} catch {
|
|
1604
|
+
continue;
|
|
1605
|
+
}
|
|
1606
|
+
await fs.unlink(file).catch((error) => {
|
|
1607
|
+
if (error?.code !== 'ENOENT') throw error;
|
|
1608
|
+
});
|
|
1609
|
+
}
|
|
1610
|
+
};
|
|
1611
|
+
try {
|
|
1612
|
+
for (const [descriptorPosition, descriptor] of pages.entries()) {
|
|
1613
|
+
if (
|
|
1614
|
+
!isRecord(descriptor) ||
|
|
1615
|
+
!isSafeBoundedInteger(descriptor.pageIndex, 0, MAX_CAPTURE_PAGES - 1) ||
|
|
1616
|
+
descriptor.pageIndex !== descriptorPosition ||
|
|
1617
|
+
typeof descriptor.stagedPath !== 'string' ||
|
|
1618
|
+
typeof descriptor.pageHash !== 'string' ||
|
|
1619
|
+
!/^[a-f0-9]{64}$/u.test(descriptor.pageHash)
|
|
1620
|
+
)
|
|
1621
|
+
throw new Error('capture staged page identity validation failed');
|
|
1622
|
+
const stagedPath = resolve(descriptor.stagedPath);
|
|
1623
|
+
if (
|
|
1624
|
+
dirname(stagedPath) !== resolve(directory) ||
|
|
1625
|
+
!CAPTURE_SOURCE_STAGE_NAME.test(basename(stagedPath))
|
|
1626
|
+
)
|
|
1627
|
+
throw new Error('capture staged page path validation failed');
|
|
1628
|
+
const stagedInfo = await assertPrivateFile(stagedPath, 'capture staged page', fs);
|
|
1629
|
+
ownedStagePaths.add(stagedPath);
|
|
1630
|
+
const page = JSON.parse(
|
|
1631
|
+
await readPrivateFile(stagedPath, 'capture staged page', MAX_CAPTURE_PAGE_BYTES, fs),
|
|
1632
|
+
);
|
|
1633
|
+
if (
|
|
1634
|
+
!isRecord(page) ||
|
|
1635
|
+
page.threadId !== threadId ||
|
|
1636
|
+
!Array.isArray(page.turns) ||
|
|
1637
|
+
hashJson(page) !== descriptor.pageHash
|
|
1638
|
+
)
|
|
1639
|
+
throw new Error('capture staged page integrity validation failed');
|
|
1640
|
+
const payload = {
|
|
1641
|
+
version: 'codex-capture-segment-v1',
|
|
1642
|
+
threadId,
|
|
1643
|
+
sourceRevision,
|
|
1644
|
+
pageIndex: descriptor.pageIndex,
|
|
1645
|
+
pageHash: descriptor.pageHash,
|
|
1646
|
+
itemCount: page.turns.reduce(
|
|
1647
|
+
(count, turn) => count + (Array.isArray(turn.items) ? turn.items.length : 0),
|
|
1648
|
+
0,
|
|
1649
|
+
),
|
|
1650
|
+
page,
|
|
1651
|
+
};
|
|
1652
|
+
validateCaptureSegment(payload, threadId, sourceRevision);
|
|
1653
|
+
const serialized = `${JSON.stringify(payload)}\n`;
|
|
1654
|
+
if (Buffer.byteLength(serialized, 'utf8') > MAX_CAPTURE_PAGE_BYTES)
|
|
1655
|
+
throw new Error('capture page exceeds bound');
|
|
1656
|
+
const stagedState = await readCaptureDirectoryState(directory, fs);
|
|
1657
|
+
if (
|
|
1658
|
+
stagedState.totalBytes - stagedInfo.size + Buffer.byteLength(serialized, 'utf8') >
|
|
1659
|
+
MAX_CAPTURE_DIRECTORY_BYTES + MAX_CAPTURE_STAGING_HEADROOM_BYTES
|
|
1660
|
+
)
|
|
1661
|
+
throw captureCapacityError(stagedState);
|
|
1662
|
+
await fs.writeFile(stagedPath, serialized, { flag: 'w', mode: 0o600 });
|
|
1663
|
+
const rewritten = await assertPrivateFile(stagedPath, 'capture staged segment', fs);
|
|
1664
|
+
if (rewritten.size !== Buffer.byteLength(serialized, 'utf8'))
|
|
1665
|
+
throw new Error('capture staged segment read-back size mismatch');
|
|
1666
|
+
validateCaptureSegment(
|
|
1667
|
+
JSON.parse(await fs.readFile(stagedPath, 'utf8')),
|
|
1668
|
+
threadId,
|
|
1669
|
+
sourceRevision,
|
|
1670
|
+
);
|
|
1671
|
+
segments.push({
|
|
1672
|
+
name: captureSegmentName(threadId, sourceRevision, descriptor.pageIndex),
|
|
1673
|
+
pageHash: descriptor.pageHash,
|
|
1674
|
+
itemCount: payload.itemCount,
|
|
1675
|
+
bytes: Buffer.byteLength(serialized, 'utf8'),
|
|
1676
|
+
stagedPath,
|
|
1677
|
+
});
|
|
1678
|
+
}
|
|
1679
|
+
} catch (error) {
|
|
1680
|
+
await cleanupSource();
|
|
1681
|
+
throw error;
|
|
1682
|
+
}
|
|
1683
|
+
const manifest = {
|
|
1684
|
+
version: 'codex-capture-manifest-v1',
|
|
1685
|
+
threadId,
|
|
1686
|
+
sourceRevision,
|
|
1687
|
+
pageCount: segments.length,
|
|
1688
|
+
itemCount: segments.reduce((count, segment) => count + segment.itemCount, 0),
|
|
1689
|
+
totalBytes: segments.reduce((count, segment) => count + segment.bytes, 0),
|
|
1690
|
+
segments: segments.map((segment) => segment.name),
|
|
1691
|
+
pageHashes: segments.map((segment) => segment.pageHash),
|
|
1692
|
+
};
|
|
1693
|
+
if (
|
|
1694
|
+
isRecord(metadata?.thread) &&
|
|
1695
|
+
captureManifestRevision(metadata.thread, manifest.pageHashes) !== sourceRevision
|
|
1696
|
+
) {
|
|
1697
|
+
await cleanupSource();
|
|
1698
|
+
throw new Error('capture manifest revision does not match page hashes');
|
|
1699
|
+
}
|
|
1700
|
+
if (manifest.totalBytes > MAX_CAPTURE_TOTAL_BYTES || manifest.itemCount > MAX_CAPTURE_ITEMS) {
|
|
1701
|
+
await cleanupSource();
|
|
1702
|
+
throw new Error('capture exceeds segmented bound');
|
|
1703
|
+
}
|
|
1704
|
+
const operation = async () => {
|
|
1705
|
+
const state = await readCaptureDirectoryState(directory, fs);
|
|
1706
|
+
if (state.unsafeRecovery || state.blockedEntries) {
|
|
1707
|
+
await cleanupSource();
|
|
1708
|
+
throw captureCapacityError(state);
|
|
1709
|
+
}
|
|
1710
|
+
const staged = [];
|
|
1711
|
+
const publishedTargets = [];
|
|
1712
|
+
try {
|
|
1713
|
+
for (const segment of segments) {
|
|
1714
|
+
const target = resolve(directory, segment.name);
|
|
1715
|
+
try {
|
|
1716
|
+
await assertPrivateFile(target, 'capture segment', fs);
|
|
1717
|
+
const existing = JSON.parse(
|
|
1718
|
+
await readPrivateFile(target, 'capture segment', MAX_CAPTURE_PAGE_BYTES, fs),
|
|
1719
|
+
);
|
|
1720
|
+
validateCaptureSegment(existing, threadId, sourceRevision);
|
|
1721
|
+
if (existing.pageHash !== segment.pageHash) throw new Error('capture segment collision');
|
|
1722
|
+
continue;
|
|
1723
|
+
} catch (error) {
|
|
1724
|
+
if (error?.code !== 'ENOENT') throw error;
|
|
1725
|
+
}
|
|
1726
|
+
staged.push({ temporary: segment.stagedPath, target, bytes: segment.bytes });
|
|
1727
|
+
}
|
|
1728
|
+
const protectedNames = new Set(segments.map((segment) => segment.name));
|
|
1729
|
+
const groups = new Map();
|
|
1730
|
+
for (const entry of state.managedFiles) {
|
|
1731
|
+
if (
|
|
1732
|
+
!entry.immutable ||
|
|
1733
|
+
protectedNames.has(entry.name) ||
|
|
1734
|
+
state.referencedImmutableNames.has(entry.name) ||
|
|
1735
|
+
state.protectedGroups.has(entry.group)
|
|
1736
|
+
)
|
|
1737
|
+
continue;
|
|
1738
|
+
const members = groups.get(entry.group ?? entry.name) ?? [];
|
|
1739
|
+
members.push(entry);
|
|
1740
|
+
groups.set(entry.group ?? entry.name, members);
|
|
1741
|
+
}
|
|
1742
|
+
const retired = [];
|
|
1743
|
+
let projected = state.totalBytes;
|
|
1744
|
+
let projectedFiles = state.managedFiles.length + staged.length;
|
|
1745
|
+
for (const members of [...groups.values()].sort(
|
|
1746
|
+
(left, right) =>
|
|
1747
|
+
Math.min(...left.map((entry) => entry.mtimeMs)) -
|
|
1748
|
+
Math.min(...right.map((entry) => entry.mtimeMs)),
|
|
1749
|
+
)) {
|
|
1750
|
+
if (projected <= MAX_CAPTURE_DIRECTORY_BYTES && projectedFiles <= MAX_CAPTURE_FILES) break;
|
|
1751
|
+
const bytes = members.reduce((sum, entry) => sum + entry.size, 0);
|
|
1752
|
+
projected -= bytes;
|
|
1753
|
+
projectedFiles -= members.length;
|
|
1754
|
+
retired.push(...members);
|
|
1755
|
+
}
|
|
1756
|
+
if (projected > MAX_CAPTURE_DIRECTORY_BYTES || projectedFiles > MAX_CAPTURE_PUBLICATION_FILES)
|
|
1757
|
+
throw captureCapacityError({
|
|
1758
|
+
totalBytes: projected,
|
|
1759
|
+
managedFileCount: projectedFiles,
|
|
1760
|
+
managedFiles: state.managedFiles,
|
|
1761
|
+
});
|
|
1762
|
+
const retiredPaths = [];
|
|
1763
|
+
try {
|
|
1764
|
+
for (const entry of retired) {
|
|
1765
|
+
const stagedPath = `${entry.file}.${process.pid}.${randomUUID()}.capture-retire`;
|
|
1766
|
+
await fs.rename(entry.file, stagedPath);
|
|
1767
|
+
retiredPaths.push({ original: entry.file, staged: stagedPath });
|
|
1768
|
+
}
|
|
1769
|
+
for (const item of staged) await fs.rename(item.temporary, item.target);
|
|
1770
|
+
publishedTargets.push(...staged.map((item) => item.target));
|
|
1771
|
+
for (const segment of segments) {
|
|
1772
|
+
const verify = JSON.parse(
|
|
1773
|
+
await readPrivateFile(
|
|
1774
|
+
resolve(directory, segment.name),
|
|
1775
|
+
'capture segment',
|
|
1776
|
+
MAX_CAPTURE_PAGE_BYTES,
|
|
1777
|
+
fs,
|
|
1778
|
+
),
|
|
1779
|
+
);
|
|
1780
|
+
validateCaptureSegment(verify, threadId, sourceRevision);
|
|
1781
|
+
}
|
|
1782
|
+
await cleanupSource();
|
|
1783
|
+
for (const item of retiredPaths) await fs.unlink(item.staged).catch(() => undefined);
|
|
1784
|
+
return manifest;
|
|
1785
|
+
} catch (error) {
|
|
1786
|
+
for (const item of retiredPaths)
|
|
1787
|
+
await fs.rename(item.staged, item.original).catch(() => undefined);
|
|
1788
|
+
if (error?.code === 'ENOENT') throw captureCapacityError(state);
|
|
1789
|
+
throw error;
|
|
1790
|
+
}
|
|
1791
|
+
} catch (error) {
|
|
1792
|
+
for (const item of staged) {
|
|
1793
|
+
await fs.unlink(item.temporary).catch(() => undefined);
|
|
1794
|
+
await fs.unlink(item.target).catch(() => undefined);
|
|
1795
|
+
}
|
|
1796
|
+
for (const target of publishedTargets) await fs.unlink(target).catch(() => undefined);
|
|
1797
|
+
await cleanupSource();
|
|
1798
|
+
throw error;
|
|
1799
|
+
}
|
|
1800
|
+
};
|
|
1801
|
+
return operation();
|
|
1802
|
+
}
|
|
1803
|
+
|
|
1804
|
+
/**
|
|
1805
|
+
* Test seam for the segmented capture store. Pages are written as private
|
|
1806
|
+
* staging files first, then published as one lock-bound capture group.
|
|
1807
|
+
*/
|
|
1808
|
+
export async function __testWriteSegmentedCapture(
|
|
1809
|
+
directory,
|
|
1810
|
+
{ threadId, sourceRevision, metadata = { thread: { id: threadId } }, pages },
|
|
1811
|
+
{ fs = captureFs } = {},
|
|
1812
|
+
) {
|
|
1813
|
+
await ensurePrivateDirectory(directory, fs);
|
|
1814
|
+
return withCaptureStoreLock(
|
|
1815
|
+
directory,
|
|
1816
|
+
async () => {
|
|
1817
|
+
const staged = [];
|
|
1818
|
+
try {
|
|
1819
|
+
for (const [pageIndex, page] of pages.entries()) {
|
|
1820
|
+
const serialized = `${JSON.stringify(page)}\n`;
|
|
1821
|
+
const state = await readCaptureDirectoryState(directory, fs);
|
|
1822
|
+
if (
|
|
1823
|
+
state.totalBytes + Buffer.byteLength(serialized, 'utf8') >
|
|
1824
|
+
MAX_CAPTURE_DIRECTORY_BYTES + MAX_CAPTURE_STAGING_HEADROOM_BYTES
|
|
1825
|
+
)
|
|
1826
|
+
throw captureCapacityError(state);
|
|
1827
|
+
const stagedPath = resolve(
|
|
1828
|
+
directory,
|
|
1829
|
+
`.capture-stage.${safeThreadName(threadId)}.${randomUUID()}.${pageIndex}.tmp`,
|
|
1830
|
+
);
|
|
1831
|
+
await fs.writeFile(stagedPath, serialized, { flag: 'wx', mode: 0o600 });
|
|
1832
|
+
staged.push({
|
|
1833
|
+
pageIndex,
|
|
1834
|
+
stagedPath,
|
|
1835
|
+
pageHash: hashJson(page),
|
|
1836
|
+
bytes: Buffer.byteLength(serialized, 'utf8'),
|
|
1837
|
+
});
|
|
1838
|
+
}
|
|
1839
|
+
return await writeSegmentedCapture(directory, {
|
|
1840
|
+
threadId,
|
|
1841
|
+
sourceRevision,
|
|
1842
|
+
metadata,
|
|
1843
|
+
pages: staged,
|
|
1844
|
+
lockHeld: true,
|
|
1845
|
+
fs,
|
|
1846
|
+
});
|
|
1847
|
+
} catch (error) {
|
|
1848
|
+
for (const page of staged) await fs.unlink(page.stagedPath).catch(() => undefined);
|
|
1849
|
+
throw error;
|
|
1850
|
+
}
|
|
1851
|
+
},
|
|
1852
|
+
fs,
|
|
1853
|
+
);
|
|
1854
|
+
}
|
|
1855
|
+
|
|
1856
|
+
export async function __testWriteSegmentedCaptureFromStages(
|
|
1857
|
+
directory,
|
|
1858
|
+
{ threadId, sourceRevision, metadata = { thread: { id: threadId } }, pages },
|
|
1859
|
+
{ fs = captureFs } = {},
|
|
1860
|
+
) {
|
|
1861
|
+
await ensurePrivateDirectory(directory, fs);
|
|
1862
|
+
return withCaptureStoreLock(
|
|
1863
|
+
directory,
|
|
1864
|
+
() =>
|
|
1865
|
+
writeSegmentedCapture(directory, {
|
|
1866
|
+
threadId,
|
|
1867
|
+
sourceRevision,
|
|
1868
|
+
metadata,
|
|
1869
|
+
pages,
|
|
1870
|
+
lockHeld: true,
|
|
1871
|
+
fs,
|
|
1872
|
+
}),
|
|
1873
|
+
fs,
|
|
486
1874
|
);
|
|
487
1875
|
}
|
|
488
1876
|
|
|
489
|
-
function
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
return
|
|
1877
|
+
export async function __testPublishSegmentedCapture(
|
|
1878
|
+
directory,
|
|
1879
|
+
{ threadId, sourceRevision, metadata = { thread: { id: threadId } }, pages },
|
|
1880
|
+
{ fs = captureFs } = {},
|
|
1881
|
+
) {
|
|
1882
|
+
await ensurePrivateDirectory(directory, fs);
|
|
1883
|
+
return withCaptureStoreLock(
|
|
1884
|
+
directory,
|
|
1885
|
+
async () => {
|
|
1886
|
+
const staged = [];
|
|
1887
|
+
let segmentsPublished = false;
|
|
1888
|
+
try {
|
|
1889
|
+
for (const [pageIndex, page] of pages.entries()) {
|
|
1890
|
+
const serialized = `${JSON.stringify(page)}\n`;
|
|
1891
|
+
const state = await readCaptureDirectoryState(directory, fs);
|
|
1892
|
+
if (
|
|
1893
|
+
state.totalBytes + Buffer.byteLength(serialized, 'utf8') >
|
|
1894
|
+
MAX_CAPTURE_DIRECTORY_BYTES + MAX_CAPTURE_STAGING_HEADROOM_BYTES
|
|
1895
|
+
)
|
|
1896
|
+
throw captureCapacityError(state);
|
|
1897
|
+
const stagedPath = resolve(
|
|
1898
|
+
directory,
|
|
1899
|
+
`.capture-stage.${safeThreadName(threadId)}.${randomUUID()}.${pageIndex}.tmp`,
|
|
1900
|
+
);
|
|
1901
|
+
await fs.writeFile(stagedPath, serialized, { flag: 'wx', mode: 0o600 });
|
|
1902
|
+
staged.push({
|
|
1903
|
+
pageIndex,
|
|
1904
|
+
stagedPath,
|
|
1905
|
+
pageHash: hashJson(page),
|
|
1906
|
+
bytes: Buffer.byteLength(serialized, 'utf8'),
|
|
1907
|
+
});
|
|
1908
|
+
}
|
|
1909
|
+
const manifest = await writeSegmentedCapture(directory, {
|
|
1910
|
+
threadId,
|
|
1911
|
+
sourceRevision,
|
|
1912
|
+
metadata,
|
|
1913
|
+
pages: staged,
|
|
1914
|
+
lockHeld: true,
|
|
1915
|
+
fs,
|
|
1916
|
+
});
|
|
1917
|
+
segmentsPublished = true;
|
|
1918
|
+
const value = {
|
|
1919
|
+
version: 2,
|
|
1920
|
+
threadId,
|
|
1921
|
+
sourceRevision,
|
|
1922
|
+
threadRead: metadata,
|
|
1923
|
+
captureManifest: manifest,
|
|
1924
|
+
};
|
|
1925
|
+
await writeBoundedCapture(
|
|
1926
|
+
resolve(directory, `${strictThreadName(threadId)}.json`),
|
|
1927
|
+
`${JSON.stringify(value)}\n`,
|
|
1928
|
+
value,
|
|
1929
|
+
fs,
|
|
1930
|
+
{
|
|
1931
|
+
lockHeld: true,
|
|
1932
|
+
pairedSourcePath: resolve(
|
|
1933
|
+
directory,
|
|
1934
|
+
`${strictThreadName(threadId)}.${sourceRevision}.json`,
|
|
1935
|
+
),
|
|
1936
|
+
},
|
|
1937
|
+
);
|
|
1938
|
+
return { manifest, value };
|
|
1939
|
+
} catch (error) {
|
|
1940
|
+
for (const page of staged) await fs.unlink(page.stagedPath).catch(() => undefined);
|
|
1941
|
+
if (segmentsPublished)
|
|
1942
|
+
await retireUnreferencedCaptureGroup(
|
|
1943
|
+
directory,
|
|
1944
|
+
`${threadId}.${sourceRevision}`,
|
|
1945
|
+
fs,
|
|
1946
|
+
true,
|
|
1947
|
+
).catch(() => undefined);
|
|
1948
|
+
throw error;
|
|
1949
|
+
}
|
|
1950
|
+
},
|
|
1951
|
+
fs,
|
|
1952
|
+
);
|
|
496
1953
|
}
|
|
497
1954
|
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
}
|
|
510
|
-
await chmod(target, 0o700);
|
|
511
|
-
return target;
|
|
512
|
-
}
|
|
513
|
-
async function assertPrivateFile(file, label) {
|
|
514
|
-
const info = await lstat(file);
|
|
515
|
-
if (info.isSymbolicLink() || !info.isFile() || (info.mode & 0o077) !== 0)
|
|
516
|
-
throw new Error(`${label} is not a private regular file`);
|
|
517
|
-
return info;
|
|
518
|
-
}
|
|
519
|
-
async function readPrivateFile(file, label, maxBytes = null) {
|
|
520
|
-
const info = await assertPrivateFile(file, label);
|
|
521
|
-
if (maxBytes !== null && info.size > maxBytes) throw new Error(`${label} exceeds its bound`);
|
|
522
|
-
return readFile(file, 'utf8');
|
|
1955
|
+
// Test-only seam for deterministic capture-store fault injection. Production
|
|
1956
|
+
// code continues to use the immutable writer above with the real filesystem.
|
|
1957
|
+
export async function __testWriteBoundedCapture(
|
|
1958
|
+
file,
|
|
1959
|
+
value,
|
|
1960
|
+
{ fs = captureFs, now = Date.now, sleep } = {},
|
|
1961
|
+
) {
|
|
1962
|
+
const serialized = `${JSON.stringify(value)}\n`;
|
|
1963
|
+
if (Buffer.byteLength(serialized, 'utf8') > MAX_CAPTURE_BYTES)
|
|
1964
|
+
throw new Error('capture snapshot exceeds bound');
|
|
1965
|
+
validateCaptureSnapshot(value, value.sourceRevision);
|
|
1966
|
+
return writeBoundedCapture(file, serialized, value, fs, { now, sleep });
|
|
523
1967
|
}
|
|
524
1968
|
|
|
525
|
-
async function
|
|
526
|
-
const
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
1969
|
+
export async function maintainCaptureStorage(directory, { fs = captureFs } = {}) {
|
|
1970
|
+
const target = await validatePrivateDirectoryPath(directory, fs);
|
|
1971
|
+
return withCaptureStoreLock(
|
|
1972
|
+
target,
|
|
1973
|
+
async ({ removedFiles = 0, removedBytes = 0 } = {}) => {
|
|
1974
|
+
const state = await readCaptureDirectoryState(target, fs);
|
|
1975
|
+
if (state.unsafeRecovery || state.blockedEntries) throw captureCapacityError(state);
|
|
1976
|
+
return {
|
|
1977
|
+
reclaimedFiles: removedFiles,
|
|
1978
|
+
reclaimedBytes: removedBytes,
|
|
1979
|
+
managedFiles: state.managedFiles.length,
|
|
1980
|
+
directoryBytes: state.totalBytes,
|
|
1981
|
+
fileLimit: MAX_CAPTURE_FILES,
|
|
1982
|
+
directoryByteLimit: MAX_CAPTURE_DIRECTORY_BYTES,
|
|
1983
|
+
};
|
|
1984
|
+
},
|
|
1985
|
+
fs,
|
|
530
1986
|
);
|
|
531
|
-
try {
|
|
532
|
-
await assertPrivateFile(target, 'checkpoint target');
|
|
533
|
-
} catch (error) {
|
|
534
|
-
if (error?.code !== 'ENOENT') throw error;
|
|
535
|
-
}
|
|
536
|
-
const temporary = `${target}.${process.pid}.${randomUUID()}.tmp`;
|
|
537
|
-
try {
|
|
538
|
-
await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, { flag: 'wx', mode: 0o600 });
|
|
539
|
-
await rename(temporary, target);
|
|
540
|
-
} catch (error) {
|
|
541
|
-
try {
|
|
542
|
-
await unlink(temporary);
|
|
543
|
-
} catch {
|
|
544
|
-
/* preserve the original failure */
|
|
545
|
-
}
|
|
546
|
-
throw error;
|
|
547
|
-
}
|
|
548
1987
|
}
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
);
|
|
555
|
-
|
|
556
|
-
throw new Error('immutable capture already belongs to another source');
|
|
557
|
-
} catch (error) {
|
|
558
|
-
if (error?.code !== 'ENOENT') throw error;
|
|
559
|
-
}
|
|
560
|
-
await atomicWritePrivate(file, value);
|
|
1988
|
+
|
|
1989
|
+
export async function __testHoldCaptureStoreLock(directory, readyPath) {
|
|
1990
|
+
await ensurePrivateDirectory(directory);
|
|
1991
|
+
return withCaptureStoreLock(directory, async () => {
|
|
1992
|
+
await writeFile(readyPath, 'ready\n', { flag: 'wx', mode: 0o600 });
|
|
1993
|
+
await new Promise((resolvePromise) => process.stdin.once('end', resolvePromise));
|
|
1994
|
+
});
|
|
561
1995
|
}
|
|
562
1996
|
async function atomicWriteTextPrivate(file, text) {
|
|
563
1997
|
if (Buffer.byteLength(text) > MAX_NOTE_BYTES) throw new Error('recovery notes exceed bound');
|
|
@@ -599,6 +2033,125 @@ function validateThreadRead(value, threadId) {
|
|
|
599
2033
|
return value;
|
|
600
2034
|
}
|
|
601
2035
|
|
|
2036
|
+
function validateThreadMetadata(value, threadId) {
|
|
2037
|
+
if (!isRecord(value) || !isRecord(value.thread) || value.thread.id !== threadId)
|
|
2038
|
+
throw new Error('thread/read returned invalid thread metadata');
|
|
2039
|
+
return value;
|
|
2040
|
+
}
|
|
2041
|
+
|
|
2042
|
+
export function parseTurnsPage(value, threadId, pageIndex) {
|
|
2043
|
+
if (!isRecord(value)) throw new Error('thread/turns/list returned invalid page');
|
|
2044
|
+
const data = Array.isArray(value.data)
|
|
2045
|
+
? value.data
|
|
2046
|
+
: Array.isArray(value.turns)
|
|
2047
|
+
? value.turns
|
|
2048
|
+
: null;
|
|
2049
|
+
if (data === null || data.some((turn) => !isRecord(turn) || typeof turn.id !== 'string'))
|
|
2050
|
+
throw new Error('thread/turns/list returned invalid turns');
|
|
2051
|
+
const page = { threadId, turns: data };
|
|
2052
|
+
const serializedBytes = Buffer.byteLength(JSON.stringify(page), 'utf8');
|
|
2053
|
+
if (serializedBytes > MAX_CAPTURE_PAGE_BYTES) {
|
|
2054
|
+
const error = new Error('capture page exceeds bound');
|
|
2055
|
+
error.diagnosticReason = 'capture_size_exceeded';
|
|
2056
|
+
error.captureBytes = serializedBytes;
|
|
2057
|
+
error.captureLimitBytes = MAX_CAPTURE_PAGE_BYTES;
|
|
2058
|
+
throw error;
|
|
2059
|
+
}
|
|
2060
|
+
const cursor = value.nextCursor ?? value.next_cursor ?? null;
|
|
2061
|
+
if (cursor !== null && (typeof cursor !== 'string' || cursor.length > 512))
|
|
2062
|
+
throw new Error('thread/turns/list returned invalid cursor');
|
|
2063
|
+
return { page, pageIndex, cursor };
|
|
2064
|
+
}
|
|
2065
|
+
|
|
2066
|
+
function summarizeCaptureItem(item) {
|
|
2067
|
+
if (!isRecord(item)) return { type: 'unknown' };
|
|
2068
|
+
const summary = { type: typeof item.type === 'string' ? item.type : 'unknown' };
|
|
2069
|
+
if (typeof item.id === 'string') summary.id = item.id;
|
|
2070
|
+
if (summary.type === 'userMessage') {
|
|
2071
|
+
const parts = Array.isArray(item.content)
|
|
2072
|
+
? item.content.filter((part) => isRecord(part) && typeof part.text === 'string')
|
|
2073
|
+
: [];
|
|
2074
|
+
const contentChars = parts.reduce((total, part) => total + part.text.length, 0);
|
|
2075
|
+
summary.content = parts.slice(0, 64).map((part) => ({
|
|
2076
|
+
type: part.type,
|
|
2077
|
+
text: part.text.slice(0, MAX_NOTE_ITEM_CHARS),
|
|
2078
|
+
}));
|
|
2079
|
+
summary.contentChars = contentChars;
|
|
2080
|
+
summary.bodyComplete =
|
|
2081
|
+
parts.length <= 64 && parts.every((part) => part.text.length <= MAX_NOTE_ITEM_CHARS);
|
|
2082
|
+
} else if (summary.type === 'agentMessage' || summary.type === 'plan') {
|
|
2083
|
+
const text = typeof item.text === 'string' ? item.text : '';
|
|
2084
|
+
summary.text = text.slice(0, MAX_NOTE_ITEM_CHARS);
|
|
2085
|
+
summary.textChars = text.length;
|
|
2086
|
+
summary.bodyComplete = text.length <= MAX_NOTE_ITEM_CHARS;
|
|
2087
|
+
} else if (summary.type === 'commandExecution') {
|
|
2088
|
+
const command = typeof item.command === 'string' ? item.command : '';
|
|
2089
|
+
summary.command = command.slice(0, MAX_NOTE_ITEM_CHARS);
|
|
2090
|
+
summary.commandComplete = command.length <= MAX_NOTE_ITEM_CHARS;
|
|
2091
|
+
for (const key of ['status', 'exitCode']) {
|
|
2092
|
+
if (key in item && (typeof item[key] === 'string' || typeof item[key] === 'number'))
|
|
2093
|
+
summary[key] = item[key];
|
|
2094
|
+
}
|
|
2095
|
+
summary.aggregatedOutput = String(item.aggregatedOutput ?? item.output ?? '').slice(0, 1024);
|
|
2096
|
+
summary.outputChars = String(item.aggregatedOutput ?? item.output ?? '').length;
|
|
2097
|
+
}
|
|
2098
|
+
return summary;
|
|
2099
|
+
}
|
|
2100
|
+
|
|
2101
|
+
function summarizeCaptureTurn(turn) {
|
|
2102
|
+
return {
|
|
2103
|
+
...(typeof turn.id === 'string' ? { id: turn.id } : {}),
|
|
2104
|
+
...(typeof turn.status === 'string' ? { status: turn.status } : {}),
|
|
2105
|
+
items: turn.items.map(summarizeCaptureItem),
|
|
2106
|
+
};
|
|
2107
|
+
}
|
|
2108
|
+
|
|
2109
|
+
export function mergePaginatedTurns(metadata, pages, threadId) {
|
|
2110
|
+
validateThreadMetadata(metadata, threadId);
|
|
2111
|
+
if (!Array.isArray(pages) || pages.length < 1 || pages.length > MAX_CAPTURE_PAGES)
|
|
2112
|
+
throw new Error('capture page count exceeds bound');
|
|
2113
|
+
const turns = [];
|
|
2114
|
+
const ids = new Set();
|
|
2115
|
+
const itemIds = new Set();
|
|
2116
|
+
let itemCount = 0;
|
|
2117
|
+
for (const [pagePosition, entry] of pages.entries()) {
|
|
2118
|
+
const page = entry?.page;
|
|
2119
|
+
if (!isRecord(page) || page.threadId !== threadId || !Array.isArray(page.turns))
|
|
2120
|
+
throw new Error('capture page identity validation failed');
|
|
2121
|
+
if (entry.pageIndex !== pagePosition) throw new Error('capture pages are out of order');
|
|
2122
|
+
for (const turn of page.turns) {
|
|
2123
|
+
if (!isRecord(turn) || typeof turn.id !== 'string' || ids.has(turn.id))
|
|
2124
|
+
throw new Error('capture history contains duplicate turn');
|
|
2125
|
+
ids.add(turn.id);
|
|
2126
|
+
if (!Array.isArray(turn.items)) throw new Error('capture turn items are unavailable');
|
|
2127
|
+
for (const item of turn.items) {
|
|
2128
|
+
if (isRecord(item) && typeof item.id === 'string') {
|
|
2129
|
+
if (itemIds.has(item.id)) throw new Error('capture history contains duplicate item');
|
|
2130
|
+
itemIds.add(item.id);
|
|
2131
|
+
}
|
|
2132
|
+
}
|
|
2133
|
+
itemCount += turn.items.length;
|
|
2134
|
+
if (itemCount > MAX_CAPTURE_ITEMS) {
|
|
2135
|
+
const error = new Error('capture item count exceeds bound');
|
|
2136
|
+
error.captureItemCount = itemCount;
|
|
2137
|
+
error.captureItemLimit = MAX_CAPTURE_ITEMS;
|
|
2138
|
+
throw error;
|
|
2139
|
+
}
|
|
2140
|
+
turns.push(turn);
|
|
2141
|
+
}
|
|
2142
|
+
}
|
|
2143
|
+
if (turns.length === 0) throw new Error('thread/turns/list returned no recoverable turns');
|
|
2144
|
+
return { thread: { ...metadata.thread, turns } };
|
|
2145
|
+
}
|
|
2146
|
+
|
|
2147
|
+
function captureSourceRevision(capture) {
|
|
2148
|
+
if (capture.pages === null) return hashJson(capture.read);
|
|
2149
|
+
return captureManifestRevision(
|
|
2150
|
+
capture.metadata.thread,
|
|
2151
|
+
capture.pages.map(({ pageHash }) => pageHash),
|
|
2152
|
+
);
|
|
2153
|
+
}
|
|
2154
|
+
|
|
602
2155
|
export function buildRecoveryCandidates(read, revision, policy = null, options = {}) {
|
|
603
2156
|
const turns = read.thread.turns;
|
|
604
2157
|
const rawItems = turns.flatMap((turn, turnIndex) =>
|
|
@@ -613,14 +2166,23 @@ export function buildRecoveryCandidates(read, revision, policy = null, options =
|
|
|
613
2166
|
throw new Error('recovery candidate bound is invalid');
|
|
614
2167
|
if (rawItems.length > maxItems && options.allowWindow !== true)
|
|
615
2168
|
throw new Error('recovery candidate count exceeds bound');
|
|
616
|
-
if (rawItems.length > MAX_RECOVERY_SOURCE_INDEX)
|
|
617
|
-
|
|
2169
|
+
if (rawItems.length > MAX_RECOVERY_SOURCE_INDEX) {
|
|
2170
|
+
const error = new Error('recovery source index exceeds bound');
|
|
2171
|
+
error.captureItemCount = rawItems.length;
|
|
2172
|
+
error.captureItemLimit = MAX_RECOVERY_SOURCE_INDEX;
|
|
2173
|
+
throw error;
|
|
2174
|
+
}
|
|
618
2175
|
let items = rawItems.map(({ item: rawItem, turnIndex, itemIndex }, index) => {
|
|
619
2176
|
const item = isRecord(rawItem) ? rawItem : {};
|
|
620
2177
|
const type = typeof item.type === 'string' ? item.type : 'unknown';
|
|
621
2178
|
const commandOutput =
|
|
622
2179
|
type === 'commandExecution' ? String(item.aggregatedOutput ?? item.output ?? '') : '';
|
|
2180
|
+
const sourceOutputChars =
|
|
2181
|
+
type === 'commandExecution' && isSafeCounter(item.outputChars)
|
|
2182
|
+
? item.outputChars
|
|
2183
|
+
: commandOutput.length;
|
|
623
2184
|
const command = typeof item.command === 'string' ? item.command : 'unknown';
|
|
2185
|
+
const bodyComplete = item.bodyComplete !== false;
|
|
624
2186
|
const status = typeof item.status === 'string' ? item.status : 'unknown';
|
|
625
2187
|
const exitCode = typeof item.exitCode === 'number' ? item.exitCode : null;
|
|
626
2188
|
const localText =
|
|
@@ -666,7 +2228,8 @@ export function buildRecoveryCandidates(read, revision, policy = null, options =
|
|
|
666
2228
|
) &&
|
|
667
2229
|
(commandName !== 'git' ||
|
|
668
2230
|
['status', 'diff', 'log', 'show', 'rev-parse'].includes(commandWords[1])));
|
|
669
|
-
const unknownCommand =
|
|
2231
|
+
const unknownCommand =
|
|
2232
|
+
type === 'commandExecution' && (!readOnlyCommand || item.commandComplete === false);
|
|
670
2233
|
const unknown = !['userMessage', 'agentMessage', 'plan', 'commandExecution'].includes(type);
|
|
671
2234
|
const recentCount = policy?.preserveRecentMessages ?? 6;
|
|
672
2235
|
const recent = index >= Math.max(0, rawItems.length - recentCount);
|
|
@@ -688,7 +2251,14 @@ export function buildRecoveryCandidates(read, revision, policy = null, options =
|
|
|
688
2251
|
if (unknownCommand) pinReasons.push('unknown');
|
|
689
2252
|
if (unknown) pinReasons.push('unknown');
|
|
690
2253
|
if (recent && pinReasons.length === 0) pinReasons.push('recent');
|
|
691
|
-
const localChars =
|
|
2254
|
+
const localChars =
|
|
2255
|
+
type === 'commandExecution'
|
|
2256
|
+
? sourceOutputChars
|
|
2257
|
+
: type === 'userMessage' && isSafeCounter(item.contentChars)
|
|
2258
|
+
? item.contentChars
|
|
2259
|
+
: (type === 'agentMessage' || type === 'plan') && isSafeCounter(item.textChars)
|
|
2260
|
+
? item.textChars
|
|
2261
|
+
: localText.length;
|
|
692
2262
|
return {
|
|
693
2263
|
id: `recovery-${index + 1}`,
|
|
694
2264
|
text: redact(localText, 16_384),
|
|
@@ -700,7 +2270,8 @@ export function buildRecoveryCandidates(read, revision, policy = null, options =
|
|
|
700
2270
|
sourceItemId: typeof item.id === 'string' ? item.id : `item-${index + 1}`,
|
|
701
2271
|
sourceLocation: { flatIndex: index, turnIndex, itemIndex },
|
|
702
2272
|
sourceRefs: null,
|
|
703
|
-
sourceBodyChars:
|
|
2273
|
+
sourceBodyChars: sourceOutputChars,
|
|
2274
|
+
bodyComplete,
|
|
704
2275
|
kind,
|
|
705
2276
|
};
|
|
706
2277
|
});
|
|
@@ -1025,8 +2596,16 @@ export function recoveryNotesText(
|
|
|
1025
2596
|
isSafeBoundedInteger(ref.itemIndex, 0, MAX_RECOVERY_SOURCE_INDEX),
|
|
1026
2597
|
)
|
|
1027
2598
|
.sort((left, right) => left.flatIndex - right.flatIndex);
|
|
1028
|
-
const
|
|
2599
|
+
const unique = [];
|
|
2600
|
+
const seen = new Set();
|
|
1029
2601
|
for (const ref of sorted) {
|
|
2602
|
+
const key = `${ref.flatIndex}:${ref.turnIndex}:${ref.itemIndex}:${String(ref.sourceItemId ?? '')}`;
|
|
2603
|
+
if (seen.has(key)) continue;
|
|
2604
|
+
seen.add(key);
|
|
2605
|
+
unique.push(ref);
|
|
2606
|
+
}
|
|
2607
|
+
const ranges = [];
|
|
2608
|
+
for (const ref of unique) {
|
|
1030
2609
|
const previous = ranges.at(-1);
|
|
1031
2610
|
if (previous && ref.flatIndex === previous.end + 1) previous.end = ref.flatIndex;
|
|
1032
2611
|
else ranges.push({ start: ref.flatIndex, end: ref.flatIndex });
|
|
@@ -1037,8 +2616,8 @@ export function recoveryNotesText(
|
|
|
1037
2616
|
range.start === range.end ? String(range.start) : `${range.start}-${range.end}`,
|
|
1038
2617
|
)
|
|
1039
2618
|
.join(','),
|
|
1040
|
-
locations:
|
|
1041
|
-
ids:
|
|
2619
|
+
locations: unique.map((ref) => `${ref.turnIndex}:${ref.itemIndex}`).join(','),
|
|
2620
|
+
ids: unique
|
|
1042
2621
|
.map((ref) =>
|
|
1043
2622
|
typeof ref.sourceItemId === 'string' ? escapeReference(ref.sourceItemId) : 'unknown',
|
|
1044
2623
|
)
|
|
@@ -1076,6 +2655,7 @@ export function recoveryNotesText(
|
|
|
1076
2655
|
const inline =
|
|
1077
2656
|
item.kind !== 'tool_result' &&
|
|
1078
2657
|
item.kind !== 'reference' &&
|
|
2658
|
+
item.bodyComplete !== false &&
|
|
1079
2659
|
item.localText.length > 0 &&
|
|
1080
2660
|
item.localText.length <= MAX_NOTE_ITEM_CHARS &&
|
|
1081
2661
|
Buffer.byteLength(item.localText, 'utf8') <= inlineBudget;
|
|
@@ -1099,27 +2679,6 @@ export function recoveryNotesText(
|
|
|
1099
2679
|
return `${lines.join('\n')}`;
|
|
1100
2680
|
}
|
|
1101
2681
|
|
|
1102
|
-
function recoveryInjectionState(read, notes) {
|
|
1103
|
-
const matches = read.thread.turns
|
|
1104
|
-
.flatMap((turn) => (Array.isArray(turn.items) ? turn.items : []))
|
|
1105
|
-
.filter(
|
|
1106
|
-
(item) =>
|
|
1107
|
-
isRecord(item) &&
|
|
1108
|
-
((item.type === 'message' &&
|
|
1109
|
-
item.role === 'user' &&
|
|
1110
|
-
Array.isArray(item.content) &&
|
|
1111
|
-
item.content.some(
|
|
1112
|
-
(part) => isRecord(part) && part.type === 'input_text' && part.text === notes,
|
|
1113
|
-
)) ||
|
|
1114
|
-
(item.type === 'userMessage' &&
|
|
1115
|
-
Array.isArray(item.content) &&
|
|
1116
|
-
item.content.some(
|
|
1117
|
-
(part) => isRecord(part) && part.type === 'text' && part.text === notes,
|
|
1118
|
-
))),
|
|
1119
|
-
);
|
|
1120
|
-
return matches.length === 0 ? 'absent' : matches.length === 1 ? 'present' : 'ambiguous';
|
|
1121
|
-
}
|
|
1122
|
-
|
|
1123
2682
|
/** Own a Codex app-server process and request native compaction after a fresh usage event. */
|
|
1124
2683
|
export function createCodexContextProxy({
|
|
1125
2684
|
child,
|
|
@@ -1153,6 +2712,8 @@ export function createCodexContextProxy({
|
|
|
1153
2712
|
const effectiveThreshold = contextPolicy?.compactAtPercent ?? threshold;
|
|
1154
2713
|
const threads = new Map();
|
|
1155
2714
|
const ownedRequests = new Map();
|
|
2715
|
+
const ownedThreadIds = new Set();
|
|
2716
|
+
const clientRequestBindings = new Map();
|
|
1156
2717
|
const internalReads = new Map();
|
|
1157
2718
|
const retiredRequests = new Set();
|
|
1158
2719
|
const telemetryByThread = new Map();
|
|
@@ -1262,9 +2823,10 @@ export function createCodexContextProxy({
|
|
|
1262
2823
|
attachment.invalidated = true;
|
|
1263
2824
|
cleanupRequest(attachment.readId);
|
|
1264
2825
|
cleanupRequest(attachment.injectId);
|
|
2826
|
+
for (const id of attachment.auxIds ?? []) cleanupRequest(id);
|
|
1265
2827
|
}
|
|
1266
2828
|
|
|
1267
|
-
function fail(threadId, reason) {
|
|
2829
|
+
function fail(threadId, reason, cause = null) {
|
|
1268
2830
|
const state = threads.get(threadId);
|
|
1269
2831
|
const pending = state?.pending;
|
|
1270
2832
|
invalidateAttachment(state);
|
|
@@ -1280,9 +2842,11 @@ export function createCodexContextProxy({
|
|
|
1280
2842
|
emitDiagnostic({
|
|
1281
2843
|
phase: 'thread_failure',
|
|
1282
2844
|
stage: pending.stage ?? 'native',
|
|
1283
|
-
reason: diagnosticReasonForError(new Error(reason)),
|
|
2845
|
+
reason: diagnosticReasonForError(cause ?? new Error(reason)),
|
|
1284
2846
|
threadId,
|
|
1285
2847
|
operationId,
|
|
2848
|
+
...(boundedFailureCounts(cause) === undefined ? {} : { counts: boundedFailureCounts(cause) }),
|
|
2849
|
+
...(safeRpcCode(cause?.rpcCode) === null ? {} : { rpcCode: cause.rpcCode }),
|
|
1286
2850
|
});
|
|
1287
2851
|
stderr.write(`codex-context: native compaction preserved (${reason})\n`);
|
|
1288
2852
|
}
|
|
@@ -1325,10 +2889,27 @@ export function createCodexContextProxy({
|
|
|
1325
2889
|
recoveryLoadPromise: null,
|
|
1326
2890
|
recoveryAttach: null,
|
|
1327
2891
|
telemetryConnected: false,
|
|
2892
|
+
owned: false,
|
|
2893
|
+
// Codex can publish an idle/usage snapshot before a resumed thread has
|
|
2894
|
+
// finished loading. Native MCP and compact requests sent in that gap
|
|
2895
|
+
// are rejected as invalid requests by the 0.163 app-server.
|
|
2896
|
+
resumePending: false,
|
|
2897
|
+
resumeRetryRequested: false,
|
|
2898
|
+
resumeRequests: new Set(),
|
|
1328
2899
|
};
|
|
1329
2900
|
threads.set(threadId, created);
|
|
1330
2901
|
return created;
|
|
1331
2902
|
}
|
|
2903
|
+
function claimOwnedThread(threadId) {
|
|
2904
|
+
const existing = threads.get(threadId);
|
|
2905
|
+
if (!existing && ownedThreadIds.size >= MAX_OWNED_THREAD_IDS) return null;
|
|
2906
|
+
const state = existing ?? threadState(threadId);
|
|
2907
|
+
if (state.owned) return state;
|
|
2908
|
+
if (ownedThreadIds.size >= MAX_OWNED_THREAD_IDS) return null;
|
|
2909
|
+
state.owned = true;
|
|
2910
|
+
ownedThreadIds.add(threadId);
|
|
2911
|
+
return state;
|
|
2912
|
+
}
|
|
1332
2913
|
function sendOwned(method, params, kind, threadId, waiter) {
|
|
1333
2914
|
if (ownedRequests.size >= MAX_OWNED_REQUESTS)
|
|
1334
2915
|
throw new Error('too many pending adapter requests');
|
|
@@ -1528,6 +3109,160 @@ export function createCodexContextProxy({
|
|
|
1528
3109
|
if (pending.canceled || state.pending !== pending || !state.idle || state.revision !== revision)
|
|
1529
3110
|
throw new Error('Codex state changed during compaction');
|
|
1530
3111
|
}
|
|
3112
|
+
async function readThreadCapture(
|
|
3113
|
+
threadId,
|
|
3114
|
+
pending,
|
|
3115
|
+
firstWaiter = null,
|
|
3116
|
+
{ onPage = null, readId: existingReadId = null, verifyOnly = false } = {},
|
|
3117
|
+
) {
|
|
3118
|
+
const waiter = firstWaiter ?? deferred();
|
|
3119
|
+
const readId =
|
|
3120
|
+
existingReadId ??
|
|
3121
|
+
sendOwned('thread/read', { threadId, includeTurns: false }, 'read', threadId, waiter);
|
|
3122
|
+
if (pending) {
|
|
3123
|
+
pending.readId = readId;
|
|
3124
|
+
pending.auxIds.push(readId);
|
|
3125
|
+
}
|
|
3126
|
+
const metadata = validateThreadMetadata(await waiter.promise, threadId);
|
|
3127
|
+
if (Array.isArray(metadata.thread.turns) && metadata.thread.turns.length > 0) {
|
|
3128
|
+
const legacy = validateThreadRead(metadata, threadId);
|
|
3129
|
+
await onPage?.(metadata.thread);
|
|
3130
|
+
return { read: legacy, metadata, pages: null };
|
|
3131
|
+
}
|
|
3132
|
+
const pages = [];
|
|
3133
|
+
const seenTurnIds = new Set();
|
|
3134
|
+
const seenItemIds = new Set();
|
|
3135
|
+
let totalItems = 0;
|
|
3136
|
+
if (!verifyOnly) await ensurePrivateDirectory(checkpointDir);
|
|
3137
|
+
const summaryTurns = [];
|
|
3138
|
+
const stagedFiles = [];
|
|
3139
|
+
let stagedBytes = 0;
|
|
3140
|
+
const cleanupStaged = async () => {
|
|
3141
|
+
for (const file of stagedFiles) await unlink(file).catch(() => undefined);
|
|
3142
|
+
};
|
|
3143
|
+
const seenCursors = new Set();
|
|
3144
|
+
let cursor = null;
|
|
3145
|
+
for (let pageIndex = 0; pageIndex < MAX_CAPTURE_PAGES; pageIndex += 1) {
|
|
3146
|
+
let pageResponse;
|
|
3147
|
+
let pageLimit = 16;
|
|
3148
|
+
for (let attempt = 0; attempt < 5; attempt += 1) {
|
|
3149
|
+
const pageWaiter = deferred();
|
|
3150
|
+
const params = {
|
|
3151
|
+
threadId,
|
|
3152
|
+
itemsView: 'full',
|
|
3153
|
+
sortDirection: 'asc',
|
|
3154
|
+
limit: pageLimit,
|
|
3155
|
+
};
|
|
3156
|
+
if (cursor !== null) params.cursor = cursor;
|
|
3157
|
+
const pageId = sendOwned('thread/turns/list', params, 'page', threadId, pageWaiter);
|
|
3158
|
+
if (pending) pending.auxIds.push(pageId);
|
|
3159
|
+
try {
|
|
3160
|
+
pageResponse = await pageWaiter.promise;
|
|
3161
|
+
break;
|
|
3162
|
+
} catch (error) {
|
|
3163
|
+
if (error?.diagnosticReason === 'capture_size_exceeded' && attempt < 4) {
|
|
3164
|
+
pageLimit = Math.max(1, Math.floor(pageLimit / 2));
|
|
3165
|
+
continue;
|
|
3166
|
+
}
|
|
3167
|
+
await cleanupStaged();
|
|
3168
|
+
if (error?.rpcCode === -32601 && pages.length === 0) {
|
|
3169
|
+
const legacyWaiter = deferred();
|
|
3170
|
+
const legacyId = sendOwned(
|
|
3171
|
+
'thread/read',
|
|
3172
|
+
{ threadId, includeTurns: true },
|
|
3173
|
+
'read',
|
|
3174
|
+
threadId,
|
|
3175
|
+
legacyWaiter,
|
|
3176
|
+
);
|
|
3177
|
+
if (pending) pending.auxIds.push(legacyId);
|
|
3178
|
+
const legacy = validateThreadRead(await legacyWaiter.promise, threadId);
|
|
3179
|
+
return { read: legacy, metadata, pages: null };
|
|
3180
|
+
}
|
|
3181
|
+
throw error;
|
|
3182
|
+
}
|
|
3183
|
+
}
|
|
3184
|
+
if (pageResponse === undefined) throw new Error('thread/turns/list page unavailable');
|
|
3185
|
+
try {
|
|
3186
|
+
const parsed = parseTurnsPage(pageResponse, threadId, pageIndex);
|
|
3187
|
+
if (parsed.page.turns.length === 0 && parsed.cursor !== null)
|
|
3188
|
+
throw new Error('thread/turns/list returned empty page with cursor');
|
|
3189
|
+
for (const turn of parsed.page.turns) {
|
|
3190
|
+
if (seenTurnIds.has(turn.id)) throw new Error('capture history contains duplicate turn');
|
|
3191
|
+
seenTurnIds.add(turn.id);
|
|
3192
|
+
if (!Array.isArray(turn.items)) throw new Error('capture turn items are unavailable');
|
|
3193
|
+
totalItems += turn.items.length;
|
|
3194
|
+
if (totalItems > MAX_CAPTURE_ITEMS) {
|
|
3195
|
+
const error = new Error('capture item count exceeds bound');
|
|
3196
|
+
error.captureItemCount = totalItems;
|
|
3197
|
+
error.captureItemLimit = MAX_CAPTURE_ITEMS;
|
|
3198
|
+
throw error;
|
|
3199
|
+
}
|
|
3200
|
+
for (const item of turn.items) {
|
|
3201
|
+
if (isRecord(item) && typeof item.id === 'string') {
|
|
3202
|
+
if (seenItemIds.has(item.id))
|
|
3203
|
+
throw new Error('capture history contains duplicate item');
|
|
3204
|
+
seenItemIds.add(item.id);
|
|
3205
|
+
}
|
|
3206
|
+
}
|
|
3207
|
+
}
|
|
3208
|
+
const serializedPage = `${JSON.stringify(parsed.page)}\n`;
|
|
3209
|
+
await onPage?.(parsed.page);
|
|
3210
|
+
const pageBytes = Buffer.byteLength(serializedPage, 'utf8');
|
|
3211
|
+
stagedBytes += pageBytes;
|
|
3212
|
+
if (stagedBytes > MAX_CAPTURE_TOTAL_BYTES)
|
|
3213
|
+
throw new Error('capture exceeds segmented bound');
|
|
3214
|
+
let stagedPath = null;
|
|
3215
|
+
if (!verifyOnly) {
|
|
3216
|
+
const storageState = await readCaptureDirectoryState(checkpointDir);
|
|
3217
|
+
if (
|
|
3218
|
+
storageState.totalBytes + pageBytes >
|
|
3219
|
+
MAX_CAPTURE_DIRECTORY_BYTES + MAX_CAPTURE_STAGING_HEADROOM_BYTES
|
|
3220
|
+
)
|
|
3221
|
+
throw captureCapacityError(storageState);
|
|
3222
|
+
stagedPath = resolve(
|
|
3223
|
+
checkpointDir,
|
|
3224
|
+
`.capture-stage.${safeThreadName(threadId)}.${randomUUID()}.${pageIndex}.tmp`,
|
|
3225
|
+
);
|
|
3226
|
+
await writeFile(stagedPath, serializedPage, { flag: 'wx', mode: 0o600 });
|
|
3227
|
+
stagedFiles.push(stagedPath);
|
|
3228
|
+
}
|
|
3229
|
+
pages.push({
|
|
3230
|
+
pageIndex,
|
|
3231
|
+
stagedPath,
|
|
3232
|
+
pageHash: hashJson(parsed.page),
|
|
3233
|
+
itemCount: parsed.page.turns.reduce(
|
|
3234
|
+
(count, turn) => count + (Array.isArray(turn.items) ? turn.items.length : 0),
|
|
3235
|
+
0,
|
|
3236
|
+
),
|
|
3237
|
+
bytes: pageBytes,
|
|
3238
|
+
cursor: parsed.cursor,
|
|
3239
|
+
});
|
|
3240
|
+
summaryTurns.push(...parsed.page.turns.map(summarizeCaptureTurn));
|
|
3241
|
+
if (parsed.cursor === null)
|
|
3242
|
+
return {
|
|
3243
|
+
read: {
|
|
3244
|
+
thread: {
|
|
3245
|
+
...metadata.thread,
|
|
3246
|
+
turns: summaryTurns,
|
|
3247
|
+
},
|
|
3248
|
+
},
|
|
3249
|
+
metadata,
|
|
3250
|
+
pages,
|
|
3251
|
+
stagedFiles,
|
|
3252
|
+
};
|
|
3253
|
+
if (seenCursors.has(parsed.cursor)) throw new Error('thread/turns/list cursor repeated');
|
|
3254
|
+
seenCursors.add(parsed.cursor);
|
|
3255
|
+
cursor = parsed.cursor;
|
|
3256
|
+
if (pageIndex === MAX_CAPTURE_PAGES - 1)
|
|
3257
|
+
throw new Error('thread/turns/list page count exceeds bound');
|
|
3258
|
+
} catch (error) {
|
|
3259
|
+
await cleanupStaged();
|
|
3260
|
+
throw error;
|
|
3261
|
+
}
|
|
3262
|
+
}
|
|
3263
|
+
await cleanupStaged();
|
|
3264
|
+
throw new Error('thread/turns/list page count exceeds bound');
|
|
3265
|
+
}
|
|
1531
3266
|
async function loadRecoveryState(threadId, state) {
|
|
1532
3267
|
if (state.recoveryLoaded) return state.recovery;
|
|
1533
3268
|
if (!state.recoveryLoadPromise) {
|
|
@@ -1596,6 +3331,7 @@ export function createCodexContextProxy({
|
|
|
1596
3331
|
await atomicWritePrivate(
|
|
1597
3332
|
resolve(checkpointDir, `${safeThreadName(threadId)}.recovery.json`),
|
|
1598
3333
|
record,
|
|
3334
|
+
{ captureLock: true },
|
|
1599
3335
|
);
|
|
1600
3336
|
}
|
|
1601
3337
|
async function readRecoveryNotes(threadId, record) {
|
|
@@ -1634,6 +3370,7 @@ export function createCodexContextProxy({
|
|
|
1634
3370
|
injectSent: false,
|
|
1635
3371
|
readId: null,
|
|
1636
3372
|
injectId: null,
|
|
3373
|
+
auxIds: [],
|
|
1637
3374
|
deferred: injectWaiter,
|
|
1638
3375
|
timer: null,
|
|
1639
3376
|
};
|
|
@@ -1643,23 +3380,44 @@ export function createCodexContextProxy({
|
|
|
1643
3380
|
attachment.canceled = true;
|
|
1644
3381
|
cleanupRequest(attachment.readId);
|
|
1645
3382
|
cleanupRequest(attachment.injectId);
|
|
3383
|
+
for (const id of attachment.auxIds) cleanupRequest(id);
|
|
1646
3384
|
injectWaiter.reject(new Error('recovery attachment timed out'));
|
|
1647
3385
|
}, timeoutMs);
|
|
1648
3386
|
try {
|
|
1649
3387
|
const notes = await readRecoveryNotes(threadId, record);
|
|
1650
3388
|
if (attachment.invalidated || attachment.canceled)
|
|
1651
3389
|
throw new Error('recovery attachment was invalidated before read');
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
3390
|
+
let latest = null;
|
|
3391
|
+
let injection = 'absent';
|
|
3392
|
+
const capture = await readThreadCapture(threadId, attachment, readWaiter, {
|
|
3393
|
+
verifyOnly: true,
|
|
3394
|
+
onPage: async (page) => {
|
|
3395
|
+
const pageLatest = page.turns?.at(-1);
|
|
3396
|
+
if (pageLatest) latest = pageLatest;
|
|
3397
|
+
const found = page.turns
|
|
3398
|
+
?.flatMap((turn) => (Array.isArray(turn.items) ? turn.items : []))
|
|
3399
|
+
.some(
|
|
3400
|
+
(item) =>
|
|
3401
|
+
isRecord(item) &&
|
|
3402
|
+
((item.type === 'message' &&
|
|
3403
|
+
item.role === 'user' &&
|
|
3404
|
+
Array.isArray(item.content) &&
|
|
3405
|
+
item.content.some(
|
|
3406
|
+
(part) => isRecord(part) && part.type === 'input_text' && part.text === notes,
|
|
3407
|
+
)) ||
|
|
3408
|
+
(item.type === 'userMessage' &&
|
|
3409
|
+
Array.isArray(item.content) &&
|
|
3410
|
+
item.content.some(
|
|
3411
|
+
(part) => isRecord(part) && part.type === 'text' && part.text === notes,
|
|
3412
|
+
))),
|
|
3413
|
+
);
|
|
3414
|
+
if (found) injection = injection === 'present' ? 'ambiguous' : 'present';
|
|
3415
|
+
},
|
|
3416
|
+
});
|
|
3417
|
+
for (const descriptor of capture.pages ?? [])
|
|
3418
|
+
if (descriptor.stagedPath) await unlink(descriptor.stagedPath).catch(() => undefined);
|
|
1660
3419
|
if (attachment.invalidated || attachment.canceled)
|
|
1661
3420
|
throw new Error('recovery attachment was invalidated during read');
|
|
1662
|
-
const latest = read.thread.turns.at(-1);
|
|
1663
3421
|
if (!latest || latest.id !== record.turnId || latest.status !== 'completed') {
|
|
1664
3422
|
attachment.stale = true;
|
|
1665
3423
|
throw new Error('recovery compaction turn is no longer current');
|
|
@@ -1668,7 +3426,6 @@ export function createCodexContextProxy({
|
|
|
1668
3426
|
attachment.stale = true;
|
|
1669
3427
|
throw new Error('recovery attachment encountered a newer turn');
|
|
1670
3428
|
}
|
|
1671
|
-
const injection = recoveryInjectionState(read, notes);
|
|
1672
3429
|
if (injection === 'ambiguous') {
|
|
1673
3430
|
attachment.stale = true;
|
|
1674
3431
|
throw new Error('recovery attachment has ambiguous prior injections');
|
|
@@ -1751,6 +3508,13 @@ export function createCodexContextProxy({
|
|
|
1751
3508
|
};
|
|
1752
3509
|
await persistRecoveryState(threadId, attached);
|
|
1753
3510
|
state.recovery = attached;
|
|
3511
|
+
emitDiagnostic({
|
|
3512
|
+
phase: 'recovery_attached',
|
|
3513
|
+
stage: 'recovery',
|
|
3514
|
+
reason: 'attached',
|
|
3515
|
+
threadId,
|
|
3516
|
+
operationId: record.operationId,
|
|
3517
|
+
});
|
|
1754
3518
|
emitOutcome({ threadId, status: 'recovery_attached', turnId: record.turnId });
|
|
1755
3519
|
} catch {
|
|
1756
3520
|
if (attachment.injectSent) {
|
|
@@ -1784,11 +3548,17 @@ export function createCodexContextProxy({
|
|
|
1784
3548
|
if (attachment.timer) attachment.timer();
|
|
1785
3549
|
cleanupRequest(attachment.readId);
|
|
1786
3550
|
cleanupRequest(attachment.injectId);
|
|
3551
|
+
for (const id of attachment.auxIds) cleanupRequest(id);
|
|
1787
3552
|
if (state.recoveryAttach === attachment) state.recoveryAttach = null;
|
|
1788
3553
|
}
|
|
1789
3554
|
}
|
|
1790
3555
|
async function requestCompaction(threadId, state) {
|
|
1791
|
-
|
|
3556
|
+
if (!state.owned) return;
|
|
3557
|
+
if (state.resumePending) {
|
|
3558
|
+
state.resumeRetryRequested = true;
|
|
3559
|
+
return;
|
|
3560
|
+
}
|
|
3561
|
+
if (recoveryEnabled && recoveryDisclosed) await loadRecoveryState(threadId, state);
|
|
1792
3562
|
if (
|
|
1793
3563
|
state.recovery &&
|
|
1794
3564
|
['pending_attachment', 'recovery_uncertain'].includes(state.recovery.status)
|
|
@@ -1847,42 +3617,95 @@ export function createCodexContextProxy({
|
|
|
1847
3617
|
pending.stage = 'capture';
|
|
1848
3618
|
pending.readId = sendOwned(
|
|
1849
3619
|
'thread/read',
|
|
1850
|
-
{ threadId, includeTurns:
|
|
3620
|
+
{ threadId, includeTurns: false },
|
|
1851
3621
|
'read',
|
|
1852
3622
|
threadId,
|
|
1853
3623
|
readWaiter,
|
|
1854
3624
|
);
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
3625
|
+
await ensurePrivateDirectory(checkpointDir);
|
|
3626
|
+
const publication = await withCaptureStoreLock(checkpointDir, async () => {
|
|
3627
|
+
const capture = await readThreadCapture(threadId, pending, readWaiter, {
|
|
3628
|
+
readId: pending.readId,
|
|
3629
|
+
});
|
|
3630
|
+
const read = capture.read;
|
|
3631
|
+
assertCurrent(threadId, state, pending, revision);
|
|
3632
|
+
const sourceRevision = captureSourceRevision(capture);
|
|
3633
|
+
const compactSnapshot = {
|
|
3634
|
+
version: 1,
|
|
3635
|
+
status: 'requested',
|
|
3636
|
+
capturedAt: new Date(now()).toISOString(),
|
|
3637
|
+
threadId,
|
|
3638
|
+
sourceTurnId,
|
|
3639
|
+
revision,
|
|
3640
|
+
sourceRevision,
|
|
3641
|
+
statusAtCheckpoint: state.status,
|
|
3642
|
+
usage: state.usage,
|
|
3643
|
+
threadRead: read,
|
|
3644
|
+
recovery: {
|
|
3645
|
+
mode: 'codex-native-compaction',
|
|
3646
|
+
action: 'resume-the-owned-app-server-session',
|
|
3647
|
+
historyMutation: false,
|
|
3648
|
+
},
|
|
3649
|
+
};
|
|
3650
|
+
let snapshot = compactSnapshot;
|
|
3651
|
+
if (capture.pages !== null) {
|
|
3652
|
+
const manifest = await writeSegmentedCapture(checkpointDir, {
|
|
3653
|
+
threadId,
|
|
3654
|
+
sourceRevision,
|
|
3655
|
+
metadata: capture.metadata,
|
|
3656
|
+
pages: capture.pages,
|
|
3657
|
+
lockHeld: true,
|
|
3658
|
+
});
|
|
3659
|
+
snapshot = {
|
|
3660
|
+
...compactSnapshot,
|
|
3661
|
+
version: 2,
|
|
3662
|
+
threadRead: capture.metadata,
|
|
3663
|
+
captureManifest: manifest,
|
|
3664
|
+
};
|
|
3665
|
+
} else if (Buffer.byteLength(JSON.stringify(compactSnapshot), 'utf8') > MAX_CAPTURE_BYTES) {
|
|
3666
|
+
throw new Error('capture snapshot exceeds bound');
|
|
3667
|
+
}
|
|
3668
|
+
const immutableCapturePath = resolve(
|
|
3669
|
+
checkpointDir,
|
|
3670
|
+
`${safeThreadName(threadId)}.${sourceRevision}.json`,
|
|
3671
|
+
);
|
|
3672
|
+
try {
|
|
3673
|
+
await atomicWritePrivate(
|
|
3674
|
+
resolve(checkpointDir, `${safeThreadName(threadId)}.json`),
|
|
3675
|
+
snapshot,
|
|
3676
|
+
{
|
|
3677
|
+
maxBytes: MAX_CAPTURE_BYTES,
|
|
3678
|
+
compact: true,
|
|
3679
|
+
capture: true,
|
|
3680
|
+
captureLockHeld: true,
|
|
3681
|
+
pairedSourcePath: immutableCapturePath,
|
|
3682
|
+
},
|
|
3683
|
+
);
|
|
3684
|
+
} catch (error) {
|
|
3685
|
+
await retireUnreferencedCaptureGroup(
|
|
3686
|
+
checkpointDir,
|
|
3687
|
+
`${threadId}.${sourceRevision}`,
|
|
3688
|
+
captureFs,
|
|
3689
|
+
true,
|
|
3690
|
+
).catch(() => undefined);
|
|
3691
|
+
throw error;
|
|
3692
|
+
}
|
|
3693
|
+
return { capture, read, sourceRevision, snapshot, immutableCapturePath };
|
|
3694
|
+
});
|
|
3695
|
+
const { capture, read, sourceRevision, snapshot, immutableCapturePath } = publication;
|
|
3696
|
+
if (capture.pages !== null && recoveryEnabled && recoveryDisclosed) {
|
|
3697
|
+
const verificationWaiter = deferred();
|
|
3698
|
+
const verificationCapture = await readThreadCapture(threadId, pending, verificationWaiter, {
|
|
3699
|
+
verifyOnly: true,
|
|
3700
|
+
});
|
|
3701
|
+
try {
|
|
3702
|
+
if (captureSourceRevision(verificationCapture) !== sourceRevision)
|
|
3703
|
+
throw new Error('recovery source changed before provider admission');
|
|
3704
|
+
} finally {
|
|
3705
|
+
for (const descriptor of verificationCapture.pages ?? [])
|
|
3706
|
+
if (descriptor.stagedPath) await unlink(descriptor.stagedPath).catch(() => undefined);
|
|
3707
|
+
}
|
|
3708
|
+
}
|
|
1886
3709
|
emitDiagnostic({
|
|
1887
3710
|
phase: 'capture',
|
|
1888
3711
|
stage: 'capture',
|
|
@@ -1894,6 +3717,15 @@ export function createCodexContextProxy({
|
|
|
1894
3717
|
(count, turn) => count + (turn.items?.length ?? 0),
|
|
1895
3718
|
0,
|
|
1896
3719
|
),
|
|
3720
|
+
...(capture.pages === null
|
|
3721
|
+
? {}
|
|
3722
|
+
: {
|
|
3723
|
+
page_count: capture.pages.length,
|
|
3724
|
+
capture_bytes: capture.pages.reduce(
|
|
3725
|
+
(count, descriptor) => count + descriptor.bytes,
|
|
3726
|
+
0,
|
|
3727
|
+
),
|
|
3728
|
+
}),
|
|
1897
3729
|
},
|
|
1898
3730
|
});
|
|
1899
3731
|
assertCurrent(threadId, state, pending, revision);
|
|
@@ -2170,18 +4002,18 @@ export function createCodexContextProxy({
|
|
|
2170
4002
|
});
|
|
2171
4003
|
}
|
|
2172
4004
|
const rereadWaiter = deferred();
|
|
2173
|
-
const
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
|
|
2179
|
-
|
|
2180
|
-
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
4005
|
+
const rereadCapture = await readThreadCapture(threadId, pending, rereadWaiter, {
|
|
4006
|
+
verifyOnly: true,
|
|
4007
|
+
});
|
|
4008
|
+
try {
|
|
4009
|
+
if (captureSourceRevision(rereadCapture) !== sourceRevision)
|
|
4010
|
+
throw new Error('recovery source changed during selection');
|
|
4011
|
+
assertCurrent(threadId, state, pending, revision);
|
|
4012
|
+
} finally {
|
|
4013
|
+
for (const descriptor of rereadCapture.pages ?? [])
|
|
4014
|
+
if (descriptor.stagedPath)
|
|
4015
|
+
await unlink(descriptor.stagedPath).catch(() => undefined);
|
|
4016
|
+
}
|
|
2185
4017
|
recoveryNotes = recoveryNotesText(
|
|
2186
4018
|
threadId,
|
|
2187
4019
|
sourceRevision,
|
|
@@ -2240,6 +4072,7 @@ export function createCodexContextProxy({
|
|
|
2240
4072
|
await atomicWritePrivate(
|
|
2241
4073
|
resolve(checkpointDir, `${safeThreadName(threadId)}.completed.json`),
|
|
2242
4074
|
completed,
|
|
4075
|
+
{ maxBytes: MAX_CAPTURE_BYTES, compact: true, capture: true },
|
|
2243
4076
|
);
|
|
2244
4077
|
if (recoveryNotes !== undefined) {
|
|
2245
4078
|
state.recovery = {
|
|
@@ -2274,8 +4107,11 @@ export function createCodexContextProxy({
|
|
|
2274
4107
|
reason: diagnosticReasonForError(error),
|
|
2275
4108
|
threadId,
|
|
2276
4109
|
operationId: pending.operationId,
|
|
4110
|
+
...(boundedFailureCounts(error) === undefined
|
|
4111
|
+
? {}
|
|
4112
|
+
: { counts: boundedFailureCounts(error) }),
|
|
2277
4113
|
});
|
|
2278
|
-
fail(threadId, 'checkpoint, request, or native completion failed');
|
|
4114
|
+
fail(threadId, 'checkpoint, request, or native completion failed', error);
|
|
2279
4115
|
} finally {
|
|
2280
4116
|
if (pending.timer) pending.timer();
|
|
2281
4117
|
cleanupRequest(pending.readId);
|
|
@@ -2297,6 +4133,7 @@ export function createCodexContextProxy({
|
|
|
2297
4133
|
retire(message.id);
|
|
2298
4134
|
if (
|
|
2299
4135
|
owned.kind === 'read' ||
|
|
4136
|
+
owned.kind === 'page' ||
|
|
2300
4137
|
owned.kind === 'select' ||
|
|
2301
4138
|
owned.kind === 'budget' ||
|
|
2302
4139
|
owned.kind === 'inject' ||
|
|
@@ -2305,18 +4142,33 @@ export function createCodexContextProxy({
|
|
|
2305
4142
|
const waiter = internalReads.get(message.id);
|
|
2306
4143
|
internalReads.delete(message.id);
|
|
2307
4144
|
if (waiter) {
|
|
2308
|
-
if ('error' in message) waiter.reject(
|
|
4145
|
+
if ('error' in message) waiter.reject(rpcFailure(owned.kind, message.error));
|
|
2309
4146
|
else waiter.resolve(message.result);
|
|
2310
4147
|
}
|
|
2311
|
-
} else if ('error' in message)
|
|
2312
|
-
|
|
4148
|
+
} else if ('error' in message) {
|
|
4149
|
+
const failure = rpcFailure(owned.kind, message.error);
|
|
4150
|
+
fail(owned.threadId, 'app-server request rejected', failure);
|
|
4151
|
+
}
|
|
4152
|
+
if ('error' in message) {
|
|
4153
|
+
const failure = rpcFailure(owned.kind, message.error);
|
|
2313
4154
|
emitDiagnostic({
|
|
2314
4155
|
phase: owned.kind,
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
?
|
|
2318
|
-
: '
|
|
4156
|
+
stage:
|
|
4157
|
+
owned.kind === 'read' || owned.kind === 'page'
|
|
4158
|
+
? 'capture'
|
|
4159
|
+
: owned.kind === 'budget'
|
|
4160
|
+
? 'budget'
|
|
4161
|
+
: owned.kind === 'select'
|
|
4162
|
+
? 'selection'
|
|
4163
|
+
: owned.kind === 'inject'
|
|
4164
|
+
? 'recovery'
|
|
4165
|
+
: 'transport',
|
|
4166
|
+
reason: 'protocol_error',
|
|
4167
|
+
threadId: owned.threadId,
|
|
4168
|
+
operationId: threads.get(owned.threadId)?.pending?.operationId,
|
|
4169
|
+
...(safeRpcCode(failure.rpcCode) === null ? {} : { rpcCode: failure.rpcCode }),
|
|
2319
4170
|
});
|
|
4171
|
+
}
|
|
2320
4172
|
if (owned.kind === 'compact') {
|
|
2321
4173
|
const state = threads.get(owned.threadId);
|
|
2322
4174
|
if (state?.pending) state.pending.compactId = null;
|
|
@@ -2329,17 +4181,19 @@ export function createCodexContextProxy({
|
|
|
2329
4181
|
typeof message.params.threadId === 'string' &&
|
|
2330
4182
|
isRecord(message.params.status)
|
|
2331
4183
|
) {
|
|
2332
|
-
const state =
|
|
4184
|
+
const state = threads.get(message.params.threadId);
|
|
4185
|
+
if (!state) return true;
|
|
2333
4186
|
state.status =
|
|
2334
4187
|
typeof message.params.status.type === 'string' ? message.params.status.type : 'unknown';
|
|
4188
|
+
state.idle = state.status === 'idle';
|
|
4189
|
+
if (!state.idle) invalidateAttachment(state);
|
|
4190
|
+
if (!state.owned) return true;
|
|
2335
4191
|
if (!state.telemetryConnected) {
|
|
2336
4192
|
state.telemetryConnected = true;
|
|
2337
4193
|
telemetryFor(message.params.threadId).connected({
|
|
2338
4194
|
triggerPercent: effectiveThreshold,
|
|
2339
4195
|
});
|
|
2340
4196
|
}
|
|
2341
|
-
state.idle = state.status === 'idle';
|
|
2342
|
-
if (!state.idle) invalidateAttachment(state);
|
|
2343
4197
|
if (state.idle) void requestCompaction(message.params.threadId, state);
|
|
2344
4198
|
}
|
|
2345
4199
|
if (
|
|
@@ -2360,19 +4214,10 @@ export function createCodexContextProxy({
|
|
|
2360
4214
|
stderr.write('codex-context: invalid usage sample skipped\n');
|
|
2361
4215
|
return true;
|
|
2362
4216
|
}
|
|
2363
|
-
const state =
|
|
4217
|
+
const state = threads.get(params.threadId);
|
|
4218
|
+
if (!state) return true;
|
|
2364
4219
|
state.usage = params.tokenUsage;
|
|
2365
4220
|
state.usageReceivedAt = now();
|
|
2366
|
-
telemetryFor(params.threadId).usage({
|
|
2367
|
-
measurement: {
|
|
2368
|
-
unit: 'tokens',
|
|
2369
|
-
used: params.tokenUsage.last.totalTokens,
|
|
2370
|
-
...(hasKnownContextWindow(params.tokenUsage)
|
|
2371
|
-
? { capacity: params.tokenUsage.modelContextWindow }
|
|
2372
|
-
: {}),
|
|
2373
|
-
},
|
|
2374
|
-
triggerPercent: effectiveThreshold,
|
|
2375
|
-
});
|
|
2376
4221
|
state.turnId = params.turnId;
|
|
2377
4222
|
state.revision = `${state.turnId ?? 'unknown'}:${String(params.tokenUsage.last?.totalTokens ?? '')}:${String(params.tokenUsage.modelContextWindow ?? '')}`;
|
|
2378
4223
|
if (
|
|
@@ -2384,6 +4229,17 @@ export function createCodexContextProxy({
|
|
|
2384
4229
|
state.turnId !== state.recovery.sourceTurnId
|
|
2385
4230
|
)
|
|
2386
4231
|
invalidateAttachment(state);
|
|
4232
|
+
if (!state.owned) return true;
|
|
4233
|
+
telemetryFor(params.threadId).usage({
|
|
4234
|
+
measurement: {
|
|
4235
|
+
unit: 'tokens',
|
|
4236
|
+
used: params.tokenUsage.last.totalTokens,
|
|
4237
|
+
...(hasKnownContextWindow(params.tokenUsage)
|
|
4238
|
+
? { capacity: params.tokenUsage.modelContextWindow }
|
|
4239
|
+
: {}),
|
|
4240
|
+
},
|
|
4241
|
+
triggerPercent: effectiveThreshold,
|
|
4242
|
+
});
|
|
2387
4243
|
if (state.idle) void requestCompaction(params.threadId, state);
|
|
2388
4244
|
}
|
|
2389
4245
|
if (
|
|
@@ -2487,11 +4343,86 @@ export function createCodexContextProxy({
|
|
|
2487
4343
|
typeof message.params.threadId === 'string'
|
|
2488
4344
|
) {
|
|
2489
4345
|
fail(message.params.threadId, 'thread closed');
|
|
4346
|
+
ownedThreadIds.delete(message.params.threadId);
|
|
2490
4347
|
threads.delete(message.params.threadId);
|
|
2491
4348
|
}
|
|
2492
4349
|
return true;
|
|
2493
4350
|
}
|
|
2494
4351
|
|
|
4352
|
+
/**
|
|
4353
|
+
* Observe TUI-owned requests without changing their wire representation.
|
|
4354
|
+
* The app-server may emit status/usage events before a resumed thread's
|
|
4355
|
+
* response arrives, so the adapter must defer owned work until that response
|
|
4356
|
+
* confirms that the thread is loaded.
|
|
4357
|
+
*/
|
|
4358
|
+
function observeClientRequest(message) {
|
|
4359
|
+
if (!isRecord(message)) return;
|
|
4360
|
+
const requestId = message.id;
|
|
4361
|
+
const validRequestId =
|
|
4362
|
+
(typeof requestId === 'string' && requestId.length > 0 && requestId.length <= 128) ||
|
|
4363
|
+
Number.isSafeInteger(requestId);
|
|
4364
|
+
if (!validRequestId) return;
|
|
4365
|
+
const requestKey = clientRequestKey(requestId);
|
|
4366
|
+
if (requestKey === null) return;
|
|
4367
|
+
if (['thread/start', 'thread/resume', 'thread/fork'].includes(message.method)) {
|
|
4368
|
+
const threadId =
|
|
4369
|
+
isRecord(message.params) && typeof message.params.threadId === 'string'
|
|
4370
|
+
? message.params.threadId
|
|
4371
|
+
: null;
|
|
4372
|
+
if (
|
|
4373
|
+
clientRequestBindings.has(requestKey) ||
|
|
4374
|
+
clientRequestBindings.size >= MAX_CLIENT_REQUEST_BINDINGS
|
|
4375
|
+
)
|
|
4376
|
+
return;
|
|
4377
|
+
clientRequestBindings.set(requestKey, { method: message.method, threadId });
|
|
4378
|
+
if (message.method === 'thread/resume') {
|
|
4379
|
+
if (threadId === null || threadId.length === 0 || threadId.length > 256) {
|
|
4380
|
+
clientRequestBindings.delete(requestKey);
|
|
4381
|
+
return;
|
|
4382
|
+
}
|
|
4383
|
+
const state = threadState(threadId);
|
|
4384
|
+
state.resumePending = true;
|
|
4385
|
+
state.resumeRetryRequested = false;
|
|
4386
|
+
state.resumeRequests.add(requestKey);
|
|
4387
|
+
}
|
|
4388
|
+
}
|
|
4389
|
+
}
|
|
4390
|
+
|
|
4391
|
+
function observeClientResponse(message) {
|
|
4392
|
+
if (!isRecord(message) || message.id === undefined) return;
|
|
4393
|
+
const requestKey = clientRequestKey(message.id);
|
|
4394
|
+
if (requestKey === null) return;
|
|
4395
|
+
const binding = clientRequestBindings.get(requestKey);
|
|
4396
|
+
if (binding) {
|
|
4397
|
+
clientRequestBindings.delete(requestKey);
|
|
4398
|
+
if (binding.method === 'thread/resume' && binding.threadId !== null) {
|
|
4399
|
+
const state = threadState(binding.threadId);
|
|
4400
|
+
state.resumeRequests.delete(requestKey);
|
|
4401
|
+
state.resumePending = state.resumeRequests.size > 0;
|
|
4402
|
+
state.resumeRetryRequested = false;
|
|
4403
|
+
const responseThreadId = message.result?.thread?.id;
|
|
4404
|
+
if (
|
|
4405
|
+
message.error === undefined &&
|
|
4406
|
+
typeof responseThreadId === 'string' &&
|
|
4407
|
+
responseThreadId === binding.threadId
|
|
4408
|
+
) {
|
|
4409
|
+
const claimed = claimOwnedThread(binding.threadId);
|
|
4410
|
+
if (claimed && !state.resumePending && claimed.idle)
|
|
4411
|
+
void requestCompaction(binding.threadId, claimed);
|
|
4412
|
+
} else if (!state.owned && state.resumeRequests.size === 0 && state.pending === null) {
|
|
4413
|
+
threads.delete(binding.threadId);
|
|
4414
|
+
}
|
|
4415
|
+
} else if (message.error === undefined) {
|
|
4416
|
+
const selectedId = message.result?.thread?.id;
|
|
4417
|
+
if (typeof selectedId === 'string' && selectedId.length > 0 && selectedId.length <= 256) {
|
|
4418
|
+
const selected = claimOwnedThread(selectedId);
|
|
4419
|
+
if (!selected) return;
|
|
4420
|
+
if (selected.idle) void requestCompaction(selectedId, selected);
|
|
4421
|
+
}
|
|
4422
|
+
}
|
|
4423
|
+
}
|
|
4424
|
+
}
|
|
4425
|
+
|
|
2495
4426
|
const decoder = new StringDecoder('utf8');
|
|
2496
4427
|
let inputBuffer = '';
|
|
2497
4428
|
let passthrough = false;
|
|
@@ -2508,17 +4439,19 @@ export function createCodexContextProxy({
|
|
|
2508
4439
|
}
|
|
2509
4440
|
};
|
|
2510
4441
|
const processLine = (line) => {
|
|
2511
|
-
if (Buffer.byteLength(line) > MAX_LINE_BYTES) {
|
|
4442
|
+
if (Buffer.byteLength(line, 'utf8') + 1 > MAX_LINE_BYTES) {
|
|
2512
4443
|
const ownedId = internalRequestId(line);
|
|
2513
4444
|
if (ownedId !== null) {
|
|
2514
4445
|
stderr.write('codex-context: oversized internal response rejected\n');
|
|
2515
|
-
failOversizedInternal(ownedId);
|
|
4446
|
+
failOversizedInternal(ownedId, Buffer.byteLength(line, 'utf8') + 1);
|
|
2516
4447
|
} else forward(`${line}\n`);
|
|
2517
4448
|
return;
|
|
2518
4449
|
}
|
|
2519
4450
|
let forwardLine = true;
|
|
2520
4451
|
try {
|
|
2521
|
-
|
|
4452
|
+
const message = JSON.parse(line.endsWith('\r') ? line.slice(0, -1) : line);
|
|
4453
|
+
observeClientResponse(message);
|
|
4454
|
+
forwardLine = handle(message);
|
|
2522
4455
|
} catch {
|
|
2523
4456
|
/* preserve malformed external protocol lines */
|
|
2524
4457
|
}
|
|
@@ -2536,15 +4469,31 @@ export function createCodexContextProxy({
|
|
|
2536
4469
|
if (state.pending) fail(threadId, reason);
|
|
2537
4470
|
}
|
|
2538
4471
|
}
|
|
2539
|
-
function failOversizedInternal(id) {
|
|
4472
|
+
function failOversizedInternal(id, responseBytes) {
|
|
2540
4473
|
const owned = ownedRequests.get(id);
|
|
2541
4474
|
if (!owned) {
|
|
2542
4475
|
failCompactions('oversized internal response');
|
|
2543
4476
|
return;
|
|
2544
4477
|
}
|
|
4478
|
+
if (owned.kind === 'page') {
|
|
4479
|
+
ownedRequests.delete(id);
|
|
4480
|
+
const waiter = internalReads.get(id);
|
|
4481
|
+
internalReads.delete(id);
|
|
4482
|
+
retire(id);
|
|
4483
|
+
const failure = new Error('oversized internal page response');
|
|
4484
|
+
failure.diagnosticReason = 'capture_size_exceeded';
|
|
4485
|
+
failure.responseBytes = responseBytes;
|
|
4486
|
+
failure.responseLimitBytes = MAX_LINE_BYTES;
|
|
4487
|
+
waiter?.reject(failure);
|
|
4488
|
+
return;
|
|
4489
|
+
}
|
|
2545
4490
|
cleanupRequest(id);
|
|
2546
4491
|
if (owned.kind === 'telemetry') return;
|
|
2547
|
-
|
|
4492
|
+
const failure = new Error('oversized internal response');
|
|
4493
|
+
failure.diagnosticReason = 'capture_size_exceeded';
|
|
4494
|
+
failure.responseBytes = responseBytes;
|
|
4495
|
+
failure.responseLimitBytes = MAX_LINE_BYTES;
|
|
4496
|
+
fail(owned.threadId, 'oversized internal response', failure);
|
|
2548
4497
|
}
|
|
2549
4498
|
child.stdout.on('data', (chunk) => {
|
|
2550
4499
|
if (stopped) return;
|
|
@@ -2567,13 +4516,16 @@ export function createCodexContextProxy({
|
|
|
2567
4516
|
passthrough = false;
|
|
2568
4517
|
} else processLine(line);
|
|
2569
4518
|
}
|
|
2570
|
-
if (Buffer.byteLength(inputBuffer) > MAX_LINE_BYTES) {
|
|
4519
|
+
if (Buffer.byteLength(inputBuffer, 'utf8') + 1 > MAX_LINE_BYTES) {
|
|
2571
4520
|
if (passthrough) {
|
|
2572
4521
|
forward(inputBuffer);
|
|
2573
4522
|
inputBuffer = '';
|
|
2574
4523
|
} else if (internalRequestId(inputBuffer) !== null) {
|
|
2575
4524
|
stderr.write('codex-context: oversized internal response rejected\n');
|
|
2576
|
-
failOversizedInternal(
|
|
4525
|
+
failOversizedInternal(
|
|
4526
|
+
internalRequestId(inputBuffer),
|
|
4527
|
+
Buffer.byteLength(inputBuffer, 'utf8') + 1,
|
|
4528
|
+
);
|
|
2577
4529
|
discardOversized = true;
|
|
2578
4530
|
inputBuffer = '';
|
|
2579
4531
|
} else {
|
|
@@ -2610,6 +4562,8 @@ export function createCodexContextProxy({
|
|
|
2610
4562
|
}
|
|
2611
4563
|
return {
|
|
2612
4564
|
threads,
|
|
4565
|
+
observeClientRequest,
|
|
4566
|
+
observeClientResponse,
|
|
2613
4567
|
async stop() {
|
|
2614
4568
|
if (stopped) {
|
|
2615
4569
|
await Promise.all(
|
|
@@ -2626,6 +4580,8 @@ export function createCodexContextProxy({
|
|
|
2626
4580
|
void reporter.flush();
|
|
2627
4581
|
}
|
|
2628
4582
|
stopped = true;
|
|
4583
|
+
clientRequestBindings.clear();
|
|
4584
|
+
ownedThreadIds.clear();
|
|
2629
4585
|
failAll('adapter stopped');
|
|
2630
4586
|
await Promise.all(
|
|
2631
4587
|
[...diagnosticSinks.values()].map((sink) =>
|
|
@@ -2693,6 +4649,35 @@ async function main() {
|
|
|
2693
4649
|
policy,
|
|
2694
4650
|
createDiagnosticSink: diagnosticFactory,
|
|
2695
4651
|
});
|
|
4652
|
+
const clientDecoder = new StringDecoder('utf8');
|
|
4653
|
+
let clientInputBuffer = '';
|
|
4654
|
+
let discardOversizedClientFrame = false;
|
|
4655
|
+
process.stdin.on('data', (chunk) => {
|
|
4656
|
+
const decoded = clientDecoder.write(chunk);
|
|
4657
|
+
if (discardOversizedClientFrame) {
|
|
4658
|
+
const newline = decoded.indexOf('\n');
|
|
4659
|
+
if (newline < 0) return;
|
|
4660
|
+
discardOversizedClientFrame = false;
|
|
4661
|
+
clientInputBuffer = decoded.slice(newline + 1);
|
|
4662
|
+
} else clientInputBuffer += decoded;
|
|
4663
|
+
while (true) {
|
|
4664
|
+
const newline = clientInputBuffer.indexOf('\n');
|
|
4665
|
+
if (newline < 0) break;
|
|
4666
|
+
const line = clientInputBuffer.slice(0, newline).replace(/\r$/u, '');
|
|
4667
|
+
clientInputBuffer = clientInputBuffer.slice(newline + 1);
|
|
4668
|
+
if (!line) continue;
|
|
4669
|
+
if (Buffer.byteLength(line, 'utf8') + 1 > MAX_LINE_BYTES) continue;
|
|
4670
|
+
try {
|
|
4671
|
+
proxy.observeClientRequest(JSON.parse(line));
|
|
4672
|
+
} catch {
|
|
4673
|
+
/* Preserve malformed/native frames unchanged; the app-server owns validation. */
|
|
4674
|
+
}
|
|
4675
|
+
}
|
|
4676
|
+
if (Buffer.byteLength(clientInputBuffer, 'utf8') + 1 > MAX_LINE_BYTES) {
|
|
4677
|
+
clientInputBuffer = '';
|
|
4678
|
+
discardOversizedClientFrame = true;
|
|
4679
|
+
}
|
|
4680
|
+
});
|
|
2696
4681
|
process.stdin.pipe(child.stdin);
|
|
2697
4682
|
process.stdin.on('end', () => child.stdin.end());
|
|
2698
4683
|
child.on('exit', async (code, signal) => {
|