@brunobracaioli/aibrain 0.2.0-beta.1 → 0.3.0-beta.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +67 -4
- package/dist/assets/claude/.claude-plugin/plugin.json +49 -1
- package/dist/assets/claude/hooks/context-compaction.ts +6062 -734
- package/dist/assets/claude/hooks/provider-window.js +263 -0
- package/dist/assets/claude/hooks/state-fitting.d.mts +47 -0
- package/dist/assets/claude/hooks/state-fitting.mjs +324 -0
- package/dist/assets/codex-context-compaction.mjs +2690 -266
- package/dist/assets/context-selection-fitting.mjs +194 -0
- package/dist/assets/context-state-fitting.mjs +324 -0
- package/dist/assets/context-telemetry-reporter.mjs +305 -0
- package/dist/assets/jev-paid-pilot-terminal.mjs +591 -125
- package/dist/claude-checkpoint.mjs +1338 -0
- package/dist/claude-observation.mjs +1336 -0
- package/dist/cli.mjs +250 -20
- package/dist/codex-compatibility.mjs +467 -0
- package/dist/codex-diagnostics.mjs +426 -0
- package/dist/codex-permissions.mjs +590 -0
- package/dist/codex-protocol-fingerprint.json +131 -0
- package/dist/context-telemetry.mjs +257 -0
- package/dist/launch.mjs +341 -22
- package/dist/mcp.mjs +329 -7
- package/dist/monitor.mjs +11 -0
- package/dist/package.json +1 -1
- package/dist/platform.mjs +33 -3
- package/dist/state.mjs +107 -4
- package/dist/telemetry-report.mjs +133 -0
- package/package.json +1 -1
|
@@ -1,24 +1,125 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
chmod,
|
|
5
|
+
lstat,
|
|
6
|
+
mkdir,
|
|
7
|
+
open,
|
|
8
|
+
readFile,
|
|
9
|
+
readdir,
|
|
10
|
+
rename,
|
|
11
|
+
unlink,
|
|
12
|
+
writeFile,
|
|
13
|
+
} from 'node:fs/promises';
|
|
4
14
|
import { spawn } from 'node:child_process';
|
|
5
15
|
import { createHash, randomUUID } from 'node:crypto';
|
|
6
16
|
import { dirname, relative, resolve } from 'node:path';
|
|
7
17
|
import { StringDecoder } from 'node:string_decoder';
|
|
18
|
+
import { pathToFileURL } from 'node:url';
|
|
19
|
+
import { createTelemetryReporter, TELEMETRY_TIMEOUT_MS } from './context-telemetry-reporter.mjs';
|
|
20
|
+
import { preflightSelectionState } from './context-selection-fitting.mjs';
|
|
21
|
+
|
|
22
|
+
const captureFs = {
|
|
23
|
+
chmod,
|
|
24
|
+
lstat,
|
|
25
|
+
mkdir,
|
|
26
|
+
open,
|
|
27
|
+
readFile,
|
|
28
|
+
readdir,
|
|
29
|
+
rename,
|
|
30
|
+
unlink,
|
|
31
|
+
writeFile,
|
|
32
|
+
};
|
|
8
33
|
|
|
9
34
|
const DEFAULT_THRESHOLD = 85;
|
|
10
35
|
const COOLDOWN_MS = 60_000;
|
|
11
36
|
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
12
|
-
const MAX_LINE_BYTES =
|
|
13
|
-
const MAX_CAPTURE_BYTES =
|
|
37
|
+
const MAX_LINE_BYTES = 8 * 1024 * 1024;
|
|
38
|
+
const MAX_CAPTURE_BYTES = 8 * 1024 * 1024;
|
|
39
|
+
const MAX_CAPTURE_PAGE_BYTES = 8 * 1024 * 1024;
|
|
40
|
+
const MAX_CAPTURE_TOTAL_BYTES = 128 * 1024 * 1024;
|
|
41
|
+
const MAX_CAPTURE_PAGES = 32;
|
|
42
|
+
const MAX_CAPTURE_ITEMS = 65_536;
|
|
43
|
+
const MAX_CAPTURE_DIRECTORY_BYTES = 256 * 1024 * 1024;
|
|
44
|
+
const MAX_CAPTURE_STAGING_HEADROOM_BYTES = 16 * 1024 * 1024;
|
|
45
|
+
const MAX_CAPTURE_FILES = 128;
|
|
46
|
+
const CAPTURE_LOCK_WAIT_MS = 1_000;
|
|
14
47
|
const MAX_OWNED_REQUESTS = 1_000;
|
|
15
48
|
const MAX_RETIRED_REQUESTS = 2_000;
|
|
49
|
+
const MAX_CLIENT_REQUEST_BINDINGS = 128;
|
|
50
|
+
const MAX_OWNED_THREAD_IDS = 256;
|
|
16
51
|
const MAX_RECOVERY_ITEMS = 128;
|
|
52
|
+
const MAX_RECOVERY_SOURCE_INDEX = 65_536;
|
|
17
53
|
const MAX_NOTE_BYTES = 128 * 1024;
|
|
18
54
|
const MAX_SELECTION_BYTES = 512 * 1024;
|
|
19
55
|
const MAX_POLICY_BYTES = 8 * 1024;
|
|
20
56
|
const MAX_NOTE_ITEM_CHARS = 16_384;
|
|
21
57
|
const USAGE_FRESHNESS_MS = 60_000;
|
|
58
|
+
const MAX_DIAGNOSTIC_STRING = 128;
|
|
59
|
+
const MAX_DIAGNOSTIC_BYTES = MAX_CAPTURE_DIRECTORY_BYTES;
|
|
60
|
+
const MAX_BUDGET_SNAPSHOT_AGE_MS = 15_000;
|
|
61
|
+
const CONTEXT_ERROR_CODES = new Set([
|
|
62
|
+
'INVALID_INPUT',
|
|
63
|
+
'FEATURE_DISABLED',
|
|
64
|
+
'PAID_PLAN_REQUIRED',
|
|
65
|
+
'ENTITLEMENT_UNAVAILABLE',
|
|
66
|
+
'RATE_LIMITED',
|
|
67
|
+
'PROVIDER_NOT_CONFIGURED',
|
|
68
|
+
'BUDGET_EXCEEDED',
|
|
69
|
+
'PROVIDER_TIMEOUT',
|
|
70
|
+
'PROVIDER_UNAVAILABLE',
|
|
71
|
+
'INVALID_PROVIDER_RESPONSE',
|
|
72
|
+
]);
|
|
73
|
+
const CONTEXT_FAILURE_DETAILS = new Set(['state_budget_exceeded', 'shared_admission_denied']);
|
|
74
|
+
const CONTEXT_ADMISSION_REASONS = new Set([
|
|
75
|
+
'request_replayed',
|
|
76
|
+
'account_request_limit',
|
|
77
|
+
'account_token_limit',
|
|
78
|
+
'global_request_limit',
|
|
79
|
+
'global_token_limit',
|
|
80
|
+
'account_in_flight',
|
|
81
|
+
'database_unavailable',
|
|
82
|
+
]);
|
|
83
|
+
const SHARED_QUOTA_ADMISSION_REASONS = new Set([
|
|
84
|
+
'account_request_limit',
|
|
85
|
+
'account_token_limit',
|
|
86
|
+
'global_request_limit',
|
|
87
|
+
'global_token_limit',
|
|
88
|
+
'account_in_flight',
|
|
89
|
+
]);
|
|
90
|
+
const DIAGNOSTIC_STAGES = new Set([
|
|
91
|
+
'capture',
|
|
92
|
+
'budget',
|
|
93
|
+
'selection',
|
|
94
|
+
'notes',
|
|
95
|
+
'native',
|
|
96
|
+
'recovery',
|
|
97
|
+
'transport',
|
|
98
|
+
]);
|
|
99
|
+
const DIAGNOSTIC_REASONS = new Set([
|
|
100
|
+
'requested',
|
|
101
|
+
'completed',
|
|
102
|
+
'candidate_window_bounded',
|
|
103
|
+
'candidate_over_bound',
|
|
104
|
+
'protected_over_bound',
|
|
105
|
+
'history_unavailable',
|
|
106
|
+
'stale_snapshot',
|
|
107
|
+
'unavailable',
|
|
108
|
+
'admission_denied',
|
|
109
|
+
'invalid_response',
|
|
110
|
+
'request_failed',
|
|
111
|
+
'source_changed',
|
|
112
|
+
'timeout',
|
|
113
|
+
'thread_closed',
|
|
114
|
+
'protocol_error',
|
|
115
|
+
'capture_size_exceeded',
|
|
116
|
+
'capture_item_count_exceeded',
|
|
117
|
+
'private_storage_failed',
|
|
118
|
+
'storage_capacity_exceeded',
|
|
119
|
+
'attached',
|
|
120
|
+
'uncertain',
|
|
121
|
+
'disconnected',
|
|
122
|
+
]);
|
|
22
123
|
const FITTING_STAGES = new Set([
|
|
23
124
|
'original',
|
|
24
125
|
'inputs_1000',
|
|
@@ -233,10 +334,13 @@ function isValidUsage(value) {
|
|
|
233
334
|
isRecord(value) &&
|
|
234
335
|
isValidBreakdown(value.last) &&
|
|
235
336
|
isValidBreakdown(value.total) &&
|
|
236
|
-
|
|
237
|
-
|
|
337
|
+
(value.modelContextWindow === null ||
|
|
338
|
+
(isSafeCounter(value.modelContextWindow) && value.modelContextWindow > 0))
|
|
238
339
|
);
|
|
239
340
|
}
|
|
341
|
+
function hasKnownContextWindow(value) {
|
|
342
|
+
return isRecord(value) && isSafeCounter(value.modelContextWindow) && value.modelContextWindow > 0;
|
|
343
|
+
}
|
|
240
344
|
function redact(value, limit = 16_384) {
|
|
241
345
|
return String(value)
|
|
242
346
|
.replace(/(authorization\s*[:=]\s*bearer\s+)[^\s]+/gi, '$1[REDACTED]')
|
|
@@ -247,69 +351,978 @@ function redact(value, limit = 16_384) {
|
|
|
247
351
|
.slice(0, limit);
|
|
248
352
|
}
|
|
249
353
|
|
|
250
|
-
|
|
354
|
+
function isUuid(value) {
|
|
355
|
+
return (
|
|
356
|
+
typeof value === 'string' &&
|
|
357
|
+
/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu.test(value)
|
|
358
|
+
);
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
function boundedDiagnosticValue(value) {
|
|
362
|
+
return typeof value === 'string' && value.length > 0 && value.length <= MAX_DIAGNOSTIC_STRING
|
|
363
|
+
? value
|
|
364
|
+
: null;
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
function isSafeCorrelationId(value) {
|
|
368
|
+
return typeof value === 'string' && /^[A-Za-z0-9._:-]{1,128}$/u.test(value);
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
function boundedCorrelationValue(value) {
|
|
372
|
+
return typeof value === 'string' && value.length > 0 && value.length <= 256 ? value : null;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
function safeRpcCode(value) {
|
|
376
|
+
return Number.isSafeInteger(value) && value >= -32_768 && value <= 32_767 ? value : null;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
function clientRequestKey(value) {
|
|
380
|
+
if (typeof value === 'string' && value.length > 0 && value.length <= 128) return `s:${value}`;
|
|
381
|
+
if (Number.isSafeInteger(value)) return `n:${value}`;
|
|
382
|
+
return null;
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/**
|
|
386
|
+
* Closed private diagnostic contract consumed by the client sink. It contains
|
|
387
|
+
* stage/reason enums, pseudonymous session identity, operation/correlation
|
|
388
|
+
* identifiers and bounded counts only. It never accepts raw MCP/provider text,
|
|
389
|
+
* prompts, notes, paths or thread IDs.
|
|
390
|
+
*/
|
|
391
|
+
export function normalizeCodexDiagnostic(event, { threadId = null } = {}) {
|
|
392
|
+
if (!isRecord(event)) return null;
|
|
393
|
+
const stage = DIAGNOSTIC_STAGES.has(event.stage)
|
|
394
|
+
? event.stage
|
|
395
|
+
: DIAGNOSTIC_STAGES.has(event.phase)
|
|
396
|
+
? event.phase
|
|
397
|
+
: null;
|
|
398
|
+
const reason = DIAGNOSTIC_REASONS.has(event.reason)
|
|
399
|
+
? event.reason
|
|
400
|
+
: event.reasonCode === 'candidate_window_bounded'
|
|
401
|
+
? 'candidate_window_bounded'
|
|
402
|
+
: null;
|
|
403
|
+
if (stage === null || reason === null) return null;
|
|
404
|
+
const operationId = isUuid(event.operationId) ? event.operationId : undefined;
|
|
405
|
+
const correlationId = isSafeCorrelationId(event.correlationId) ? event.correlationId : null;
|
|
406
|
+
const rpcCode = safeRpcCode(event.rpcCode ?? event.rpc_code);
|
|
407
|
+
const counts = isRecord(event.counts)
|
|
408
|
+
? Object.fromEntries(
|
|
409
|
+
Object.entries(event.counts).filter(
|
|
410
|
+
([key, value]) =>
|
|
411
|
+
/^[a-z_]{1,48}$/u.test(key) &&
|
|
412
|
+
isSafeBoundedInteger(value, 0, /_bytes$/u.test(key) ? MAX_DIAGNOSTIC_BYTES : 1_000_000),
|
|
413
|
+
),
|
|
414
|
+
)
|
|
415
|
+
: undefined;
|
|
416
|
+
const result = {
|
|
417
|
+
schema_version: 1,
|
|
418
|
+
phase: boundedDiagnosticValue(event.phase) ?? `${stage}_${reason}`,
|
|
419
|
+
stage,
|
|
420
|
+
reason,
|
|
421
|
+
...(operationId === undefined ? {} : { operation_id: operationId }),
|
|
422
|
+
...(correlationId === null ? {} : { correlation_id: correlationId }),
|
|
423
|
+
...(rpcCode === null ? {} : { rpc_code: rpcCode }),
|
|
424
|
+
...(typeof threadId === 'string' && threadId.length > 0
|
|
425
|
+
? { session_key: hashText(threadId) }
|
|
426
|
+
: {}),
|
|
427
|
+
...(counts && Object.keys(counts).length > 0 ? { counts } : {}),
|
|
428
|
+
};
|
|
429
|
+
return result;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
function parseJsonText(value) {
|
|
433
|
+
if (typeof value !== 'string' || Buffer.byteLength(value, 'utf8') > MAX_SELECTION_BYTES)
|
|
434
|
+
return null;
|
|
435
|
+
try {
|
|
436
|
+
const parsed = JSON.parse(value);
|
|
437
|
+
return isRecord(parsed) ? parsed : null;
|
|
438
|
+
} catch {
|
|
439
|
+
return null;
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
function responseStructuredContent(response) {
|
|
444
|
+
if (!isRecord(response)) return null;
|
|
445
|
+
if (isRecord(response.structuredContent)) return response.structuredContent;
|
|
446
|
+
const content = Array.isArray(response.content) ? response.content : [];
|
|
447
|
+
const textPart = content.find((part) => isRecord(part) && part.type === 'text');
|
|
448
|
+
return textPart ? parseJsonText(textPart.text) : null;
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
/** Parse only the closed Context error envelope or known fixed-code text form. */
|
|
452
|
+
export function parseContextError(response) {
|
|
453
|
+
const structured = responseStructuredContent(response);
|
|
454
|
+
let value = structured;
|
|
455
|
+
if (value === null) {
|
|
456
|
+
const content = Array.isArray(response?.content) ? response.content : [];
|
|
457
|
+
const textPart = content.find((part) => isRecord(part) && part.type === 'text');
|
|
458
|
+
const text = typeof textPart?.text === 'string' ? textPart.text : '';
|
|
459
|
+
const fixed =
|
|
460
|
+
/^(INVALID_INPUT|FEATURE_DISABLED|PAID_PLAN_REQUIRED|ENTITLEMENT_UNAVAILABLE|RATE_LIMITED|PROVIDER_NOT_CONFIGURED|BUDGET_EXCEEDED|PROVIDER_TIMEOUT|PROVIDER_UNAVAILABLE|INVALID_PROVIDER_RESPONSE)(?::|$)/u.exec(
|
|
461
|
+
text,
|
|
462
|
+
);
|
|
463
|
+
if (!fixed) return null;
|
|
464
|
+
value = { error_code: fixed[1] };
|
|
465
|
+
}
|
|
466
|
+
if (
|
|
467
|
+
!CONTEXT_ERROR_CODES.has(value.error_code) ||
|
|
468
|
+
(value.failure_detail !== undefined && !CONTEXT_FAILURE_DETAILS.has(value.failure_detail)) ||
|
|
469
|
+
(value.admission_reason !== undefined &&
|
|
470
|
+
!CONTEXT_ADMISSION_REASONS.has(value.admission_reason)) ||
|
|
471
|
+
(value.correlation_id !== undefined && !isSafeCorrelationId(value.correlation_id)) ||
|
|
472
|
+
(value.request_id !== undefined && !isUuid(value.request_id)) ||
|
|
473
|
+
(value.retry_after_ms !== undefined &&
|
|
474
|
+
!isSafeBoundedInteger(value.retry_after_ms, 0, 3_600_000))
|
|
475
|
+
)
|
|
476
|
+
return null;
|
|
477
|
+
return {
|
|
478
|
+
errorCode: value.error_code,
|
|
479
|
+
failureDetail: value.failure_detail ?? null,
|
|
480
|
+
admissionReason: value.admission_reason ?? null,
|
|
481
|
+
correlationId: boundedCorrelationValue(value.correlation_id),
|
|
482
|
+
requestId: isUuid(value.request_id) ? value.request_id : null,
|
|
483
|
+
retryAfterMs: value.retry_after_ms ?? null,
|
|
484
|
+
quota:
|
|
485
|
+
value.error_code === 'RATE_LIMITED' ||
|
|
486
|
+
value.failure_detail === 'shared_admission_denied' ||
|
|
487
|
+
SHARED_QUOTA_ADMISSION_REASONS.has(value.admission_reason),
|
|
488
|
+
};
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
export function parseBudgetSnapshot(value) {
|
|
492
|
+
if (
|
|
493
|
+
!isRecord(value) ||
|
|
494
|
+
!hasOnlyKeys(value, [
|
|
495
|
+
'schema_version',
|
|
496
|
+
'server_time',
|
|
497
|
+
'expires_at',
|
|
498
|
+
'window_end',
|
|
499
|
+
'available_requests',
|
|
500
|
+
'available_tokens',
|
|
501
|
+
'output_token_allowance',
|
|
502
|
+
'max_batches_per_operation',
|
|
503
|
+
'account_in_flight',
|
|
504
|
+
'retry_after_ms',
|
|
505
|
+
'provider_batches',
|
|
506
|
+
'provider_model',
|
|
507
|
+
'correlation_id',
|
|
508
|
+
]) ||
|
|
509
|
+
value.schema_version !== 1 ||
|
|
510
|
+
!isTimestamp(value.server_time) ||
|
|
511
|
+
!isTimestamp(value.expires_at) ||
|
|
512
|
+
!isTimestamp(value.window_end) ||
|
|
513
|
+
Date.parse(value.expires_at) <= Date.parse(value.server_time) ||
|
|
514
|
+
Date.parse(value.expires_at) - Date.parse(value.server_time) > MAX_BUDGET_SNAPSHOT_AGE_MS ||
|
|
515
|
+
Date.parse(value.expires_at) > Date.parse(value.window_end) ||
|
|
516
|
+
!isSafeBoundedInteger(value.available_requests, 0, 1_000_000) ||
|
|
517
|
+
!isSafeBoundedInteger(value.available_tokens, 0, Number.MAX_SAFE_INTEGER) ||
|
|
518
|
+
!isSafeBoundedInteger(value.output_token_allowance, 0, Number.MAX_SAFE_INTEGER) ||
|
|
519
|
+
!isSafeBoundedInteger(value.max_batches_per_operation, 0, 4) ||
|
|
520
|
+
typeof value.account_in_flight !== 'boolean' ||
|
|
521
|
+
!isSafeBoundedInteger(value.retry_after_ms, 0, 3_600_000) ||
|
|
522
|
+
value.provider_batches !== 0 ||
|
|
523
|
+
value.provider_model !== null ||
|
|
524
|
+
!isSafeCorrelationId(value.correlation_id)
|
|
525
|
+
)
|
|
526
|
+
return null;
|
|
527
|
+
return value;
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
function budgetSnapshotFresh(snapshot, receivedAt, currentTime) {
|
|
531
|
+
if (!isRecord(snapshot) || !Number.isSafeInteger(receivedAt) || currentTime < receivedAt)
|
|
532
|
+
return false;
|
|
533
|
+
const lifetime = Date.parse(snapshot.expires_at) - Date.parse(snapshot.server_time);
|
|
534
|
+
return (
|
|
535
|
+
lifetime > 0 && lifetime <= MAX_BUDGET_SNAPSHOT_AGE_MS && currentTime - receivedAt <= lifetime
|
|
536
|
+
);
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
function diagnosticReasonForError(error) {
|
|
540
|
+
if (DIAGNOSTIC_REASONS.has(error?.diagnosticReason)) return error.diagnosticReason;
|
|
541
|
+
if (safeRpcCode(error?.rpcCode) !== null) return 'protocol_error';
|
|
542
|
+
const message = error instanceof Error ? error.message : String(error ?? '');
|
|
543
|
+
if (
|
|
544
|
+
/snapshot exceeds bound|capture exceeds|capture size|oversized internal response/iu.test(
|
|
545
|
+
message,
|
|
546
|
+
)
|
|
547
|
+
)
|
|
548
|
+
return 'capture_size_exceeded';
|
|
549
|
+
if (/capture item count|source index exceeds bound/iu.test(message))
|
|
550
|
+
return 'capture_item_count_exceeded';
|
|
551
|
+
if (/capture storage capacity|storage capacity|capture directory/iu.test(message))
|
|
552
|
+
return 'storage_capacity_exceeded';
|
|
553
|
+
if (/capture source identity validation/iu.test(message)) return 'source_changed';
|
|
554
|
+
if (/capture candidate|checkpoint target/iu.test(message)) return 'private_storage_failed';
|
|
555
|
+
if (/thread\/(?:read|turns\/list)|capture history|page unavailable/iu.test(message))
|
|
556
|
+
return 'history_unavailable';
|
|
557
|
+
if (/checkpoint|recovery notes|private (?:file|storage)|storage|EACCES|ENOSPC/iu.test(message))
|
|
558
|
+
return 'private_storage_failed';
|
|
559
|
+
if (/tim(?:e|ed)[ -]?out/iu.test(message)) return 'timeout';
|
|
560
|
+
if (/closed|deleted/iu.test(message)) return 'thread_closed';
|
|
561
|
+
if (/source changed|state changed|stale/iu.test(message)) return 'source_changed';
|
|
562
|
+
if (/candidate|history|thread\/read/iu.test(message)) return 'history_unavailable';
|
|
563
|
+
return 'request_failed';
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
function rpcFailure(kind, error) {
|
|
567
|
+
const failure = new Error(`${kind} request rejected by the app-server`);
|
|
568
|
+
failure.name = 'CodexProtocolError';
|
|
569
|
+
failure.diagnosticReason = 'protocol_error';
|
|
570
|
+
failure.rpcCode = safeRpcCode(error?.code);
|
|
571
|
+
return failure;
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
function boundedFailureCounts(error) {
|
|
575
|
+
if (!isRecord(error)) return undefined;
|
|
576
|
+
const counts = {};
|
|
577
|
+
for (const [source, target] of [
|
|
578
|
+
['captureItemCount', 'capture_item_count'],
|
|
579
|
+
['captureItemLimit', 'capture_item_limit'],
|
|
580
|
+
['captureBytes', 'capture_bytes'],
|
|
581
|
+
['captureLimitBytes', 'capture_limit_bytes'],
|
|
582
|
+
['responseBytes', 'response_bytes'],
|
|
583
|
+
['responseLimitBytes', 'response_limit_bytes'],
|
|
584
|
+
['captureDirectoryBytes', 'capture_directory_bytes'],
|
|
585
|
+
['captureDirectoryLimitBytes', 'capture_directory_limit_bytes'],
|
|
586
|
+
['captureFileCount', 'capture_file_count'],
|
|
587
|
+
['captureFileLimit', 'capture_file_limit'],
|
|
588
|
+
]) {
|
|
589
|
+
const maximum = target.includes('directory_bytes')
|
|
590
|
+
? MAX_CAPTURE_DIRECTORY_BYTES
|
|
591
|
+
: target.includes('bytes')
|
|
592
|
+
? MAX_DIAGNOSTIC_BYTES
|
|
593
|
+
: 1_000_000;
|
|
594
|
+
if (isSafeBoundedInteger(error[source], 0, maximum)) counts[target] = error[source];
|
|
595
|
+
}
|
|
596
|
+
return Object.keys(counts).length > 0 ? counts : undefined;
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
async function ensurePrivateDirectory(directory, fsOps = captureFs) {
|
|
251
600
|
const target = resolve(directory);
|
|
252
|
-
await mkdir(target, { recursive: true, mode: 0o700 });
|
|
601
|
+
await fsOps.mkdir(target, { recursive: true, mode: 0o700 });
|
|
253
602
|
const parts = target.split('/');
|
|
254
603
|
let current = parts[0] === '' ? '/' : parts[0];
|
|
255
604
|
for (const part of parts.slice(parts[0] === '' ? 1 : 1)) {
|
|
256
605
|
if (!part) continue;
|
|
257
606
|
current = current === '/' ? `/${part}` : `${current}/${part}`;
|
|
258
|
-
const info = await lstat(current);
|
|
607
|
+
const info = await fsOps.lstat(current);
|
|
259
608
|
if (info.isSymbolicLink() || !info.isDirectory())
|
|
260
609
|
throw new Error('checkpoint directory is unsafe');
|
|
261
610
|
}
|
|
262
|
-
await chmod(target, 0o700);
|
|
611
|
+
await fsOps.chmod(target, 0o700);
|
|
263
612
|
return target;
|
|
264
613
|
}
|
|
265
|
-
async function assertPrivateFile(file, label) {
|
|
266
|
-
const info = await lstat(file);
|
|
614
|
+
async function assertPrivateFile(file, label, fsOps = captureFs) {
|
|
615
|
+
const info = await fsOps.lstat(file);
|
|
267
616
|
if (info.isSymbolicLink() || !info.isFile() || (info.mode & 0o077) !== 0)
|
|
268
617
|
throw new Error(`${label} is not a private regular file`);
|
|
269
618
|
return info;
|
|
270
619
|
}
|
|
271
|
-
async function readPrivateFile(file, label, maxBytes = null) {
|
|
272
|
-
const info = await assertPrivateFile(file, label);
|
|
620
|
+
async function readPrivateFile(file, label, maxBytes = null, fsOps = captureFs) {
|
|
621
|
+
const info = await assertPrivateFile(file, label, fsOps);
|
|
273
622
|
if (maxBytes !== null && info.size > maxBytes) throw new Error(`${label} exceeds its bound`);
|
|
274
|
-
return readFile(file, 'utf8');
|
|
623
|
+
return fsOps.readFile(file, 'utf8');
|
|
275
624
|
}
|
|
276
625
|
|
|
277
|
-
|
|
278
|
-
const
|
|
626
|
+
function captureCapacityError(state) {
|
|
627
|
+
const error = new Error('capture storage capacity exceeded');
|
|
628
|
+
error.diagnosticReason = 'storage_capacity_exceeded';
|
|
629
|
+
error.captureDirectoryBytes = state?.totalBytes ?? MAX_CAPTURE_DIRECTORY_BYTES;
|
|
630
|
+
error.captureDirectoryLimitBytes = MAX_CAPTURE_DIRECTORY_BYTES;
|
|
631
|
+
error.captureFileCount = state?.managedFiles?.length ?? MAX_CAPTURE_FILES;
|
|
632
|
+
error.captureFileLimit = MAX_CAPTURE_FILES;
|
|
633
|
+
return error;
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
function validateCaptureSnapshot(value, expectedSourceRevision = null) {
|
|
637
|
+
if (
|
|
638
|
+
!isRecord(value) ||
|
|
639
|
+
![1, 2].includes(value.version) ||
|
|
640
|
+
typeof value.threadId !== 'string' ||
|
|
641
|
+
typeof value.sourceRevision !== 'string' ||
|
|
642
|
+
!/^[a-f0-9]{64}$/u.test(value.sourceRevision) ||
|
|
643
|
+
!isRecord(value.threadRead) ||
|
|
644
|
+
value.threadRead.thread?.id !== value.threadId ||
|
|
645
|
+
(value.version === 1 && hashJson(value.threadRead) !== value.sourceRevision) ||
|
|
646
|
+
(value.version === 2 &&
|
|
647
|
+
(!isRecord(value.captureManifest) ||
|
|
648
|
+
value.captureManifest.version !== 'codex-capture-manifest-v1' ||
|
|
649
|
+
value.captureManifest.sourceRevision !== value.sourceRevision ||
|
|
650
|
+
value.captureManifest.threadId !== value.threadId ||
|
|
651
|
+
!isSafeBoundedInteger(value.captureManifest.pageCount, 1, MAX_CAPTURE_PAGES) ||
|
|
652
|
+
!isSafeBoundedInteger(value.captureManifest.itemCount, 0, MAX_CAPTURE_ITEMS) ||
|
|
653
|
+
!isSafeBoundedInteger(value.captureManifest.totalBytes, 1, MAX_CAPTURE_TOTAL_BYTES) ||
|
|
654
|
+
!Array.isArray(value.captureManifest.segments) ||
|
|
655
|
+
value.captureManifest.segments.length !== value.captureManifest.pageCount ||
|
|
656
|
+
!Array.isArray(value.captureManifest.pageHashes) ||
|
|
657
|
+
value.captureManifest.pageHashes.length !== value.captureManifest.pageCount ||
|
|
658
|
+
value.captureManifest.pageHashes.some((hash) => !/^[a-f0-9]{64}$/u.test(hash)) ||
|
|
659
|
+
captureManifestRevision(value.threadRead.thread, value.captureManifest.pageHashes) !==
|
|
660
|
+
value.sourceRevision ||
|
|
661
|
+
value.captureManifest.segments.some(
|
|
662
|
+
(segment, index) =>
|
|
663
|
+
segment !== captureSegmentName(value.threadId, value.sourceRevision, index),
|
|
664
|
+
))) ||
|
|
665
|
+
(expectedSourceRevision !== null && value.sourceRevision !== expectedSourceRevision)
|
|
666
|
+
)
|
|
667
|
+
throw new Error('capture source identity validation failed');
|
|
668
|
+
strictThreadName(value.threadId);
|
|
669
|
+
return value;
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
function validateCaptureSegment(value, expectedThreadId = null, expectedSourceRevision = null) {
|
|
673
|
+
if (
|
|
674
|
+
!isRecord(value) ||
|
|
675
|
+
value.version !== 'codex-capture-segment-v1' ||
|
|
676
|
+
typeof value.threadId !== 'string' ||
|
|
677
|
+
(expectedThreadId !== null && value.threadId !== expectedThreadId) ||
|
|
678
|
+
typeof value.sourceRevision !== 'string' ||
|
|
679
|
+
!/^[a-f0-9]{64}$/u.test(value.sourceRevision) ||
|
|
680
|
+
(expectedSourceRevision !== null && value.sourceRevision !== expectedSourceRevision) ||
|
|
681
|
+
!isSafeBoundedInteger(value.pageIndex, 0, MAX_CAPTURE_PAGES - 1) ||
|
|
682
|
+
!isRecord(value.page) ||
|
|
683
|
+
value.page.threadId !== value.threadId ||
|
|
684
|
+
!Array.isArray(value.page.turns) ||
|
|
685
|
+
typeof value.pageHash !== 'string' ||
|
|
686
|
+
!/^[a-f0-9]{64}$/u.test(value.pageHash) ||
|
|
687
|
+
hashJson(value.page) !== value.pageHash ||
|
|
688
|
+
!isSafeBoundedInteger(value.itemCount, 0, MAX_CAPTURE_ITEMS)
|
|
689
|
+
)
|
|
690
|
+
throw new Error('capture segment identity validation failed');
|
|
691
|
+
return value;
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
function captureSegmentName(threadId, sourceRevision, pageIndex) {
|
|
695
|
+
return `${strictThreadName(threadId)}.${sourceRevision}.segment-${String(pageIndex).padStart(4, '0')}.json`;
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
function captureManifestRevision(metadataThread, pageHashes) {
|
|
699
|
+
const thread = Object.fromEntries(
|
|
700
|
+
Object.entries(metadataThread).filter(([key]) => key !== 'turns'),
|
|
701
|
+
);
|
|
702
|
+
return hashJson({ thread, pages: pageHashes });
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
function captureNameForSnapshot(value, suffix = '') {
|
|
706
|
+
return `${strictThreadName(value.threadId)}${suffix}.json`;
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
async function readCaptureDirectoryState(directory, fsOps = captureFs) {
|
|
710
|
+
const entries = await fsOps.readdir(directory, { withFileTypes: true });
|
|
711
|
+
if (entries.length > MAX_CAPTURE_FILES * 4)
|
|
712
|
+
throw captureCapacityError({ totalBytes: MAX_CAPTURE_DIRECTORY_BYTES, managedFiles: [] });
|
|
713
|
+
const state = {
|
|
714
|
+
entryCount: entries.length,
|
|
715
|
+
totalBytes: 0,
|
|
716
|
+
managedFiles: [],
|
|
717
|
+
referencedImmutableNames: new Set(),
|
|
718
|
+
protectedGroups: new Set(),
|
|
719
|
+
manifestRefs: [],
|
|
720
|
+
blockedEntries: false,
|
|
721
|
+
unsafeRecovery: false,
|
|
722
|
+
};
|
|
723
|
+
for (const entry of entries) {
|
|
724
|
+
if (entry.name === '.capture.lock') continue;
|
|
725
|
+
const file = resolve(directory, entry.name);
|
|
726
|
+
const info = await fsOps.lstat(file);
|
|
727
|
+
state.totalBytes += info.size;
|
|
728
|
+
if (
|
|
729
|
+
info.isSymbolicLink() ||
|
|
730
|
+
!info.isFile() ||
|
|
731
|
+
(typeof process.getuid === 'function' && info.uid !== process.getuid()) ||
|
|
732
|
+
(info.mode & 0o077) !== 0
|
|
733
|
+
) {
|
|
734
|
+
state.blockedEntries = true;
|
|
735
|
+
continue;
|
|
736
|
+
}
|
|
737
|
+
if (!entry.name.endsWith('.json')) continue;
|
|
738
|
+
let value;
|
|
739
|
+
try {
|
|
740
|
+
if (info.size > MAX_CAPTURE_BYTES) {
|
|
741
|
+
if (entry.name.endsWith('.recovery.json')) state.unsafeRecovery = true;
|
|
742
|
+
continue;
|
|
743
|
+
}
|
|
744
|
+
value = JSON.parse(await fsOps.readFile(file, 'utf8'));
|
|
745
|
+
} catch {
|
|
746
|
+
if (entry.name.endsWith('.recovery.json')) state.unsafeRecovery = true;
|
|
747
|
+
continue;
|
|
748
|
+
}
|
|
749
|
+
if (entry.name.endsWith('.recovery.json')) {
|
|
750
|
+
if (
|
|
751
|
+
isRecord(value) &&
|
|
752
|
+
typeof value.threadId === 'string' &&
|
|
753
|
+
/^[a-f0-9]{64}$/u.test(value.sourceRevision) &&
|
|
754
|
+
typeof value.notes === 'string' &&
|
|
755
|
+
typeof value.notesHash === 'string' &&
|
|
756
|
+
value.notesHash === hashText(value.notes)
|
|
757
|
+
) {
|
|
758
|
+
try {
|
|
759
|
+
state.referencedImmutableNames.add(
|
|
760
|
+
`${strictThreadName(value.threadId)}.${value.sourceRevision}.json`,
|
|
761
|
+
);
|
|
762
|
+
state.protectedGroups.add(`${strictThreadName(value.threadId)}.${value.sourceRevision}`);
|
|
763
|
+
} catch {
|
|
764
|
+
state.unsafeRecovery = true;
|
|
765
|
+
}
|
|
766
|
+
} else state.unsafeRecovery = true;
|
|
767
|
+
if (isRecord(value) && typeof value.notes === 'string') {
|
|
768
|
+
const references = [
|
|
769
|
+
...value.notes.matchAll(/capture_file:\s+.*?([A-Za-z0-9._-]+\.[a-f0-9]{64}\.json)/gu),
|
|
770
|
+
];
|
|
771
|
+
for (const match of references) state.referencedImmutableNames.add(match[1]);
|
|
772
|
+
}
|
|
773
|
+
continue;
|
|
774
|
+
}
|
|
775
|
+
if (isRecord(value) && value.version === 'codex-capture-segment-v1') {
|
|
776
|
+
try {
|
|
777
|
+
validateCaptureSegment(value);
|
|
778
|
+
const expectedName = captureSegmentName(
|
|
779
|
+
value.threadId,
|
|
780
|
+
value.sourceRevision,
|
|
781
|
+
value.pageIndex,
|
|
782
|
+
);
|
|
783
|
+
if (entry.name !== expectedName) continue;
|
|
784
|
+
state.managedFiles.push({
|
|
785
|
+
file,
|
|
786
|
+
name: entry.name,
|
|
787
|
+
size: info.size,
|
|
788
|
+
mtimeMs: info.mtimeMs,
|
|
789
|
+
immutable: true,
|
|
790
|
+
group: `${value.threadId}.${value.sourceRevision}`,
|
|
791
|
+
});
|
|
792
|
+
} catch {
|
|
793
|
+
state.blockedEntries = true;
|
|
794
|
+
}
|
|
795
|
+
continue;
|
|
796
|
+
}
|
|
797
|
+
try {
|
|
798
|
+
validateCaptureSnapshot(value);
|
|
799
|
+
} catch {
|
|
800
|
+
continue;
|
|
801
|
+
}
|
|
802
|
+
const sourceName = `${strictThreadName(value.threadId)}.${value.sourceRevision}.json`;
|
|
803
|
+
if (
|
|
804
|
+
!new Set([
|
|
805
|
+
captureNameForSnapshot(value),
|
|
806
|
+
captureNameForSnapshot(value, '.completed'),
|
|
807
|
+
sourceName,
|
|
808
|
+
]).has(entry.name)
|
|
809
|
+
)
|
|
810
|
+
continue;
|
|
811
|
+
state.managedFiles.push({
|
|
812
|
+
file,
|
|
813
|
+
name: entry.name,
|
|
814
|
+
size: info.size,
|
|
815
|
+
mtimeMs: info.mtimeMs,
|
|
816
|
+
immutable: entry.name === sourceName,
|
|
817
|
+
group: `${value.threadId}.${value.sourceRevision}`,
|
|
818
|
+
});
|
|
819
|
+
if (!entry.name.endsWith('.' + value.sourceRevision + '.json'))
|
|
820
|
+
state.referencedImmutableNames.add(sourceName);
|
|
821
|
+
if (value.version === 2 && entry.name !== sourceName) {
|
|
822
|
+
for (const segment of value.captureManifest.segments) {
|
|
823
|
+
if (typeof segment === 'string') state.referencedImmutableNames.add(segment);
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
if (value.version === 2) state.manifestRefs.push(value.captureManifest);
|
|
827
|
+
if (
|
|
828
|
+
value.version === 2 &&
|
|
829
|
+
state.protectedGroups.has(`${value.threadId}.${value.sourceRevision}`)
|
|
830
|
+
) {
|
|
831
|
+
for (const segment of value.captureManifest.segments)
|
|
832
|
+
state.referencedImmutableNames.add(segment);
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
const managedNames = new Set(state.managedFiles.map((entry) => entry.name));
|
|
836
|
+
for (const manifest of state.manifestRefs) {
|
|
837
|
+
if (manifest.segments.some((segment) => !managedNames.has(segment)))
|
|
838
|
+
state.blockedEntries = true;
|
|
839
|
+
}
|
|
840
|
+
for (const entry of state.managedFiles) {
|
|
841
|
+
if (entry.group && state.referencedImmutableNames.has(entry.name))
|
|
842
|
+
state.protectedGroups.add(entry.group);
|
|
843
|
+
}
|
|
844
|
+
return state;
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
async function withCaptureStoreLock(
|
|
848
|
+
directory,
|
|
849
|
+
task,
|
|
850
|
+
fsOps = captureFs,
|
|
851
|
+
{
|
|
852
|
+
now = Date.now,
|
|
853
|
+
sleep = (milliseconds) =>
|
|
854
|
+
new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds)),
|
|
855
|
+
} = {},
|
|
856
|
+
) {
|
|
857
|
+
const lockPath = resolve(directory, '.capture.lock');
|
|
858
|
+
const started = now();
|
|
859
|
+
while (true) {
|
|
860
|
+
let handle;
|
|
861
|
+
try {
|
|
862
|
+
handle = await fsOps.open(lockPath, 'wx', 0o600);
|
|
863
|
+
try {
|
|
864
|
+
return await task();
|
|
865
|
+
} finally {
|
|
866
|
+
await handle.close();
|
|
867
|
+
await fsOps.unlink(lockPath).catch(() => undefined);
|
|
868
|
+
}
|
|
869
|
+
} catch (error) {
|
|
870
|
+
if (handle) throw error;
|
|
871
|
+
if (error?.code !== 'EEXIST') throw error;
|
|
872
|
+
if (now() - started >= CAPTURE_LOCK_WAIT_MS)
|
|
873
|
+
throw captureCapacityError({ totalBytes: 0, managedFiles: [] });
|
|
874
|
+
await sleep(10);
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
async function writePrivateFile(file, serialized, fsOps = captureFs) {
|
|
880
|
+
const directory = await ensurePrivateDirectory(dirname(file), fsOps);
|
|
279
881
|
const target = resolve(
|
|
280
882
|
directory,
|
|
281
883
|
file.startsWith(directory) ? file.slice(directory.length + 1) : file,
|
|
282
884
|
);
|
|
283
885
|
try {
|
|
284
|
-
await assertPrivateFile(target, 'checkpoint target');
|
|
886
|
+
await assertPrivateFile(target, 'checkpoint target', fsOps);
|
|
285
887
|
} catch (error) {
|
|
286
888
|
if (error?.code !== 'ENOENT') throw error;
|
|
287
889
|
}
|
|
288
890
|
const temporary = `${target}.${process.pid}.${randomUUID()}.tmp`;
|
|
289
891
|
try {
|
|
290
|
-
await writeFile(temporary,
|
|
291
|
-
await rename(temporary, target);
|
|
892
|
+
await fsOps.writeFile(temporary, serialized, { flag: 'wx', mode: 0o600 });
|
|
893
|
+
await fsOps.rename(temporary, target);
|
|
292
894
|
} catch (error) {
|
|
895
|
+
await fsOps.unlink(temporary).catch(() => undefined);
|
|
896
|
+
throw error;
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
async function writeBoundedCapture(file, serialized, value, fsOps = captureFs, lockOptions = {}) {
|
|
901
|
+
const directory = await ensurePrivateDirectory(dirname(file), fsOps);
|
|
902
|
+
const target = resolve(
|
|
903
|
+
directory,
|
|
904
|
+
file.startsWith(directory) ? file.slice(directory.length + 1) : file,
|
|
905
|
+
);
|
|
906
|
+
const targetName = target.slice(directory.length + 1);
|
|
907
|
+
const operation = async () => {
|
|
908
|
+
const temporary = `${target}.${process.pid}.${randomUUID()}.tmp`;
|
|
909
|
+
const sourceName = `${strictThreadName(value.threadId)}.${value.sourceRevision}.json`;
|
|
910
|
+
const expectedNames = new Set([
|
|
911
|
+
captureNameForSnapshot(value),
|
|
912
|
+
captureNameForSnapshot(value, '.completed'),
|
|
913
|
+
sourceName,
|
|
914
|
+
]);
|
|
915
|
+
let existingTarget = null;
|
|
293
916
|
try {
|
|
294
|
-
await
|
|
295
|
-
|
|
296
|
-
|
|
917
|
+
existingTarget = await assertPrivateFile(target, 'capture target', fsOps);
|
|
918
|
+
const existing = JSON.parse(
|
|
919
|
+
await readPrivateFile(target, 'capture target', MAX_CAPTURE_BYTES, fsOps),
|
|
920
|
+
);
|
|
921
|
+
validateCaptureSnapshot(existing);
|
|
922
|
+
if (!expectedNames.has(targetName) || existing.threadId !== value.threadId)
|
|
923
|
+
throw new Error('capture target identity validation failed');
|
|
924
|
+
if (targetName === sourceName && existing.sourceRevision === value.sourceRevision) return;
|
|
925
|
+
} catch (error) {
|
|
926
|
+
if (error?.code !== 'ENOENT') throw error;
|
|
297
927
|
}
|
|
928
|
+
try {
|
|
929
|
+
await fsOps.writeFile(temporary, serialized, { flag: 'wx', mode: 0o600 });
|
|
930
|
+
const written = await assertPrivateFile(temporary, 'capture candidate', fsOps);
|
|
931
|
+
if (written.size !== Buffer.byteLength(serialized, 'utf8'))
|
|
932
|
+
throw new Error('capture candidate read-back size mismatch');
|
|
933
|
+
validateCaptureSnapshot(
|
|
934
|
+
JSON.parse(await fsOps.readFile(temporary, 'utf8')),
|
|
935
|
+
value.sourceRevision,
|
|
936
|
+
);
|
|
937
|
+
const state = await readCaptureDirectoryState(directory, fsOps);
|
|
938
|
+
if (state.unsafeRecovery || state.blockedEntries) throw captureCapacityError(state);
|
|
939
|
+
const protectedNames = new Set([targetName, sourceName]);
|
|
940
|
+
if (value.version === 2 && Array.isArray(value.captureManifest?.segments)) {
|
|
941
|
+
for (const segment of value.captureManifest.segments) protectedNames.add(segment);
|
|
942
|
+
}
|
|
943
|
+
const targetWasManaged = state.managedFiles.some((entry) => entry.name === targetName);
|
|
944
|
+
const eligibleGroups = new Map();
|
|
945
|
+
if (!state.blockedEntries) {
|
|
946
|
+
for (const entry of state.managedFiles) {
|
|
947
|
+
if (
|
|
948
|
+
!entry.immutable ||
|
|
949
|
+
protectedNames.has(entry.name) ||
|
|
950
|
+
state.referencedImmutableNames.has(entry.name) ||
|
|
951
|
+
state.protectedGroups.has(entry.group)
|
|
952
|
+
)
|
|
953
|
+
continue;
|
|
954
|
+
const group = entry.group ?? entry.name;
|
|
955
|
+
const members = eligibleGroups.get(group) ?? [];
|
|
956
|
+
members.push(entry);
|
|
957
|
+
eligibleGroups.set(group, members);
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
const eligible = [...eligibleGroups.values()].sort(
|
|
961
|
+
(left, right) =>
|
|
962
|
+
Math.min(...left.map((entry) => entry.mtimeMs)) -
|
|
963
|
+
Math.min(...right.map((entry) => entry.mtimeMs)),
|
|
964
|
+
);
|
|
965
|
+
const targetEntry = state.managedFiles.find((entry) => entry.name === targetName);
|
|
966
|
+
const replacedBytes = targetEntry?.size ?? 0;
|
|
967
|
+
const temporaryHeadroom = 2 * MAX_CAPTURE_BYTES;
|
|
968
|
+
let projectedPeakBytes = state.totalBytes;
|
|
969
|
+
let projectedSteadyBytes = state.totalBytes - replacedBytes;
|
|
970
|
+
let projectedFiles = state.managedFiles.length + (targetWasManaged ? 0 : 1);
|
|
971
|
+
const retire = [];
|
|
972
|
+
for (const group of eligible) {
|
|
973
|
+
if (
|
|
974
|
+
projectedPeakBytes <= MAX_CAPTURE_DIRECTORY_BYTES + temporaryHeadroom &&
|
|
975
|
+
projectedSteadyBytes <= MAX_CAPTURE_DIRECTORY_BYTES &&
|
|
976
|
+
projectedFiles <= MAX_CAPTURE_FILES
|
|
977
|
+
)
|
|
978
|
+
break;
|
|
979
|
+
const groupBytes = group.reduce((sum, entry) => sum + entry.size, 0);
|
|
980
|
+
projectedPeakBytes -= groupBytes;
|
|
981
|
+
projectedSteadyBytes -= groupBytes;
|
|
982
|
+
projectedFiles -= group.length;
|
|
983
|
+
retire.push(...group);
|
|
984
|
+
}
|
|
985
|
+
if (
|
|
986
|
+
projectedPeakBytes > MAX_CAPTURE_DIRECTORY_BYTES + temporaryHeadroom ||
|
|
987
|
+
projectedSteadyBytes > MAX_CAPTURE_DIRECTORY_BYTES ||
|
|
988
|
+
projectedFiles > MAX_CAPTURE_FILES
|
|
989
|
+
)
|
|
990
|
+
throw captureCapacityError(state);
|
|
991
|
+
const backup = existingTarget
|
|
992
|
+
? `${target}.${process.pid}.${randomUUID()}.capture-backup`
|
|
993
|
+
: null;
|
|
994
|
+
const staged = [];
|
|
995
|
+
let backupCreated = false;
|
|
996
|
+
let candidatePublished = false;
|
|
997
|
+
try {
|
|
998
|
+
if (backup !== null) {
|
|
999
|
+
await fsOps.rename(target, backup);
|
|
1000
|
+
backupCreated = true;
|
|
1001
|
+
}
|
|
1002
|
+
await fsOps.rename(temporary, target);
|
|
1003
|
+
candidatePublished = true;
|
|
1004
|
+
for (const candidate of retire) {
|
|
1005
|
+
const stagedPath = `${candidate.file}.${process.pid}.${randomUUID()}.capture-retire`;
|
|
1006
|
+
await fsOps.rename(candidate.file, stagedPath);
|
|
1007
|
+
staged.push({ original: candidate.file, staged: stagedPath });
|
|
1008
|
+
}
|
|
1009
|
+
if (backup !== null) await fsOps.unlink(backup).catch(() => undefined);
|
|
1010
|
+
for (const item of staged) await fsOps.unlink(item.staged).catch(() => undefined);
|
|
1011
|
+
} catch (error) {
|
|
1012
|
+
if (candidatePublished) await fsOps.unlink(target).catch(() => undefined);
|
|
1013
|
+
for (const item of staged)
|
|
1014
|
+
await fsOps.rename(item.staged, item.original).catch(() => undefined);
|
|
1015
|
+
if (backupCreated && backup !== null)
|
|
1016
|
+
await fsOps.rename(backup, target).catch(() => undefined);
|
|
1017
|
+
throw error;
|
|
1018
|
+
}
|
|
1019
|
+
} catch (error) {
|
|
1020
|
+
await fsOps.unlink(temporary).catch(() => undefined);
|
|
1021
|
+
throw error;
|
|
1022
|
+
}
|
|
1023
|
+
};
|
|
1024
|
+
return lockOptions.lockHeld
|
|
1025
|
+
? operation()
|
|
1026
|
+
: withCaptureStoreLock(directory, operation, fsOps, lockOptions);
|
|
1027
|
+
}
|
|
1028
|
+
|
|
1029
|
+
async function atomicWritePrivate(
|
|
1030
|
+
file,
|
|
1031
|
+
value,
|
|
1032
|
+
{
|
|
1033
|
+
maxBytes = null,
|
|
1034
|
+
compact = false,
|
|
1035
|
+
capture = false,
|
|
1036
|
+
captureLock = false,
|
|
1037
|
+
captureLockHeld = false,
|
|
1038
|
+
} = {},
|
|
1039
|
+
) {
|
|
1040
|
+
const serialized = `${compact ? JSON.stringify(value) : JSON.stringify(value, null, 2)}\n`;
|
|
1041
|
+
if (maxBytes !== null && Buffer.byteLength(serialized, 'utf8') > maxBytes) {
|
|
1042
|
+
const error = new Error('capture snapshot exceeds bound');
|
|
1043
|
+
error.diagnosticReason = 'capture_size_exceeded';
|
|
1044
|
+
error.captureBytes = Buffer.byteLength(serialized, 'utf8');
|
|
1045
|
+
error.captureLimitBytes = maxBytes;
|
|
298
1046
|
throw error;
|
|
299
1047
|
}
|
|
1048
|
+
if (capture) {
|
|
1049
|
+
const result = await writeBoundedCapture(file, serialized, value, captureFs, {
|
|
1050
|
+
lockHeld: captureLockHeld,
|
|
1051
|
+
});
|
|
1052
|
+
return result;
|
|
1053
|
+
}
|
|
1054
|
+
const directory = await ensurePrivateDirectory(dirname(file));
|
|
1055
|
+
if (captureLock) return withCaptureStoreLock(directory, () => writePrivateFile(file, serialized));
|
|
1056
|
+
return writePrivateFile(file, serialized);
|
|
1057
|
+
}
|
|
1058
|
+
async function atomicWriteImmutablePrivate(file, value, sourceRevision, captureLockHeld = false) {
|
|
1059
|
+
validateCaptureSnapshot(value, sourceRevision);
|
|
1060
|
+
await atomicWritePrivate(file, value, {
|
|
1061
|
+
maxBytes: MAX_CAPTURE_BYTES,
|
|
1062
|
+
compact: true,
|
|
1063
|
+
capture: true,
|
|
1064
|
+
captureLockHeld,
|
|
1065
|
+
});
|
|
300
1066
|
}
|
|
301
|
-
|
|
302
|
-
|
|
1067
|
+
|
|
1068
|
+
async function writeSegmentedCapture(
|
|
1069
|
+
directory,
|
|
1070
|
+
{ threadId, sourceRevision, metadata, pages, lockHeld = false, fs = captureFs },
|
|
1071
|
+
) {
|
|
1072
|
+
if (!Array.isArray(pages) || pages.length < 1 || pages.length > MAX_CAPTURE_PAGES)
|
|
1073
|
+
throw new Error('capture page count exceeds bound');
|
|
1074
|
+
const segments = [];
|
|
1075
|
+
const prepared = [];
|
|
1076
|
+
const cleanupSource = async () => {
|
|
1077
|
+
for (const descriptor of pages) {
|
|
1078
|
+
if (descriptor?.stagedPath) await fs.unlink(descriptor.stagedPath).catch(() => undefined);
|
|
1079
|
+
}
|
|
1080
|
+
};
|
|
303
1081
|
try {
|
|
304
|
-
const
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
1082
|
+
for (const [descriptorPosition, descriptor] of pages.entries()) {
|
|
1083
|
+
if (
|
|
1084
|
+
!isRecord(descriptor) ||
|
|
1085
|
+
!isSafeBoundedInteger(descriptor.pageIndex, 0, MAX_CAPTURE_PAGES - 1) ||
|
|
1086
|
+
descriptor.pageIndex !== descriptorPosition ||
|
|
1087
|
+
typeof descriptor.stagedPath !== 'string' ||
|
|
1088
|
+
typeof descriptor.pageHash !== 'string' ||
|
|
1089
|
+
!/^[a-f0-9]{64}$/u.test(descriptor.pageHash)
|
|
1090
|
+
)
|
|
1091
|
+
throw new Error('capture staged page identity validation failed');
|
|
1092
|
+
const page = JSON.parse(
|
|
1093
|
+
await readPrivateFile(
|
|
1094
|
+
descriptor.stagedPath,
|
|
1095
|
+
'capture staged page',
|
|
1096
|
+
MAX_CAPTURE_PAGE_BYTES,
|
|
1097
|
+
fs,
|
|
1098
|
+
),
|
|
1099
|
+
);
|
|
1100
|
+
if (
|
|
1101
|
+
!isRecord(page) ||
|
|
1102
|
+
page.threadId !== threadId ||
|
|
1103
|
+
!Array.isArray(page.turns) ||
|
|
1104
|
+
hashJson(page) !== descriptor.pageHash
|
|
1105
|
+
)
|
|
1106
|
+
throw new Error('capture staged page integrity validation failed');
|
|
1107
|
+
const payload = {
|
|
1108
|
+
version: 'codex-capture-segment-v1',
|
|
1109
|
+
threadId,
|
|
1110
|
+
sourceRevision,
|
|
1111
|
+
pageIndex: descriptor.pageIndex,
|
|
1112
|
+
pageHash: descriptor.pageHash,
|
|
1113
|
+
itemCount: page.turns.reduce(
|
|
1114
|
+
(count, turn) => count + (Array.isArray(turn.items) ? turn.items.length : 0),
|
|
1115
|
+
0,
|
|
1116
|
+
),
|
|
1117
|
+
page,
|
|
1118
|
+
};
|
|
1119
|
+
validateCaptureSegment(payload, threadId, sourceRevision);
|
|
1120
|
+
const serialized = `${JSON.stringify(payload)}\n`;
|
|
1121
|
+
if (Buffer.byteLength(serialized, 'utf8') > MAX_CAPTURE_PAGE_BYTES)
|
|
1122
|
+
throw new Error('capture page exceeds bound');
|
|
1123
|
+
const stagedState = await readCaptureDirectoryState(directory, fs);
|
|
1124
|
+
if (
|
|
1125
|
+
stagedState.totalBytes + Buffer.byteLength(serialized, 'utf8') >
|
|
1126
|
+
MAX_CAPTURE_DIRECTORY_BYTES + MAX_CAPTURE_STAGING_HEADROOM_BYTES
|
|
1127
|
+
)
|
|
1128
|
+
throw captureCapacityError(stagedState);
|
|
1129
|
+
const stagedFinalPath = resolve(
|
|
1130
|
+
directory,
|
|
1131
|
+
`.capture-segment-stage.${safeThreadName(threadId)}.${randomUUID()}.${descriptor.pageIndex}.tmp`,
|
|
1132
|
+
);
|
|
1133
|
+
await fs.writeFile(stagedFinalPath, serialized, { flag: 'wx', mode: 0o600 });
|
|
1134
|
+
prepared.push(stagedFinalPath);
|
|
1135
|
+
segments.push({
|
|
1136
|
+
name: captureSegmentName(threadId, sourceRevision, descriptor.pageIndex),
|
|
1137
|
+
pageHash: descriptor.pageHash,
|
|
1138
|
+
itemCount: payload.itemCount,
|
|
1139
|
+
bytes: Buffer.byteLength(serialized, 'utf8'),
|
|
1140
|
+
stagedPath: descriptor.stagedPath,
|
|
1141
|
+
stagedFinalPath,
|
|
1142
|
+
});
|
|
1143
|
+
}
|
|
309
1144
|
} catch (error) {
|
|
310
|
-
|
|
1145
|
+
for (const file of prepared) await fs.unlink(file).catch(() => undefined);
|
|
1146
|
+
await cleanupSource();
|
|
1147
|
+
throw error;
|
|
1148
|
+
}
|
|
1149
|
+
const manifest = {
|
|
1150
|
+
version: 'codex-capture-manifest-v1',
|
|
1151
|
+
threadId,
|
|
1152
|
+
sourceRevision,
|
|
1153
|
+
pageCount: segments.length,
|
|
1154
|
+
itemCount: segments.reduce((count, segment) => count + segment.itemCount, 0),
|
|
1155
|
+
totalBytes: segments.reduce((count, segment) => count + segment.bytes, 0),
|
|
1156
|
+
segments: segments.map((segment) => segment.name),
|
|
1157
|
+
pageHashes: segments.map((segment) => segment.pageHash),
|
|
1158
|
+
};
|
|
1159
|
+
if (
|
|
1160
|
+
isRecord(metadata?.thread) &&
|
|
1161
|
+
captureManifestRevision(metadata.thread, manifest.pageHashes) !== sourceRevision
|
|
1162
|
+
) {
|
|
1163
|
+
await cleanupSource();
|
|
1164
|
+
for (const file of prepared) await fs.unlink(file).catch(() => undefined);
|
|
1165
|
+
throw new Error('capture manifest revision does not match page hashes');
|
|
1166
|
+
}
|
|
1167
|
+
if (manifest.totalBytes > MAX_CAPTURE_TOTAL_BYTES || manifest.itemCount > MAX_CAPTURE_ITEMS) {
|
|
1168
|
+
await cleanupSource();
|
|
1169
|
+
for (const file of prepared) await fs.unlink(file).catch(() => undefined);
|
|
1170
|
+
throw new Error('capture exceeds segmented bound');
|
|
1171
|
+
}
|
|
1172
|
+
const operation = async () => {
|
|
1173
|
+
const state = await readCaptureDirectoryState(directory, fs);
|
|
1174
|
+
if (state.unsafeRecovery || state.blockedEntries) {
|
|
1175
|
+
await cleanupSource();
|
|
1176
|
+
for (const file of prepared) await fs.unlink(file).catch(() => undefined);
|
|
1177
|
+
throw captureCapacityError(state);
|
|
1178
|
+
}
|
|
1179
|
+
const staged = [];
|
|
1180
|
+
const publishedTargets = [];
|
|
1181
|
+
try {
|
|
1182
|
+
for (const segment of segments) {
|
|
1183
|
+
const target = resolve(directory, segment.name);
|
|
1184
|
+
try {
|
|
1185
|
+
await assertPrivateFile(target, 'capture segment', fs);
|
|
1186
|
+
const existing = JSON.parse(
|
|
1187
|
+
await readPrivateFile(target, 'capture segment', MAX_CAPTURE_PAGE_BYTES, fs),
|
|
1188
|
+
);
|
|
1189
|
+
validateCaptureSegment(existing, threadId, sourceRevision);
|
|
1190
|
+
if (existing.pageHash !== segment.pageHash) throw new Error('capture segment collision');
|
|
1191
|
+
await fs.unlink(segment.stagedFinalPath).catch(() => undefined);
|
|
1192
|
+
continue;
|
|
1193
|
+
} catch (error) {
|
|
1194
|
+
if (error?.code !== 'ENOENT') throw error;
|
|
1195
|
+
}
|
|
1196
|
+
staged.push({ temporary: segment.stagedFinalPath, target, bytes: segment.bytes });
|
|
1197
|
+
}
|
|
1198
|
+
const protectedNames = new Set(segments.map((segment) => segment.name));
|
|
1199
|
+
const groups = new Map();
|
|
1200
|
+
for (const entry of state.managedFiles) {
|
|
1201
|
+
if (
|
|
1202
|
+
!entry.immutable ||
|
|
1203
|
+
protectedNames.has(entry.name) ||
|
|
1204
|
+
state.referencedImmutableNames.has(entry.name) ||
|
|
1205
|
+
state.protectedGroups.has(entry.group)
|
|
1206
|
+
)
|
|
1207
|
+
continue;
|
|
1208
|
+
const members = groups.get(entry.group ?? entry.name) ?? [];
|
|
1209
|
+
members.push(entry);
|
|
1210
|
+
groups.set(entry.group ?? entry.name, members);
|
|
1211
|
+
}
|
|
1212
|
+
const retired = [];
|
|
1213
|
+
let projected = state.totalBytes;
|
|
1214
|
+
let projectedFiles = state.managedFiles.length + staged.length;
|
|
1215
|
+
for (const members of [...groups.values()].sort(
|
|
1216
|
+
(left, right) =>
|
|
1217
|
+
Math.min(...left.map((entry) => entry.mtimeMs)) -
|
|
1218
|
+
Math.min(...right.map((entry) => entry.mtimeMs)),
|
|
1219
|
+
)) {
|
|
1220
|
+
if (projected <= MAX_CAPTURE_DIRECTORY_BYTES && projectedFiles <= MAX_CAPTURE_FILES) break;
|
|
1221
|
+
const bytes = members.reduce((sum, entry) => sum + entry.size, 0);
|
|
1222
|
+
projected -= bytes;
|
|
1223
|
+
projectedFiles -= members.length;
|
|
1224
|
+
retired.push(...members);
|
|
1225
|
+
}
|
|
1226
|
+
if (projected > MAX_CAPTURE_DIRECTORY_BYTES || projectedFiles > MAX_CAPTURE_FILES)
|
|
1227
|
+
throw captureCapacityError({ totalBytes: projected, managedFiles: state.managedFiles });
|
|
1228
|
+
const retiredPaths = [];
|
|
1229
|
+
try {
|
|
1230
|
+
for (const entry of retired) {
|
|
1231
|
+
const stagedPath = `${entry.file}.${process.pid}.${randomUUID()}.capture-retire`;
|
|
1232
|
+
await fs.rename(entry.file, stagedPath);
|
|
1233
|
+
retiredPaths.push({ original: entry.file, staged: stagedPath });
|
|
1234
|
+
}
|
|
1235
|
+
for (const item of staged) await fs.rename(item.temporary, item.target);
|
|
1236
|
+
publishedTargets.push(...staged.map((item) => item.target));
|
|
1237
|
+
for (const segment of segments) {
|
|
1238
|
+
const verify = JSON.parse(
|
|
1239
|
+
await readPrivateFile(
|
|
1240
|
+
resolve(directory, segment.name),
|
|
1241
|
+
'capture segment',
|
|
1242
|
+
MAX_CAPTURE_PAGE_BYTES,
|
|
1243
|
+
fs,
|
|
1244
|
+
),
|
|
1245
|
+
);
|
|
1246
|
+
validateCaptureSegment(verify, threadId, sourceRevision);
|
|
1247
|
+
}
|
|
1248
|
+
for (const segment of segments) await fs.unlink(segment.stagedPath).catch(() => undefined);
|
|
1249
|
+
for (const file of prepared) await fs.unlink(file).catch(() => undefined);
|
|
1250
|
+
for (const item of retiredPaths) await fs.unlink(item.staged).catch(() => undefined);
|
|
1251
|
+
return manifest;
|
|
1252
|
+
} catch (error) {
|
|
1253
|
+
for (const item of retiredPaths)
|
|
1254
|
+
await fs.rename(item.staged, item.original).catch(() => undefined);
|
|
1255
|
+
if (error?.code === 'ENOENT') throw captureCapacityError(state);
|
|
1256
|
+
throw error;
|
|
1257
|
+
}
|
|
1258
|
+
} catch (error) {
|
|
1259
|
+
for (const item of staged) {
|
|
1260
|
+
await fs.unlink(item.temporary).catch(() => undefined);
|
|
1261
|
+
await fs.unlink(item.target).catch(() => undefined);
|
|
1262
|
+
}
|
|
1263
|
+
for (const target of publishedTargets) await fs.unlink(target).catch(() => undefined);
|
|
1264
|
+
for (const segment of segments) await fs.unlink(segment.stagedPath).catch(() => undefined);
|
|
1265
|
+
for (const file of prepared) await fs.unlink(file).catch(() => undefined);
|
|
1266
|
+
throw error;
|
|
1267
|
+
}
|
|
1268
|
+
};
|
|
1269
|
+
return lockHeld ? operation() : withCaptureStoreLock(directory, operation, fs);
|
|
1270
|
+
}
|
|
1271
|
+
|
|
1272
|
+
/**
|
|
1273
|
+
* Test seam for the segmented capture store. Pages are written as private
|
|
1274
|
+
* staging files first, then published as one lock-bound capture group.
|
|
1275
|
+
*/
|
|
1276
|
+
export async function __testWriteSegmentedCapture(
|
|
1277
|
+
directory,
|
|
1278
|
+
{ threadId, sourceRevision, metadata = { thread: { id: threadId } }, pages },
|
|
1279
|
+
{ fs = captureFs } = {},
|
|
1280
|
+
) {
|
|
1281
|
+
await ensurePrivateDirectory(directory, fs);
|
|
1282
|
+
const staged = [];
|
|
1283
|
+
try {
|
|
1284
|
+
for (const [pageIndex, page] of pages.entries()) {
|
|
1285
|
+
const serialized = `${JSON.stringify(page)}\n`;
|
|
1286
|
+
const state = await readCaptureDirectoryState(directory, fs);
|
|
1287
|
+
if (state.totalBytes + Buffer.byteLength(serialized, 'utf8') > MAX_CAPTURE_DIRECTORY_BYTES)
|
|
1288
|
+
throw captureCapacityError(state);
|
|
1289
|
+
const stagedPath = resolve(
|
|
1290
|
+
directory,
|
|
1291
|
+
`.capture-stage.test.${safeThreadName(threadId)}.${randomUUID()}.${pageIndex}.tmp`,
|
|
1292
|
+
);
|
|
1293
|
+
await fs.writeFile(stagedPath, serialized, { flag: 'wx', mode: 0o600 });
|
|
1294
|
+
staged.push({
|
|
1295
|
+
pageIndex,
|
|
1296
|
+
stagedPath,
|
|
1297
|
+
pageHash: hashJson(page),
|
|
1298
|
+
bytes: Buffer.byteLength(serialized, 'utf8'),
|
|
1299
|
+
});
|
|
1300
|
+
}
|
|
1301
|
+
return await writeSegmentedCapture(directory, {
|
|
1302
|
+
threadId,
|
|
1303
|
+
sourceRevision,
|
|
1304
|
+
metadata,
|
|
1305
|
+
pages: staged,
|
|
1306
|
+
fs,
|
|
1307
|
+
});
|
|
1308
|
+
} catch (error) {
|
|
1309
|
+
for (const page of staged) await fs.unlink(page.stagedPath).catch(() => undefined);
|
|
1310
|
+
throw error;
|
|
311
1311
|
}
|
|
312
|
-
|
|
1312
|
+
}
|
|
1313
|
+
|
|
1314
|
+
// Test-only seam for deterministic capture-store fault injection. Production
|
|
1315
|
+
// code continues to use the immutable writer above with the real filesystem.
|
|
1316
|
+
export async function __testWriteBoundedCapture(
|
|
1317
|
+
file,
|
|
1318
|
+
value,
|
|
1319
|
+
{ fs = captureFs, now = Date.now, sleep } = {},
|
|
1320
|
+
) {
|
|
1321
|
+
const serialized = `${JSON.stringify(value)}\n`;
|
|
1322
|
+
if (Buffer.byteLength(serialized, 'utf8') > MAX_CAPTURE_BYTES)
|
|
1323
|
+
throw new Error('capture snapshot exceeds bound');
|
|
1324
|
+
validateCaptureSnapshot(value, value.sourceRevision);
|
|
1325
|
+
return writeBoundedCapture(file, serialized, value, fs, { now, sleep });
|
|
313
1326
|
}
|
|
314
1327
|
async function atomicWriteTextPrivate(file, text) {
|
|
315
1328
|
if (Buffer.byteLength(text) > MAX_NOTE_BYTES) throw new Error('recovery notes exceed bound');
|
|
@@ -351,17 +1364,156 @@ function validateThreadRead(value, threadId) {
|
|
|
351
1364
|
return value;
|
|
352
1365
|
}
|
|
353
1366
|
|
|
354
|
-
|
|
1367
|
+
function validateThreadMetadata(value, threadId) {
|
|
1368
|
+
if (!isRecord(value) || !isRecord(value.thread) || value.thread.id !== threadId)
|
|
1369
|
+
throw new Error('thread/read returned invalid thread metadata');
|
|
1370
|
+
return value;
|
|
1371
|
+
}
|
|
1372
|
+
|
|
1373
|
+
export function parseTurnsPage(value, threadId, pageIndex) {
|
|
1374
|
+
if (!isRecord(value)) throw new Error('thread/turns/list returned invalid page');
|
|
1375
|
+
const data = Array.isArray(value.data)
|
|
1376
|
+
? value.data
|
|
1377
|
+
: Array.isArray(value.turns)
|
|
1378
|
+
? value.turns
|
|
1379
|
+
: null;
|
|
1380
|
+
if (data === null || data.some((turn) => !isRecord(turn) || typeof turn.id !== 'string'))
|
|
1381
|
+
throw new Error('thread/turns/list returned invalid turns');
|
|
1382
|
+
const page = { threadId, turns: data };
|
|
1383
|
+
const serializedBytes = Buffer.byteLength(JSON.stringify(page), 'utf8');
|
|
1384
|
+
if (serializedBytes > MAX_CAPTURE_PAGE_BYTES) {
|
|
1385
|
+
const error = new Error('capture page exceeds bound');
|
|
1386
|
+
error.diagnosticReason = 'capture_size_exceeded';
|
|
1387
|
+
error.captureBytes = serializedBytes;
|
|
1388
|
+
error.captureLimitBytes = MAX_CAPTURE_PAGE_BYTES;
|
|
1389
|
+
throw error;
|
|
1390
|
+
}
|
|
1391
|
+
const cursor = value.nextCursor ?? value.next_cursor ?? null;
|
|
1392
|
+
if (cursor !== null && (typeof cursor !== 'string' || cursor.length > 512))
|
|
1393
|
+
throw new Error('thread/turns/list returned invalid cursor');
|
|
1394
|
+
return { page, pageIndex, cursor };
|
|
1395
|
+
}
|
|
1396
|
+
|
|
1397
|
+
function summarizeCaptureItem(item) {
|
|
1398
|
+
if (!isRecord(item)) return { type: 'unknown' };
|
|
1399
|
+
const summary = { type: typeof item.type === 'string' ? item.type : 'unknown' };
|
|
1400
|
+
if (typeof item.id === 'string') summary.id = item.id;
|
|
1401
|
+
if (summary.type === 'userMessage') {
|
|
1402
|
+
const parts = Array.isArray(item.content)
|
|
1403
|
+
? item.content.filter((part) => isRecord(part) && typeof part.text === 'string')
|
|
1404
|
+
: [];
|
|
1405
|
+
const contentChars = parts.reduce((total, part) => total + part.text.length, 0);
|
|
1406
|
+
summary.content = parts.slice(0, 64).map((part) => ({
|
|
1407
|
+
type: part.type,
|
|
1408
|
+
text: part.text.slice(0, MAX_NOTE_ITEM_CHARS),
|
|
1409
|
+
}));
|
|
1410
|
+
summary.contentChars = contentChars;
|
|
1411
|
+
summary.bodyComplete =
|
|
1412
|
+
parts.length <= 64 && parts.every((part) => part.text.length <= MAX_NOTE_ITEM_CHARS);
|
|
1413
|
+
} else if (summary.type === 'agentMessage' || summary.type === 'plan') {
|
|
1414
|
+
const text = typeof item.text === 'string' ? item.text : '';
|
|
1415
|
+
summary.text = text.slice(0, MAX_NOTE_ITEM_CHARS);
|
|
1416
|
+
summary.textChars = text.length;
|
|
1417
|
+
summary.bodyComplete = text.length <= MAX_NOTE_ITEM_CHARS;
|
|
1418
|
+
} else if (summary.type === 'commandExecution') {
|
|
1419
|
+
const command = typeof item.command === 'string' ? item.command : '';
|
|
1420
|
+
summary.command = command.slice(0, MAX_NOTE_ITEM_CHARS);
|
|
1421
|
+
summary.commandComplete = command.length <= MAX_NOTE_ITEM_CHARS;
|
|
1422
|
+
for (const key of ['status', 'exitCode']) {
|
|
1423
|
+
if (key in item && (typeof item[key] === 'string' || typeof item[key] === 'number'))
|
|
1424
|
+
summary[key] = item[key];
|
|
1425
|
+
}
|
|
1426
|
+
summary.aggregatedOutput = String(item.aggregatedOutput ?? item.output ?? '').slice(0, 1024);
|
|
1427
|
+
summary.outputChars = String(item.aggregatedOutput ?? item.output ?? '').length;
|
|
1428
|
+
}
|
|
1429
|
+
return summary;
|
|
1430
|
+
}
|
|
1431
|
+
|
|
1432
|
+
function summarizeCaptureTurn(turn) {
|
|
1433
|
+
return {
|
|
1434
|
+
...(typeof turn.id === 'string' ? { id: turn.id } : {}),
|
|
1435
|
+
...(typeof turn.status === 'string' ? { status: turn.status } : {}),
|
|
1436
|
+
items: turn.items.map(summarizeCaptureItem),
|
|
1437
|
+
};
|
|
1438
|
+
}
|
|
1439
|
+
|
|
1440
|
+
export function mergePaginatedTurns(metadata, pages, threadId) {
|
|
1441
|
+
validateThreadMetadata(metadata, threadId);
|
|
1442
|
+
if (!Array.isArray(pages) || pages.length < 1 || pages.length > MAX_CAPTURE_PAGES)
|
|
1443
|
+
throw new Error('capture page count exceeds bound');
|
|
1444
|
+
const turns = [];
|
|
1445
|
+
const ids = new Set();
|
|
1446
|
+
const itemIds = new Set();
|
|
1447
|
+
let itemCount = 0;
|
|
1448
|
+
for (const [pagePosition, entry] of pages.entries()) {
|
|
1449
|
+
const page = entry?.page;
|
|
1450
|
+
if (!isRecord(page) || page.threadId !== threadId || !Array.isArray(page.turns))
|
|
1451
|
+
throw new Error('capture page identity validation failed');
|
|
1452
|
+
if (entry.pageIndex !== pagePosition) throw new Error('capture pages are out of order');
|
|
1453
|
+
for (const turn of page.turns) {
|
|
1454
|
+
if (!isRecord(turn) || typeof turn.id !== 'string' || ids.has(turn.id))
|
|
1455
|
+
throw new Error('capture history contains duplicate turn');
|
|
1456
|
+
ids.add(turn.id);
|
|
1457
|
+
if (!Array.isArray(turn.items)) throw new Error('capture turn items are unavailable');
|
|
1458
|
+
for (const item of turn.items) {
|
|
1459
|
+
if (isRecord(item) && typeof item.id === 'string') {
|
|
1460
|
+
if (itemIds.has(item.id)) throw new Error('capture history contains duplicate item');
|
|
1461
|
+
itemIds.add(item.id);
|
|
1462
|
+
}
|
|
1463
|
+
}
|
|
1464
|
+
itemCount += turn.items.length;
|
|
1465
|
+
if (itemCount > MAX_CAPTURE_ITEMS) {
|
|
1466
|
+
const error = new Error('capture item count exceeds bound');
|
|
1467
|
+
error.captureItemCount = itemCount;
|
|
1468
|
+
error.captureItemLimit = MAX_CAPTURE_ITEMS;
|
|
1469
|
+
throw error;
|
|
1470
|
+
}
|
|
1471
|
+
turns.push(turn);
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1474
|
+
if (turns.length === 0) throw new Error('thread/turns/list returned no recoverable turns');
|
|
1475
|
+
return { thread: { ...metadata.thread, turns } };
|
|
1476
|
+
}
|
|
1477
|
+
|
|
1478
|
+
function captureSourceRevision(capture) {
|
|
1479
|
+
if (capture.pages === null) return hashJson(capture.read);
|
|
1480
|
+
return captureManifestRevision(
|
|
1481
|
+
capture.metadata.thread,
|
|
1482
|
+
capture.pages.map(({ pageHash }) => pageHash),
|
|
1483
|
+
);
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
export function buildRecoveryCandidates(read, revision, policy = null, options = {}) {
|
|
355
1487
|
const turns = read.thread.turns;
|
|
356
|
-
const rawItems = turns.flatMap((turn) =>
|
|
357
|
-
|
|
1488
|
+
const rawItems = turns.flatMap((turn, turnIndex) =>
|
|
1489
|
+
(Array.isArray(turn.items) ? turn.items : []).map((item, itemIndex) => ({
|
|
1490
|
+
item,
|
|
1491
|
+
turnIndex,
|
|
1492
|
+
itemIndex,
|
|
1493
|
+
})),
|
|
1494
|
+
);
|
|
1495
|
+
const maxItems = options.maxItems ?? MAX_RECOVERY_ITEMS;
|
|
1496
|
+
if (!Number.isSafeInteger(maxItems) || maxItems < 1 || maxItems > MAX_RECOVERY_ITEMS)
|
|
1497
|
+
throw new Error('recovery candidate bound is invalid');
|
|
1498
|
+
if (rawItems.length > maxItems && options.allowWindow !== true)
|
|
358
1499
|
throw new Error('recovery candidate count exceeds bound');
|
|
359
|
-
|
|
1500
|
+
if (rawItems.length > MAX_RECOVERY_SOURCE_INDEX) {
|
|
1501
|
+
const error = new Error('recovery source index exceeds bound');
|
|
1502
|
+
error.captureItemCount = rawItems.length;
|
|
1503
|
+
error.captureItemLimit = MAX_RECOVERY_SOURCE_INDEX;
|
|
1504
|
+
throw error;
|
|
1505
|
+
}
|
|
1506
|
+
let items = rawItems.map(({ item: rawItem, turnIndex, itemIndex }, index) => {
|
|
360
1507
|
const item = isRecord(rawItem) ? rawItem : {};
|
|
361
1508
|
const type = typeof item.type === 'string' ? item.type : 'unknown';
|
|
362
1509
|
const commandOutput =
|
|
363
1510
|
type === 'commandExecution' ? String(item.aggregatedOutput ?? item.output ?? '') : '';
|
|
1511
|
+
const sourceOutputChars =
|
|
1512
|
+
type === 'commandExecution' && isSafeCounter(item.outputChars)
|
|
1513
|
+
? item.outputChars
|
|
1514
|
+
: commandOutput.length;
|
|
364
1515
|
const command = typeof item.command === 'string' ? item.command : 'unknown';
|
|
1516
|
+
const bodyComplete = item.bodyComplete !== false;
|
|
365
1517
|
const status = typeof item.status === 'string' ? item.status : 'unknown';
|
|
366
1518
|
const exitCode = typeof item.exitCode === 'number' ? item.exitCode : null;
|
|
367
1519
|
const localText =
|
|
@@ -407,7 +1559,8 @@ export function buildRecoveryCandidates(read, revision, policy = null) {
|
|
|
407
1559
|
) &&
|
|
408
1560
|
(commandName !== 'git' ||
|
|
409
1561
|
['status', 'diff', 'log', 'show', 'rev-parse'].includes(commandWords[1])));
|
|
410
|
-
const unknownCommand =
|
|
1562
|
+
const unknownCommand =
|
|
1563
|
+
type === 'commandExecution' && (!readOnlyCommand || item.commandComplete === false);
|
|
411
1564
|
const unknown = !['userMessage', 'agentMessage', 'plan', 'commandExecution'].includes(type);
|
|
412
1565
|
const recentCount = policy?.preserveRecentMessages ?? 6;
|
|
413
1566
|
const recent = index >= Math.max(0, rawItems.length - recentCount);
|
|
@@ -429,7 +1582,14 @@ export function buildRecoveryCandidates(read, revision, policy = null) {
|
|
|
429
1582
|
if (unknownCommand) pinReasons.push('unknown');
|
|
430
1583
|
if (unknown) pinReasons.push('unknown');
|
|
431
1584
|
if (recent && pinReasons.length === 0) pinReasons.push('recent');
|
|
432
|
-
const localChars =
|
|
1585
|
+
const localChars =
|
|
1586
|
+
type === 'commandExecution'
|
|
1587
|
+
? sourceOutputChars
|
|
1588
|
+
: type === 'userMessage' && isSafeCounter(item.contentChars)
|
|
1589
|
+
? item.contentChars
|
|
1590
|
+
: (type === 'agentMessage' || type === 'plan') && isSafeCounter(item.textChars)
|
|
1591
|
+
? item.textChars
|
|
1592
|
+
: localText.length;
|
|
433
1593
|
return {
|
|
434
1594
|
id: `recovery-${index + 1}`,
|
|
435
1595
|
text: redact(localText, 16_384),
|
|
@@ -439,10 +1599,59 @@ export function buildRecoveryCandidates(read, revision, policy = null) {
|
|
|
439
1599
|
pinReasons,
|
|
440
1600
|
source: rawItem,
|
|
441
1601
|
sourceItemId: typeof item.id === 'string' ? item.id : `item-${index + 1}`,
|
|
442
|
-
|
|
1602
|
+
sourceLocation: { flatIndex: index, turnIndex, itemIndex },
|
|
1603
|
+
sourceRefs: null,
|
|
1604
|
+
sourceBodyChars: sourceOutputChars,
|
|
1605
|
+
bodyComplete,
|
|
443
1606
|
kind,
|
|
444
1607
|
};
|
|
445
1608
|
});
|
|
1609
|
+
if (items.length > maxItems) {
|
|
1610
|
+
const protectedItems = items.filter((item) => item.protected);
|
|
1611
|
+
const anchorIds = new Set();
|
|
1612
|
+
const first = items[0];
|
|
1613
|
+
if (first?.protected) anchorIds.add(first.id);
|
|
1614
|
+
const currentGoal = items.findLast((item) => item.kind === 'user_message');
|
|
1615
|
+
if (currentGoal?.protected) anchorIds.add(currentGoal.id);
|
|
1616
|
+
const recentCount = policy?.preserveRecentMessages ?? 6;
|
|
1617
|
+
for (const item of items.slice(-recentCount)) {
|
|
1618
|
+
if (item.protected) anchorIds.add(item.id);
|
|
1619
|
+
}
|
|
1620
|
+
const individualProtected = protectedItems.filter((item) => anchorIds.has(item.id));
|
|
1621
|
+
const bundledProtected = protectedItems.filter((item) => !anchorIds.has(item.id));
|
|
1622
|
+
if (individualProtected.length >= maxItems)
|
|
1623
|
+
throw new Error('recovery protected anchors exceed bound');
|
|
1624
|
+
const selected = new Set(individualProtected.map((item) => item.id));
|
|
1625
|
+
if (bundledProtected.length > 0) {
|
|
1626
|
+
const bundle = {
|
|
1627
|
+
id: 'recovery-protected-bundle-1',
|
|
1628
|
+
text: `protected reference bundle: ${bundledProtected.length} items retained in the verified immutable capture`,
|
|
1629
|
+
localText: '',
|
|
1630
|
+
localChars: 0,
|
|
1631
|
+
protected: true,
|
|
1632
|
+
pinReasons: ['reference'],
|
|
1633
|
+
source: null,
|
|
1634
|
+
sourceItemId: null,
|
|
1635
|
+
sourceLocation: null,
|
|
1636
|
+
sourceRefs: bundledProtected.map((item) => ({
|
|
1637
|
+
...item.sourceLocation,
|
|
1638
|
+
sourceItemId: item.sourceItemId,
|
|
1639
|
+
})),
|
|
1640
|
+
sourceBodyChars: 0,
|
|
1641
|
+
kind: 'reference',
|
|
1642
|
+
};
|
|
1643
|
+
items = [...items, bundle];
|
|
1644
|
+
selected.add(bundle.id);
|
|
1645
|
+
}
|
|
1646
|
+
for (const item of items
|
|
1647
|
+
.filter((candidate) => !candidate.protected)
|
|
1648
|
+
.slice()
|
|
1649
|
+
.reverse()) {
|
|
1650
|
+
if (selected.size >= maxItems) break;
|
|
1651
|
+
selected.add(item.id);
|
|
1652
|
+
}
|
|
1653
|
+
items = items.filter((item) => selected.has(item.id));
|
|
1654
|
+
}
|
|
446
1655
|
const projection = {
|
|
447
1656
|
goal:
|
|
448
1657
|
policy?.goal !== undefined
|
|
@@ -464,7 +1673,105 @@ export function buildRecoveryCandidates(read, revision, policy = null) {
|
|
|
464
1673
|
pin_reasons: pinReasons,
|
|
465
1674
|
})),
|
|
466
1675
|
};
|
|
467
|
-
return { items, projection, projectionHash: hashJson(projection), revision };
|
|
1676
|
+
return { items, projection, projectionHash: hashJson(projection), revision };
|
|
1677
|
+
}
|
|
1678
|
+
|
|
1679
|
+
function selectionFittingProjection(candidates) {
|
|
1680
|
+
return {
|
|
1681
|
+
goal: candidates.projection.goal,
|
|
1682
|
+
items: candidates.items.map(
|
|
1683
|
+
({ id, kind, text, localChars, protected: protectedItem, pinReasons }) => ({
|
|
1684
|
+
id,
|
|
1685
|
+
kind,
|
|
1686
|
+
text,
|
|
1687
|
+
localChars,
|
|
1688
|
+
protected: protectedItem,
|
|
1689
|
+
pinReasons,
|
|
1690
|
+
}),
|
|
1691
|
+
),
|
|
1692
|
+
};
|
|
1693
|
+
}
|
|
1694
|
+
|
|
1695
|
+
function selectionWireProjection(goal, items) {
|
|
1696
|
+
return {
|
|
1697
|
+
goal,
|
|
1698
|
+
items: items.map(({ id, kind, text, localChars, protected: protectedItem, pinReasons }) => ({
|
|
1699
|
+
id,
|
|
1700
|
+
kind,
|
|
1701
|
+
text,
|
|
1702
|
+
local_chars: localChars,
|
|
1703
|
+
protected: protectedItem,
|
|
1704
|
+
pin_reasons: pinReasons,
|
|
1705
|
+
})),
|
|
1706
|
+
};
|
|
1707
|
+
}
|
|
1708
|
+
|
|
1709
|
+
function selectionFittingOptions(policy, maxRequestTokens) {
|
|
1710
|
+
return {
|
|
1711
|
+
preserveRecentItems: policy?.preserveRecentMessages ?? 6,
|
|
1712
|
+
maxStateTokens: policy?.maxStateTokens ?? 25_000,
|
|
1713
|
+
maxRequestTokens,
|
|
1714
|
+
};
|
|
1715
|
+
}
|
|
1716
|
+
|
|
1717
|
+
function fitWithinCapacity(fit, capacity) {
|
|
1718
|
+
if (!fit.ok) return false;
|
|
1719
|
+
const batchCount = fit.batches.length;
|
|
1720
|
+
return (
|
|
1721
|
+
batchCount <= capacity.requests &&
|
|
1722
|
+
batchCount * (fit.requestTokens + capacity.outputTokenAllowance) <= capacity.tokens
|
|
1723
|
+
);
|
|
1724
|
+
}
|
|
1725
|
+
|
|
1726
|
+
export function boundCandidatesToBudget(candidates, availableTokens, policy = null) {
|
|
1727
|
+
const capacity =
|
|
1728
|
+
typeof availableTokens === 'number'
|
|
1729
|
+
? {
|
|
1730
|
+
tokens: availableTokens,
|
|
1731
|
+
requests: Number.MAX_SAFE_INTEGER,
|
|
1732
|
+
maxRequestTokens: 30_000,
|
|
1733
|
+
outputTokenAllowance: 0,
|
|
1734
|
+
}
|
|
1735
|
+
: { outputTokenAllowance: 0, ...availableTokens };
|
|
1736
|
+
if (
|
|
1737
|
+
!isRecord(capacity) ||
|
|
1738
|
+
!isSafeBoundedInteger(capacity.tokens, 1, Number.MAX_SAFE_INTEGER) ||
|
|
1739
|
+
!isSafeBoundedInteger(capacity.requests, 1, Number.MAX_SAFE_INTEGER) ||
|
|
1740
|
+
!isSafeBoundedInteger(capacity.maxRequestTokens, 1_000, 30_000) ||
|
|
1741
|
+
!isSafeBoundedInteger(capacity.outputTokenAllowance, 0, Number.MAX_SAFE_INTEGER)
|
|
1742
|
+
)
|
|
1743
|
+
throw new Error('recovery budget has no usable token capacity');
|
|
1744
|
+
const protectedItems = candidates.items.filter((item) => item.protected);
|
|
1745
|
+
if (protectedItems.length > MAX_RECOVERY_ITEMS)
|
|
1746
|
+
throw new Error('recovery protected candidate count exceeds bound');
|
|
1747
|
+
const eligibleItems = candidates.items.filter((item) => !item.protected);
|
|
1748
|
+
const fitFor = (items) =>
|
|
1749
|
+
preflightSelectionState({
|
|
1750
|
+
projection: selectionFittingProjection({ ...candidates, items }),
|
|
1751
|
+
options: selectionFittingOptions(policy, capacity.maxRequestTokens),
|
|
1752
|
+
});
|
|
1753
|
+
for (let keepCount = eligibleItems.length; keepCount >= 0; keepCount -= 1) {
|
|
1754
|
+
const retainedEligible = new Set(
|
|
1755
|
+
(keepCount === 0 ? [] : eligibleItems.slice(-keepCount)).map((item) => item.id),
|
|
1756
|
+
);
|
|
1757
|
+
const items = candidates.items.filter(
|
|
1758
|
+
(item) => item.protected || retainedEligible.has(item.id),
|
|
1759
|
+
);
|
|
1760
|
+
const fit = fitFor(items);
|
|
1761
|
+
if (!fitWithinCapacity(fit, capacity)) continue;
|
|
1762
|
+
const projection = selectionFittingProjection({ ...candidates, items });
|
|
1763
|
+
const wireProjection = selectionWireProjection(candidates.projection.goal, items);
|
|
1764
|
+
return {
|
|
1765
|
+
items,
|
|
1766
|
+
projection: wireProjection,
|
|
1767
|
+
projectionHash: hashJson(wireProjection),
|
|
1768
|
+
revision: candidates.revision,
|
|
1769
|
+
bounded: items.length !== candidates.items.length,
|
|
1770
|
+
preflight: fit,
|
|
1771
|
+
providerProjection: projection,
|
|
1772
|
+
};
|
|
1773
|
+
}
|
|
1774
|
+
throw new Error('recovery protected candidate projection exceeds budget');
|
|
468
1775
|
}
|
|
469
1776
|
|
|
470
1777
|
export function parseSelection(value, input, candidates, currentTime = Date.now()) {
|
|
@@ -610,6 +1917,44 @@ export function recoveryNotesText(
|
|
|
610
1917
|
candidates,
|
|
611
1918
|
decisions,
|
|
612
1919
|
) {
|
|
1920
|
+
const formatSourceLocations = (refs) => {
|
|
1921
|
+
const sorted = refs
|
|
1922
|
+
.filter(
|
|
1923
|
+
(ref) =>
|
|
1924
|
+
isRecord(ref) &&
|
|
1925
|
+
isSafeBoundedInteger(ref.flatIndex, 0, MAX_RECOVERY_SOURCE_INDEX) &&
|
|
1926
|
+
isSafeBoundedInteger(ref.turnIndex, 0, MAX_RECOVERY_SOURCE_INDEX) &&
|
|
1927
|
+
isSafeBoundedInteger(ref.itemIndex, 0, MAX_RECOVERY_SOURCE_INDEX),
|
|
1928
|
+
)
|
|
1929
|
+
.sort((left, right) => left.flatIndex - right.flatIndex);
|
|
1930
|
+
const unique = [];
|
|
1931
|
+
const seen = new Set();
|
|
1932
|
+
for (const ref of sorted) {
|
|
1933
|
+
const key = `${ref.flatIndex}:${ref.turnIndex}:${ref.itemIndex}:${String(ref.sourceItemId ?? '')}`;
|
|
1934
|
+
if (seen.has(key)) continue;
|
|
1935
|
+
seen.add(key);
|
|
1936
|
+
unique.push(ref);
|
|
1937
|
+
}
|
|
1938
|
+
const ranges = [];
|
|
1939
|
+
for (const ref of unique) {
|
|
1940
|
+
const previous = ranges.at(-1);
|
|
1941
|
+
if (previous && ref.flatIndex === previous.end + 1) previous.end = ref.flatIndex;
|
|
1942
|
+
else ranges.push({ start: ref.flatIndex, end: ref.flatIndex });
|
|
1943
|
+
}
|
|
1944
|
+
return {
|
|
1945
|
+
positions: ranges
|
|
1946
|
+
.map((range) =>
|
|
1947
|
+
range.start === range.end ? String(range.start) : `${range.start}-${range.end}`,
|
|
1948
|
+
)
|
|
1949
|
+
.join(','),
|
|
1950
|
+
locations: unique.map((ref) => `${ref.turnIndex}:${ref.itemIndex}`).join(','),
|
|
1951
|
+
ids: unique
|
|
1952
|
+
.map((ref) =>
|
|
1953
|
+
typeof ref.sourceItemId === 'string' ? escapeReference(ref.sourceItemId) : 'unknown',
|
|
1954
|
+
)
|
|
1955
|
+
.join(','),
|
|
1956
|
+
};
|
|
1957
|
+
};
|
|
613
1958
|
const selected = new Map(
|
|
614
1959
|
decisions.filter((decision) => decision.keep).map((decision) => [decision.id, decision]),
|
|
615
1960
|
);
|
|
@@ -623,6 +1968,7 @@ export function recoveryNotesText(
|
|
|
623
1968
|
`- capture_base: ${escapeReference(captureBase)}`,
|
|
624
1969
|
`- capture_file: ${escapeReference(captureFile)} (private checkpoint directory)`,
|
|
625
1970
|
`- capture_source_sha256: ${revision}`,
|
|
1971
|
+
'- capture_verified: immutable private capture read and revision checked before native compaction',
|
|
626
1972
|
`- source: private local capture`,
|
|
627
1973
|
'',
|
|
628
1974
|
'## Selected context',
|
|
@@ -631,10 +1977,16 @@ export function recoveryNotesText(
|
|
|
631
1977
|
let inlineBudget = Math.floor(MAX_NOTE_BYTES / 2);
|
|
632
1978
|
for (const item of candidates.items) {
|
|
633
1979
|
if (!selected.has(item.id)) continue;
|
|
634
|
-
const reference =
|
|
1980
|
+
const reference = item.sourceRefs
|
|
1981
|
+
? (() => {
|
|
1982
|
+
const source = formatSourceLocations(item.sourceRefs);
|
|
1983
|
+
return `[protected reference bundle retained in verified capture ${escapeReference(captureFile)}; source_positions=${source.positions}; thread_locations=${source.locations}; source_ids=${source.ids}]`;
|
|
1984
|
+
})()
|
|
1985
|
+
: `[selected local content retained in ${escapeReference(captureFile)}; source_item=${escapeReference(item.sourceItemId)}; source_location=${item.sourceLocation?.turnIndex ?? 'unknown'}:${item.sourceLocation?.itemIndex ?? 'unknown'}; local_chars=${item.localChars}]`;
|
|
635
1986
|
const inline =
|
|
636
1987
|
item.kind !== 'tool_result' &&
|
|
637
1988
|
item.kind !== 'reference' &&
|
|
1989
|
+
item.bodyComplete !== false &&
|
|
638
1990
|
item.localText.length > 0 &&
|
|
639
1991
|
item.localText.length <= MAX_NOTE_ITEM_CHARS &&
|
|
640
1992
|
Buffer.byteLength(item.localText, 'utf8') <= inlineBudget;
|
|
@@ -658,27 +2010,6 @@ export function recoveryNotesText(
|
|
|
658
2010
|
return `${lines.join('\n')}`;
|
|
659
2011
|
}
|
|
660
2012
|
|
|
661
|
-
function recoveryInjectionState(read, notes) {
|
|
662
|
-
const matches = read.thread.turns
|
|
663
|
-
.flatMap((turn) => (Array.isArray(turn.items) ? turn.items : []))
|
|
664
|
-
.filter(
|
|
665
|
-
(item) =>
|
|
666
|
-
isRecord(item) &&
|
|
667
|
-
((item.type === 'message' &&
|
|
668
|
-
item.role === 'user' &&
|
|
669
|
-
Array.isArray(item.content) &&
|
|
670
|
-
item.content.some(
|
|
671
|
-
(part) => isRecord(part) && part.type === 'input_text' && part.text === notes,
|
|
672
|
-
)) ||
|
|
673
|
-
(item.type === 'userMessage' &&
|
|
674
|
-
Array.isArray(item.content) &&
|
|
675
|
-
item.content.some(
|
|
676
|
-
(part) => isRecord(part) && part.type === 'text' && part.text === notes,
|
|
677
|
-
))),
|
|
678
|
-
);
|
|
679
|
-
return matches.length === 0 ? 'absent' : matches.length === 1 ? 'present' : 'ambiguous';
|
|
680
|
-
}
|
|
681
|
-
|
|
682
2013
|
/** Own a Codex app-server process and request native compaction after a fresh usage event. */
|
|
683
2014
|
export function createCodexContextProxy({
|
|
684
2015
|
child,
|
|
@@ -698,6 +2029,13 @@ export function createCodexContextProxy({
|
|
|
698
2029
|
},
|
|
699
2030
|
recoveryEnabled = process.env.CODEX_CONTEXT_RECOVERY_ENABLED === '1',
|
|
700
2031
|
recoveryDisclosed = process.env.CODEX_CONTEXT_RECOVERY_DISCLOSED === '1',
|
|
2032
|
+
telemetryEnabled = process.env.CODEX_CONTEXT_TELEMETRY_ENABLED === '1',
|
|
2033
|
+
telemetryDisclosed = process.env.CODEX_CONTEXT_TELEMETRY_DISCLOSED === '1',
|
|
2034
|
+
telemetryTimeoutMs = TELEMETRY_TIMEOUT_MS,
|
|
2035
|
+
telemetrySetTimer = setTimeout,
|
|
2036
|
+
telemetryClearTimer = clearTimeout,
|
|
2037
|
+
budgetEnabled = process.env.CODEX_CONTEXT_BUDGET_ENABLED !== '0',
|
|
2038
|
+
createDiagnosticSink = null,
|
|
701
2039
|
brainServer = process.env.AI_BRAIN_MCP_SERVER ?? 'brain',
|
|
702
2040
|
clientVersion = process.env.CODEX_CLIENT_VERSION ?? '0.161.0',
|
|
703
2041
|
}) {
|
|
@@ -705,12 +2043,90 @@ export function createCodexContextProxy({
|
|
|
705
2043
|
const effectiveThreshold = contextPolicy?.compactAtPercent ?? threshold;
|
|
706
2044
|
const threads = new Map();
|
|
707
2045
|
const ownedRequests = new Map();
|
|
2046
|
+
const ownedThreadIds = new Set();
|
|
2047
|
+
const clientRequestBindings = new Map();
|
|
708
2048
|
const internalReads = new Map();
|
|
709
2049
|
const retiredRequests = new Set();
|
|
2050
|
+
const telemetryByThread = new Map();
|
|
2051
|
+
const diagnosticSinks = new Map();
|
|
710
2052
|
let stopped = false;
|
|
711
2053
|
function emitDiagnostic(event) {
|
|
712
2054
|
try {
|
|
713
|
-
|
|
2055
|
+
const phase = typeof event?.phase === 'string' ? event.phase : '';
|
|
2056
|
+
const stage =
|
|
2057
|
+
event?.stage ??
|
|
2058
|
+
(phase === 'selection_accepted' || phase === 'select' ? 'selection' : null) ??
|
|
2059
|
+
(phase === 'read' || phase === 'capture' ? 'capture' : null) ??
|
|
2060
|
+
(phase === 'compact' || phase === 'native' || phase === 'thread_failure'
|
|
2061
|
+
? 'native'
|
|
2062
|
+
: null) ??
|
|
2063
|
+
(phase === 'inject' || phase === 'recovery' ? 'recovery' : null) ??
|
|
2064
|
+
(phase === 'transport_end' || phase === 'transport_close' || phase === 'process_error'
|
|
2065
|
+
? 'transport'
|
|
2066
|
+
: null);
|
|
2067
|
+
const reason =
|
|
2068
|
+
event?.reason ??
|
|
2069
|
+
(DIAGNOSTIC_REASONS.has(event?.reasonCode) ? event.reasonCode : null) ??
|
|
2070
|
+
(phase === 'selection_accepted' ? 'completed' : null) ??
|
|
2071
|
+
(phase === 'transport_close' || phase === 'transport_end' ? 'disconnected' : null) ??
|
|
2072
|
+
(phase === 'thread_failure' ? 'protocol_error' : null) ??
|
|
2073
|
+
(event?.errorCode !== undefined ? 'protocol_error' : null);
|
|
2074
|
+
const normalized = normalizeCodexDiagnostic(
|
|
2075
|
+
{
|
|
2076
|
+
...event,
|
|
2077
|
+
stage,
|
|
2078
|
+
reason,
|
|
2079
|
+
operationId: event?.operationId ?? threads.get(event?.threadId)?.pending?.operationId,
|
|
2080
|
+
},
|
|
2081
|
+
{ threadId: event?.threadId },
|
|
2082
|
+
);
|
|
2083
|
+
if (normalized === null) return;
|
|
2084
|
+
if (
|
|
2085
|
+
typeof event?.schemaVersion === 'number' &&
|
|
2086
|
+
isSafeBoundedInteger(event.schemaVersion, 1, 2)
|
|
2087
|
+
)
|
|
2088
|
+
normalized.schemaVersion = event.schemaVersion;
|
|
2089
|
+
for (const [source, target] of [
|
|
2090
|
+
['optionsHash', 'options_hash'],
|
|
2091
|
+
['fittingStage', 'fitting_stage'],
|
|
2092
|
+
['batchCount', 'batch_count'],
|
|
2093
|
+
['providerModel', 'provider_model'],
|
|
2094
|
+
]) {
|
|
2095
|
+
if (
|
|
2096
|
+
(typeof event[source] === 'string' && event[source].length <= MAX_DIAGNOSTIC_STRING) ||
|
|
2097
|
+
(source === 'batchCount' && isSafeBoundedInteger(event[source], 0, 4)) ||
|
|
2098
|
+
(source === 'providerModel' && event[source] === null)
|
|
2099
|
+
)
|
|
2100
|
+
normalized[target] = event[source];
|
|
2101
|
+
if (source === 'optionsHash' && typeof event[source] === 'string')
|
|
2102
|
+
normalized.optionsHash = event[source];
|
|
2103
|
+
if (source === 'fittingStage' && typeof event[source] === 'string')
|
|
2104
|
+
normalized.fittingStage = event[source];
|
|
2105
|
+
if (source === 'batchCount' && isSafeBoundedInteger(event[source], 0, 4))
|
|
2106
|
+
normalized.batchCount = event[source];
|
|
2107
|
+
if (
|
|
2108
|
+
source === 'providerModel' &&
|
|
2109
|
+
(typeof event[source] === 'string' || event[source] === null)
|
|
2110
|
+
)
|
|
2111
|
+
normalized.providerModel = event[source];
|
|
2112
|
+
}
|
|
2113
|
+
onDiagnostic(normalized);
|
|
2114
|
+
if (typeof createDiagnosticSink === 'function' && typeof event?.threadId === 'string') {
|
|
2115
|
+
let sink = diagnosticSinks.get(event.threadId);
|
|
2116
|
+
if (!sink) {
|
|
2117
|
+
try {
|
|
2118
|
+
sink = createDiagnosticSink(event.threadId);
|
|
2119
|
+
if (typeof sink === 'function') diagnosticSinks.set(event.threadId, sink);
|
|
2120
|
+
} catch {
|
|
2121
|
+
sink = null;
|
|
2122
|
+
}
|
|
2123
|
+
}
|
|
2124
|
+
try {
|
|
2125
|
+
sink?.(normalized);
|
|
2126
|
+
} catch {
|
|
2127
|
+
/* private diagnostics cannot affect compaction */
|
|
2128
|
+
}
|
|
2129
|
+
}
|
|
714
2130
|
} catch {
|
|
715
2131
|
/* diagnostics must never affect compaction */
|
|
716
2132
|
}
|
|
@@ -738,9 +2154,10 @@ export function createCodexContextProxy({
|
|
|
738
2154
|
attachment.invalidated = true;
|
|
739
2155
|
cleanupRequest(attachment.readId);
|
|
740
2156
|
cleanupRequest(attachment.injectId);
|
|
2157
|
+
for (const id of attachment.auxIds ?? []) cleanupRequest(id);
|
|
741
2158
|
}
|
|
742
2159
|
|
|
743
|
-
function fail(threadId, reason) {
|
|
2160
|
+
function fail(threadId, reason, cause = null) {
|
|
744
2161
|
const state = threads.get(threadId);
|
|
745
2162
|
const pending = state?.pending;
|
|
746
2163
|
invalidateAttachment(state);
|
|
@@ -751,7 +2168,17 @@ export function createCodexContextProxy({
|
|
|
751
2168
|
cleanupRequest(pending.compactId);
|
|
752
2169
|
pending.deferred.reject(new Error(reason));
|
|
753
2170
|
state.pending = null;
|
|
754
|
-
|
|
2171
|
+
const operationId = pending.operationId;
|
|
2172
|
+
if (operationId) telemetryFor(threadId).failed({ operationId, reason: 'native_error' });
|
|
2173
|
+
emitDiagnostic({
|
|
2174
|
+
phase: 'thread_failure',
|
|
2175
|
+
stage: pending.stage ?? 'native',
|
|
2176
|
+
reason: diagnosticReasonForError(cause ?? new Error(reason)),
|
|
2177
|
+
threadId,
|
|
2178
|
+
operationId,
|
|
2179
|
+
...(boundedFailureCounts(cause) === undefined ? {} : { counts: boundedFailureCounts(cause) }),
|
|
2180
|
+
...(safeRpcCode(cause?.rpcCode) === null ? {} : { rpcCode: cause.rpcCode }),
|
|
2181
|
+
});
|
|
755
2182
|
stderr.write(`codex-context: native compaction preserved (${reason})\n`);
|
|
756
2183
|
}
|
|
757
2184
|
function failAll(reason) {
|
|
@@ -792,10 +2219,28 @@ export function createCodexContextProxy({
|
|
|
792
2219
|
recoveryLoaded: false,
|
|
793
2220
|
recoveryLoadPromise: null,
|
|
794
2221
|
recoveryAttach: null,
|
|
2222
|
+
telemetryConnected: false,
|
|
2223
|
+
owned: false,
|
|
2224
|
+
// Codex can publish an idle/usage snapshot before a resumed thread has
|
|
2225
|
+
// finished loading. Native MCP and compact requests sent in that gap
|
|
2226
|
+
// are rejected as invalid requests by the 0.163 app-server.
|
|
2227
|
+
resumePending: false,
|
|
2228
|
+
resumeRetryRequested: false,
|
|
2229
|
+
resumeRequests: new Set(),
|
|
795
2230
|
};
|
|
796
2231
|
threads.set(threadId, created);
|
|
797
2232
|
return created;
|
|
798
2233
|
}
|
|
2234
|
+
function claimOwnedThread(threadId) {
|
|
2235
|
+
const existing = threads.get(threadId);
|
|
2236
|
+
if (!existing && ownedThreadIds.size >= MAX_OWNED_THREAD_IDS) return null;
|
|
2237
|
+
const state = existing ?? threadState(threadId);
|
|
2238
|
+
if (state.owned) return state;
|
|
2239
|
+
if (ownedThreadIds.size >= MAX_OWNED_THREAD_IDS) return null;
|
|
2240
|
+
state.owned = true;
|
|
2241
|
+
ownedThreadIds.add(threadId);
|
|
2242
|
+
return state;
|
|
2243
|
+
}
|
|
799
2244
|
function sendOwned(method, params, kind, threadId, waiter) {
|
|
800
2245
|
if (ownedRequests.size >= MAX_OWNED_REQUESTS)
|
|
801
2246
|
throw new Error('too many pending adapter requests');
|
|
@@ -810,10 +2255,345 @@ export function createCodexContextProxy({
|
|
|
810
2255
|
}
|
|
811
2256
|
return id;
|
|
812
2257
|
}
|
|
2258
|
+
function telemetryFor(threadId) {
|
|
2259
|
+
const existing = telemetryByThread.get(threadId);
|
|
2260
|
+
if (existing) return existing;
|
|
2261
|
+
const reporter = createTelemetryReporter({
|
|
2262
|
+
client: 'codex-cli',
|
|
2263
|
+
clientVersion,
|
|
2264
|
+
sessionId: threadId,
|
|
2265
|
+
enabled: telemetryEnabled,
|
|
2266
|
+
disclosed: telemetryDisclosed,
|
|
2267
|
+
now,
|
|
2268
|
+
timeoutMs: telemetryTimeoutMs,
|
|
2269
|
+
setTimer: telemetrySetTimer,
|
|
2270
|
+
clearTimer: telemetryClearTimer,
|
|
2271
|
+
send: async (event) => {
|
|
2272
|
+
const waiter = deferred();
|
|
2273
|
+
void waiter.promise.catch(() => undefined);
|
|
2274
|
+
let id;
|
|
2275
|
+
try {
|
|
2276
|
+
id = sendOwned(
|
|
2277
|
+
'mcpServer/tool/call',
|
|
2278
|
+
{
|
|
2279
|
+
threadId,
|
|
2280
|
+
server: brainServer,
|
|
2281
|
+
tool: 'brain_context_report',
|
|
2282
|
+
arguments: event,
|
|
2283
|
+
},
|
|
2284
|
+
'telemetry',
|
|
2285
|
+
threadId,
|
|
2286
|
+
waiter,
|
|
2287
|
+
);
|
|
2288
|
+
let timer;
|
|
2289
|
+
try {
|
|
2290
|
+
const result = await Promise.race([
|
|
2291
|
+
waiter.promise,
|
|
2292
|
+
new Promise((_, reject) => {
|
|
2293
|
+
timer = telemetrySetTimer(
|
|
2294
|
+
() => reject(new Error('telemetry_timeout')),
|
|
2295
|
+
telemetryTimeoutMs,
|
|
2296
|
+
);
|
|
2297
|
+
}),
|
|
2298
|
+
]);
|
|
2299
|
+
if (!isRecord(result) || result.isError === true)
|
|
2300
|
+
throw new Error('telemetry_report_rejected');
|
|
2301
|
+
const structured = isRecord(result.structuredContent)
|
|
2302
|
+
? result.structuredContent
|
|
2303
|
+
: result;
|
|
2304
|
+
if (!isRecord(structured) || structured.accepted !== true)
|
|
2305
|
+
throw new Error('telemetry_report_rejected');
|
|
2306
|
+
return result;
|
|
2307
|
+
} finally {
|
|
2308
|
+
if (timer !== undefined) telemetryClearTimer(timer);
|
|
2309
|
+
}
|
|
2310
|
+
} finally {
|
|
2311
|
+
cleanupRequest(id);
|
|
2312
|
+
}
|
|
2313
|
+
},
|
|
2314
|
+
});
|
|
2315
|
+
telemetryByThread.set(threadId, reporter);
|
|
2316
|
+
return reporter;
|
|
2317
|
+
}
|
|
2318
|
+
async function requestBudget(threadId, pending) {
|
|
2319
|
+
const waiter = deferred();
|
|
2320
|
+
void waiter.promise.catch(() => undefined);
|
|
2321
|
+
const budgetRequestId = sendOwned(
|
|
2322
|
+
'mcpServer/tool/call',
|
|
2323
|
+
{
|
|
2324
|
+
threadId,
|
|
2325
|
+
server: brainServer,
|
|
2326
|
+
tool: 'brain_context_budget',
|
|
2327
|
+
arguments: { schema_version: 1 },
|
|
2328
|
+
},
|
|
2329
|
+
'budget',
|
|
2330
|
+
threadId,
|
|
2331
|
+
waiter,
|
|
2332
|
+
);
|
|
2333
|
+
pending.auxIds.push(budgetRequestId);
|
|
2334
|
+
emitDiagnostic({
|
|
2335
|
+
phase: 'budget',
|
|
2336
|
+
stage: 'budget',
|
|
2337
|
+
reason: 'requested',
|
|
2338
|
+
threadId,
|
|
2339
|
+
operationId: pending.operationId,
|
|
2340
|
+
});
|
|
2341
|
+
let response;
|
|
2342
|
+
try {
|
|
2343
|
+
response = await waiter.promise;
|
|
2344
|
+
} catch {
|
|
2345
|
+
emitDiagnostic({
|
|
2346
|
+
phase: 'budget',
|
|
2347
|
+
stage: 'budget',
|
|
2348
|
+
reason: 'unavailable',
|
|
2349
|
+
threadId,
|
|
2350
|
+
operationId: pending.operationId,
|
|
2351
|
+
});
|
|
2352
|
+
return { snapshot: null, error: null };
|
|
2353
|
+
}
|
|
2354
|
+
const error = parseContextError(response);
|
|
2355
|
+
if (error !== null && response?.isError === true) {
|
|
2356
|
+
emitDiagnostic({
|
|
2357
|
+
phase: 'budget',
|
|
2358
|
+
stage: 'budget',
|
|
2359
|
+
reason: error.quota ? 'admission_denied' : 'request_failed',
|
|
2360
|
+
threadId,
|
|
2361
|
+
operationId: pending.operationId,
|
|
2362
|
+
correlationId: error.correlationId,
|
|
2363
|
+
});
|
|
2364
|
+
return { snapshot: null, error };
|
|
2365
|
+
}
|
|
2366
|
+
const receivedAt = now();
|
|
2367
|
+
const snapshot = parseBudgetSnapshot(responseStructuredContent(response));
|
|
2368
|
+
if (snapshot === null) {
|
|
2369
|
+
emitDiagnostic({
|
|
2370
|
+
phase: 'budget',
|
|
2371
|
+
stage: 'budget',
|
|
2372
|
+
reason: 'stale_snapshot',
|
|
2373
|
+
threadId,
|
|
2374
|
+
operationId: pending.operationId,
|
|
2375
|
+
});
|
|
2376
|
+
return { snapshot: null, error: null };
|
|
2377
|
+
}
|
|
2378
|
+
emitDiagnostic({
|
|
2379
|
+
phase: 'budget',
|
|
2380
|
+
stage: 'budget',
|
|
2381
|
+
reason: 'completed',
|
|
2382
|
+
threadId,
|
|
2383
|
+
operationId: pending.operationId,
|
|
2384
|
+
correlationId: snapshot.correlation_id,
|
|
2385
|
+
counts: {
|
|
2386
|
+
available_requests: snapshot.available_requests,
|
|
2387
|
+
available_tokens: Math.min(snapshot.available_tokens, 1_000_000),
|
|
2388
|
+
},
|
|
2389
|
+
});
|
|
2390
|
+
return { snapshot, error: null, receivedAt };
|
|
2391
|
+
}
|
|
2392
|
+
function emitNativeFallback(threadId, pending, reason, correlationId = null) {
|
|
2393
|
+
telemetryFor(threadId).fallback({
|
|
2394
|
+
operationId: pending.operationId,
|
|
2395
|
+
reason:
|
|
2396
|
+
reason === 'admission_denied'
|
|
2397
|
+
? 'rate_limited'
|
|
2398
|
+
: reason === 'provider_unavailable'
|
|
2399
|
+
? 'provider_unavailable'
|
|
2400
|
+
: 'recovery_pending',
|
|
2401
|
+
});
|
|
2402
|
+
emitDiagnostic({
|
|
2403
|
+
phase: 'budget',
|
|
2404
|
+
stage: 'budget',
|
|
2405
|
+
reason: reason === 'provider_unavailable' ? 'request_failed' : reason,
|
|
2406
|
+
threadId,
|
|
2407
|
+
operationId: pending.operationId,
|
|
2408
|
+
correlationId,
|
|
2409
|
+
});
|
|
2410
|
+
}
|
|
2411
|
+
function emitOutcome(outcome) {
|
|
2412
|
+
try {
|
|
2413
|
+
onOutcome(outcome);
|
|
2414
|
+
} catch {
|
|
2415
|
+
/* lifecycle observers cannot affect compaction */
|
|
2416
|
+
}
|
|
2417
|
+
const reporter = telemetryFor(outcome.threadId);
|
|
2418
|
+
const operationId =
|
|
2419
|
+
outcome.operationId ??
|
|
2420
|
+
threads.get(outcome.threadId)?.recovery?.operationId ??
|
|
2421
|
+
threads.get(outcome.threadId)?.pending?.operationId;
|
|
2422
|
+
if (
|
|
2423
|
+
!outcome.telemetryAlreadyEmitted &&
|
|
2424
|
+
(outcome.status === 'native_completed' || outcome.status === 'completed') &&
|
|
2425
|
+
operationId
|
|
2426
|
+
)
|
|
2427
|
+
reporter.nativeCompleted({ operationId });
|
|
2428
|
+
else if (outcome.status === 'recovery_attached' && operationId)
|
|
2429
|
+
reporter.recoveryAttached({ operationId });
|
|
2430
|
+
else if (outcome.status === 'recovery_uncertain' && operationId)
|
|
2431
|
+
reporter.uncertain({ operationId, reason: 'recovery_uncertain' });
|
|
2432
|
+
else if (outcome.status === 'recovery_pending' && operationId)
|
|
2433
|
+
reporter.fallback({ operationId, reason: 'recovery_pending' });
|
|
2434
|
+
else if (outcome.status === 'recovery_abandoned' && operationId)
|
|
2435
|
+
reporter.fallback({ operationId, reason: 'stale_context' });
|
|
2436
|
+
else if (outcome.status === 'failed' && operationId)
|
|
2437
|
+
reporter.failed({ operationId, reason: 'native_error' });
|
|
2438
|
+
}
|
|
813
2439
|
function assertCurrent(threadId, state, pending, revision) {
|
|
814
2440
|
if (pending.canceled || state.pending !== pending || !state.idle || state.revision !== revision)
|
|
815
2441
|
throw new Error('Codex state changed during compaction');
|
|
816
2442
|
}
|
|
2443
|
+
async function readThreadCapture(
|
|
2444
|
+
threadId,
|
|
2445
|
+
pending,
|
|
2446
|
+
firstWaiter = null,
|
|
2447
|
+
{ onPage = null, readId: existingReadId = null, verifyOnly = false } = {},
|
|
2448
|
+
) {
|
|
2449
|
+
const waiter = firstWaiter ?? deferred();
|
|
2450
|
+
const readId =
|
|
2451
|
+
existingReadId ??
|
|
2452
|
+
sendOwned('thread/read', { threadId, includeTurns: false }, 'read', threadId, waiter);
|
|
2453
|
+
if (pending) {
|
|
2454
|
+
pending.readId = readId;
|
|
2455
|
+
pending.auxIds.push(readId);
|
|
2456
|
+
}
|
|
2457
|
+
const metadata = validateThreadMetadata(await waiter.promise, threadId);
|
|
2458
|
+
if (Array.isArray(metadata.thread.turns) && metadata.thread.turns.length > 0) {
|
|
2459
|
+
const legacy = validateThreadRead(metadata, threadId);
|
|
2460
|
+
await onPage?.(metadata.thread);
|
|
2461
|
+
return { read: legacy, metadata, pages: null };
|
|
2462
|
+
}
|
|
2463
|
+
const pages = [];
|
|
2464
|
+
const seenTurnIds = new Set();
|
|
2465
|
+
const seenItemIds = new Set();
|
|
2466
|
+
let totalItems = 0;
|
|
2467
|
+
if (!verifyOnly) await ensurePrivateDirectory(checkpointDir);
|
|
2468
|
+
const summaryTurns = [];
|
|
2469
|
+
const stagedFiles = [];
|
|
2470
|
+
let stagedBytes = 0;
|
|
2471
|
+
const cleanupStaged = async () => {
|
|
2472
|
+
for (const file of stagedFiles) await unlink(file).catch(() => undefined);
|
|
2473
|
+
};
|
|
2474
|
+
const seenCursors = new Set();
|
|
2475
|
+
let cursor = null;
|
|
2476
|
+
for (let pageIndex = 0; pageIndex < MAX_CAPTURE_PAGES; pageIndex += 1) {
|
|
2477
|
+
let pageResponse;
|
|
2478
|
+
let pageLimit = 16;
|
|
2479
|
+
for (let attempt = 0; attempt < 5; attempt += 1) {
|
|
2480
|
+
const pageWaiter = deferred();
|
|
2481
|
+
const params = {
|
|
2482
|
+
threadId,
|
|
2483
|
+
itemsView: 'full',
|
|
2484
|
+
sortDirection: 'asc',
|
|
2485
|
+
limit: pageLimit,
|
|
2486
|
+
};
|
|
2487
|
+
if (cursor !== null) params.cursor = cursor;
|
|
2488
|
+
const pageId = sendOwned('thread/turns/list', params, 'page', threadId, pageWaiter);
|
|
2489
|
+
if (pending) pending.auxIds.push(pageId);
|
|
2490
|
+
try {
|
|
2491
|
+
pageResponse = await pageWaiter.promise;
|
|
2492
|
+
break;
|
|
2493
|
+
} catch (error) {
|
|
2494
|
+
if (error?.diagnosticReason === 'capture_size_exceeded' && attempt < 4) {
|
|
2495
|
+
pageLimit = Math.max(1, Math.floor(pageLimit / 2));
|
|
2496
|
+
continue;
|
|
2497
|
+
}
|
|
2498
|
+
await cleanupStaged();
|
|
2499
|
+
if (error?.rpcCode === -32601 && pages.length === 0) {
|
|
2500
|
+
const legacyWaiter = deferred();
|
|
2501
|
+
const legacyId = sendOwned(
|
|
2502
|
+
'thread/read',
|
|
2503
|
+
{ threadId, includeTurns: true },
|
|
2504
|
+
'read',
|
|
2505
|
+
threadId,
|
|
2506
|
+
legacyWaiter,
|
|
2507
|
+
);
|
|
2508
|
+
if (pending) pending.auxIds.push(legacyId);
|
|
2509
|
+
const legacy = validateThreadRead(await legacyWaiter.promise, threadId);
|
|
2510
|
+
return { read: legacy, metadata, pages: null };
|
|
2511
|
+
}
|
|
2512
|
+
throw error;
|
|
2513
|
+
}
|
|
2514
|
+
}
|
|
2515
|
+
if (pageResponse === undefined) throw new Error('thread/turns/list page unavailable');
|
|
2516
|
+
try {
|
|
2517
|
+
const parsed = parseTurnsPage(pageResponse, threadId, pageIndex);
|
|
2518
|
+
if (parsed.page.turns.length === 0 && parsed.cursor !== null)
|
|
2519
|
+
throw new Error('thread/turns/list returned empty page with cursor');
|
|
2520
|
+
for (const turn of parsed.page.turns) {
|
|
2521
|
+
if (seenTurnIds.has(turn.id)) throw new Error('capture history contains duplicate turn');
|
|
2522
|
+
seenTurnIds.add(turn.id);
|
|
2523
|
+
if (!Array.isArray(turn.items)) throw new Error('capture turn items are unavailable');
|
|
2524
|
+
totalItems += turn.items.length;
|
|
2525
|
+
if (totalItems > MAX_CAPTURE_ITEMS) {
|
|
2526
|
+
const error = new Error('capture item count exceeds bound');
|
|
2527
|
+
error.captureItemCount = totalItems;
|
|
2528
|
+
error.captureItemLimit = MAX_CAPTURE_ITEMS;
|
|
2529
|
+
throw error;
|
|
2530
|
+
}
|
|
2531
|
+
for (const item of turn.items) {
|
|
2532
|
+
if (isRecord(item) && typeof item.id === 'string') {
|
|
2533
|
+
if (seenItemIds.has(item.id))
|
|
2534
|
+
throw new Error('capture history contains duplicate item');
|
|
2535
|
+
seenItemIds.add(item.id);
|
|
2536
|
+
}
|
|
2537
|
+
}
|
|
2538
|
+
}
|
|
2539
|
+
const serializedPage = `${JSON.stringify(parsed.page)}\n`;
|
|
2540
|
+
await onPage?.(parsed.page);
|
|
2541
|
+
const pageBytes = Buffer.byteLength(serializedPage, 'utf8');
|
|
2542
|
+
stagedBytes += pageBytes;
|
|
2543
|
+
if (stagedBytes > MAX_CAPTURE_TOTAL_BYTES)
|
|
2544
|
+
throw new Error('capture exceeds segmented bound');
|
|
2545
|
+
let stagedPath = null;
|
|
2546
|
+
if (!verifyOnly) {
|
|
2547
|
+
const storageState = await readCaptureDirectoryState(checkpointDir);
|
|
2548
|
+
if (
|
|
2549
|
+
storageState.totalBytes + pageBytes >
|
|
2550
|
+
MAX_CAPTURE_DIRECTORY_BYTES + MAX_CAPTURE_STAGING_HEADROOM_BYTES
|
|
2551
|
+
)
|
|
2552
|
+
throw captureCapacityError(storageState);
|
|
2553
|
+
stagedPath = resolve(
|
|
2554
|
+
checkpointDir,
|
|
2555
|
+
`.capture-stage.${safeThreadName(threadId)}.${randomUUID()}.${pageIndex}.tmp`,
|
|
2556
|
+
);
|
|
2557
|
+
await writeFile(stagedPath, serializedPage, { flag: 'wx', mode: 0o600 });
|
|
2558
|
+
stagedFiles.push(stagedPath);
|
|
2559
|
+
}
|
|
2560
|
+
pages.push({
|
|
2561
|
+
pageIndex,
|
|
2562
|
+
stagedPath,
|
|
2563
|
+
pageHash: hashJson(parsed.page),
|
|
2564
|
+
itemCount: parsed.page.turns.reduce(
|
|
2565
|
+
(count, turn) => count + (Array.isArray(turn.items) ? turn.items.length : 0),
|
|
2566
|
+
0,
|
|
2567
|
+
),
|
|
2568
|
+
bytes: pageBytes,
|
|
2569
|
+
cursor: parsed.cursor,
|
|
2570
|
+
});
|
|
2571
|
+
summaryTurns.push(...parsed.page.turns.map(summarizeCaptureTurn));
|
|
2572
|
+
if (parsed.cursor === null)
|
|
2573
|
+
return {
|
|
2574
|
+
read: {
|
|
2575
|
+
thread: {
|
|
2576
|
+
...metadata.thread,
|
|
2577
|
+
turns: summaryTurns,
|
|
2578
|
+
},
|
|
2579
|
+
},
|
|
2580
|
+
metadata,
|
|
2581
|
+
pages,
|
|
2582
|
+
stagedFiles,
|
|
2583
|
+
};
|
|
2584
|
+
if (seenCursors.has(parsed.cursor)) throw new Error('thread/turns/list cursor repeated');
|
|
2585
|
+
seenCursors.add(parsed.cursor);
|
|
2586
|
+
cursor = parsed.cursor;
|
|
2587
|
+
if (pageIndex === MAX_CAPTURE_PAGES - 1)
|
|
2588
|
+
throw new Error('thread/turns/list page count exceeds bound');
|
|
2589
|
+
} catch (error) {
|
|
2590
|
+
await cleanupStaged();
|
|
2591
|
+
throw error;
|
|
2592
|
+
}
|
|
2593
|
+
}
|
|
2594
|
+
await cleanupStaged();
|
|
2595
|
+
throw new Error('thread/turns/list page count exceeds bound');
|
|
2596
|
+
}
|
|
817
2597
|
async function loadRecoveryState(threadId, state) {
|
|
818
2598
|
if (state.recoveryLoaded) return state.recovery;
|
|
819
2599
|
if (!state.recoveryLoadPromise) {
|
|
@@ -851,14 +2631,22 @@ export function createCodexContextProxy({
|
|
|
851
2631
|
/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(
|
|
852
2632
|
value.requestId,
|
|
853
2633
|
) &&
|
|
2634
|
+
(value.operationId === undefined ||
|
|
2635
|
+
(typeof value.operationId === 'string' &&
|
|
2636
|
+
/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(
|
|
2637
|
+
value.operationId,
|
|
2638
|
+
))) &&
|
|
854
2639
|
typeof value.itemId === 'string' &&
|
|
855
2640
|
value.itemId.length <= 128 &&
|
|
856
2641
|
(!['pending_attachment', 'injecting', 'recovery_uncertain'].includes(value.status) ||
|
|
857
2642
|
typeof value.turnId === 'string')
|
|
858
2643
|
)
|
|
859
|
-
return
|
|
860
|
-
|
|
861
|
-
|
|
2644
|
+
return {
|
|
2645
|
+
...(value.status === 'injecting'
|
|
2646
|
+
? { ...value, status: 'recovery_uncertain' }
|
|
2647
|
+
: value),
|
|
2648
|
+
operationId: value.operationId ?? value.requestId,
|
|
2649
|
+
};
|
|
862
2650
|
} catch (error) {
|
|
863
2651
|
if (error?.code !== 'ENOENT') stderr.write('codex-context: recovery state unavailable\n');
|
|
864
2652
|
}
|
|
@@ -874,6 +2662,7 @@ export function createCodexContextProxy({
|
|
|
874
2662
|
await atomicWritePrivate(
|
|
875
2663
|
resolve(checkpointDir, `${safeThreadName(threadId)}.recovery.json`),
|
|
876
2664
|
record,
|
|
2665
|
+
{ captureLock: true },
|
|
877
2666
|
);
|
|
878
2667
|
}
|
|
879
2668
|
async function readRecoveryNotes(threadId, record) {
|
|
@@ -912,6 +2701,7 @@ export function createCodexContextProxy({
|
|
|
912
2701
|
injectSent: false,
|
|
913
2702
|
readId: null,
|
|
914
2703
|
injectId: null,
|
|
2704
|
+
auxIds: [],
|
|
915
2705
|
deferred: injectWaiter,
|
|
916
2706
|
timer: null,
|
|
917
2707
|
};
|
|
@@ -921,23 +2711,44 @@ export function createCodexContextProxy({
|
|
|
921
2711
|
attachment.canceled = true;
|
|
922
2712
|
cleanupRequest(attachment.readId);
|
|
923
2713
|
cleanupRequest(attachment.injectId);
|
|
2714
|
+
for (const id of attachment.auxIds) cleanupRequest(id);
|
|
924
2715
|
injectWaiter.reject(new Error('recovery attachment timed out'));
|
|
925
2716
|
}, timeoutMs);
|
|
926
2717
|
try {
|
|
927
2718
|
const notes = await readRecoveryNotes(threadId, record);
|
|
928
2719
|
if (attachment.invalidated || attachment.canceled)
|
|
929
2720
|
throw new Error('recovery attachment was invalidated before read');
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
2721
|
+
let latest = null;
|
|
2722
|
+
let injection = 'absent';
|
|
2723
|
+
const capture = await readThreadCapture(threadId, attachment, readWaiter, {
|
|
2724
|
+
verifyOnly: true,
|
|
2725
|
+
onPage: async (page) => {
|
|
2726
|
+
const pageLatest = page.turns?.at(-1);
|
|
2727
|
+
if (pageLatest) latest = pageLatest;
|
|
2728
|
+
const found = page.turns
|
|
2729
|
+
?.flatMap((turn) => (Array.isArray(turn.items) ? turn.items : []))
|
|
2730
|
+
.some(
|
|
2731
|
+
(item) =>
|
|
2732
|
+
isRecord(item) &&
|
|
2733
|
+
((item.type === 'message' &&
|
|
2734
|
+
item.role === 'user' &&
|
|
2735
|
+
Array.isArray(item.content) &&
|
|
2736
|
+
item.content.some(
|
|
2737
|
+
(part) => isRecord(part) && part.type === 'input_text' && part.text === notes,
|
|
2738
|
+
)) ||
|
|
2739
|
+
(item.type === 'userMessage' &&
|
|
2740
|
+
Array.isArray(item.content) &&
|
|
2741
|
+
item.content.some(
|
|
2742
|
+
(part) => isRecord(part) && part.type === 'text' && part.text === notes,
|
|
2743
|
+
))),
|
|
2744
|
+
);
|
|
2745
|
+
if (found) injection = injection === 'present' ? 'ambiguous' : 'present';
|
|
2746
|
+
},
|
|
2747
|
+
});
|
|
2748
|
+
for (const descriptor of capture.pages ?? [])
|
|
2749
|
+
if (descriptor.stagedPath) await unlink(descriptor.stagedPath).catch(() => undefined);
|
|
938
2750
|
if (attachment.invalidated || attachment.canceled)
|
|
939
2751
|
throw new Error('recovery attachment was invalidated during read');
|
|
940
|
-
const latest = read.thread.turns.at(-1);
|
|
941
2752
|
if (!latest || latest.id !== record.turnId || latest.status !== 'completed') {
|
|
942
2753
|
attachment.stale = true;
|
|
943
2754
|
throw new Error('recovery compaction turn is no longer current');
|
|
@@ -946,7 +2757,6 @@ export function createCodexContextProxy({
|
|
|
946
2757
|
attachment.stale = true;
|
|
947
2758
|
throw new Error('recovery attachment encountered a newer turn');
|
|
948
2759
|
}
|
|
949
|
-
const injection = recoveryInjectionState(read, notes);
|
|
950
2760
|
if (injection === 'ambiguous') {
|
|
951
2761
|
attachment.stale = true;
|
|
952
2762
|
throw new Error('recovery attachment has ambiguous prior injections');
|
|
@@ -962,7 +2772,14 @@ export function createCodexContextProxy({
|
|
|
962
2772
|
await persistRecoveryState(threadId, attached);
|
|
963
2773
|
if (attachment.invalidated || attachment.canceled) return;
|
|
964
2774
|
state.recovery = attached;
|
|
965
|
-
|
|
2775
|
+
emitDiagnostic({
|
|
2776
|
+
phase: 'recovery_attached',
|
|
2777
|
+
stage: 'recovery',
|
|
2778
|
+
reason: 'attached',
|
|
2779
|
+
threadId,
|
|
2780
|
+
operationId: record.operationId,
|
|
2781
|
+
});
|
|
2782
|
+
emitOutcome({ threadId, status: 'recovery_attached', turnId: record.turnId });
|
|
966
2783
|
return;
|
|
967
2784
|
}
|
|
968
2785
|
if (record.status === 'recovery_uncertain') {
|
|
@@ -973,7 +2790,14 @@ export function createCodexContextProxy({
|
|
|
973
2790
|
};
|
|
974
2791
|
await persistRecoveryState(threadId, uncertain).catch(() => undefined);
|
|
975
2792
|
state.recovery = uncertain;
|
|
976
|
-
|
|
2793
|
+
emitDiagnostic({
|
|
2794
|
+
phase: 'recovery_uncertain',
|
|
2795
|
+
stage: 'recovery',
|
|
2796
|
+
reason: 'uncertain',
|
|
2797
|
+
threadId,
|
|
2798
|
+
operationId: record.operationId,
|
|
2799
|
+
});
|
|
2800
|
+
emitOutcome({ threadId, status: 'recovery_uncertain', turnId: record.turnId });
|
|
977
2801
|
return;
|
|
978
2802
|
}
|
|
979
2803
|
if (!state.idle) throw new Error('thread became active before recovery attachment');
|
|
@@ -1015,7 +2839,14 @@ export function createCodexContextProxy({
|
|
|
1015
2839
|
};
|
|
1016
2840
|
await persistRecoveryState(threadId, attached);
|
|
1017
2841
|
state.recovery = attached;
|
|
1018
|
-
|
|
2842
|
+
emitDiagnostic({
|
|
2843
|
+
phase: 'recovery_attached',
|
|
2844
|
+
stage: 'recovery',
|
|
2845
|
+
reason: 'attached',
|
|
2846
|
+
threadId,
|
|
2847
|
+
operationId: record.operationId,
|
|
2848
|
+
});
|
|
2849
|
+
emitOutcome({ threadId, status: 'recovery_attached', turnId: record.turnId });
|
|
1019
2850
|
} catch {
|
|
1020
2851
|
if (attachment.injectSent) {
|
|
1021
2852
|
const uncertain = {
|
|
@@ -1025,7 +2856,14 @@ export function createCodexContextProxy({
|
|
|
1025
2856
|
};
|
|
1026
2857
|
await persistRecoveryState(threadId, uncertain).catch(() => undefined);
|
|
1027
2858
|
state.recovery = uncertain;
|
|
1028
|
-
|
|
2859
|
+
emitDiagnostic({
|
|
2860
|
+
phase: 'recovery_uncertain',
|
|
2861
|
+
stage: 'recovery',
|
|
2862
|
+
reason: 'uncertain',
|
|
2863
|
+
threadId,
|
|
2864
|
+
operationId: record.operationId,
|
|
2865
|
+
});
|
|
2866
|
+
emitOutcome({ threadId, status: 'recovery_uncertain', turnId: record.turnId });
|
|
1029
2867
|
} else if (!attachment.canceled) {
|
|
1030
2868
|
if (attachment.stale) {
|
|
1031
2869
|
state.recovery = {
|
|
@@ -1034,18 +2872,24 @@ export function createCodexContextProxy({
|
|
|
1034
2872
|
abandonedAt: new Date(now()).toISOString(),
|
|
1035
2873
|
};
|
|
1036
2874
|
await persistRecoveryState(threadId, state.recovery).catch(() => undefined);
|
|
1037
|
-
|
|
1038
|
-
} else
|
|
2875
|
+
emitOutcome({ threadId, status: 'recovery_abandoned', turnId: record.turnId });
|
|
2876
|
+
} else emitOutcome({ threadId, status: 'recovery_pending', turnId: record.turnId });
|
|
1039
2877
|
}
|
|
1040
2878
|
} finally {
|
|
1041
2879
|
if (attachment.timer) attachment.timer();
|
|
1042
2880
|
cleanupRequest(attachment.readId);
|
|
1043
2881
|
cleanupRequest(attachment.injectId);
|
|
2882
|
+
for (const id of attachment.auxIds) cleanupRequest(id);
|
|
1044
2883
|
if (state.recoveryAttach === attachment) state.recoveryAttach = null;
|
|
1045
2884
|
}
|
|
1046
2885
|
}
|
|
1047
2886
|
async function requestCompaction(threadId, state) {
|
|
1048
|
-
|
|
2887
|
+
if (!state.owned) return;
|
|
2888
|
+
if (state.resumePending) {
|
|
2889
|
+
state.resumeRetryRequested = true;
|
|
2890
|
+
return;
|
|
2891
|
+
}
|
|
2892
|
+
if (recoveryEnabled && recoveryDisclosed) await loadRecoveryState(threadId, state);
|
|
1049
2893
|
if (
|
|
1050
2894
|
state.recovery &&
|
|
1051
2895
|
['pending_attachment', 'recovery_uncertain'].includes(state.recovery.status)
|
|
@@ -1060,6 +2904,7 @@ export function createCodexContextProxy({
|
|
|
1060
2904
|
state.pending ||
|
|
1061
2905
|
!state.usage ||
|
|
1062
2906
|
!isValidUsage(state.usage) ||
|
|
2907
|
+
!hasKnownContextWindow(state.usage) ||
|
|
1063
2908
|
!Number.isFinite(state.usageReceivedAt) ||
|
|
1064
2909
|
currentTime < state.usageReceivedAt ||
|
|
1065
2910
|
currentTime - state.usageReceivedAt > USAGE_FRESHNESS_MS ||
|
|
@@ -1092,52 +2937,126 @@ export function createCodexContextProxy({
|
|
|
1092
2937
|
itemCompleted: false,
|
|
1093
2938
|
turnCompleted: false,
|
|
1094
2939
|
canceled: false,
|
|
2940
|
+
stage: 'capture',
|
|
2941
|
+
operationId: randomUUID(),
|
|
1095
2942
|
deferred: completion,
|
|
1096
2943
|
timer: null,
|
|
1097
2944
|
};
|
|
1098
2945
|
state.pending = pending;
|
|
1099
2946
|
pending.timer = scheduleTimeout(() => fail(threadId, 'native compaction timed out'), timeoutMs);
|
|
1100
2947
|
try {
|
|
2948
|
+
pending.stage = 'capture';
|
|
1101
2949
|
pending.readId = sendOwned(
|
|
1102
2950
|
'thread/read',
|
|
1103
|
-
{ threadId, includeTurns:
|
|
2951
|
+
{ threadId, includeTurns: false },
|
|
1104
2952
|
'read',
|
|
1105
2953
|
threadId,
|
|
1106
2954
|
readWaiter,
|
|
1107
2955
|
);
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
2956
|
+
await ensurePrivateDirectory(checkpointDir);
|
|
2957
|
+
const publication = await withCaptureStoreLock(checkpointDir, async () => {
|
|
2958
|
+
const capture = await readThreadCapture(threadId, pending, readWaiter, {
|
|
2959
|
+
readId: pending.readId,
|
|
2960
|
+
});
|
|
2961
|
+
const read = capture.read;
|
|
2962
|
+
assertCurrent(threadId, state, pending, revision);
|
|
2963
|
+
const sourceRevision = captureSourceRevision(capture);
|
|
2964
|
+
const compactSnapshot = {
|
|
2965
|
+
version: 1,
|
|
2966
|
+
status: 'requested',
|
|
2967
|
+
capturedAt: new Date(now()).toISOString(),
|
|
2968
|
+
threadId,
|
|
2969
|
+
sourceTurnId,
|
|
2970
|
+
revision,
|
|
2971
|
+
sourceRevision,
|
|
2972
|
+
statusAtCheckpoint: state.status,
|
|
2973
|
+
usage: state.usage,
|
|
2974
|
+
threadRead: read,
|
|
2975
|
+
recovery: {
|
|
2976
|
+
mode: 'codex-native-compaction',
|
|
2977
|
+
action: 'resume-the-owned-app-server-session',
|
|
2978
|
+
historyMutation: false,
|
|
2979
|
+
},
|
|
2980
|
+
};
|
|
2981
|
+
let snapshot = compactSnapshot;
|
|
2982
|
+
if (capture.pages !== null) {
|
|
2983
|
+
const manifest = await writeSegmentedCapture(checkpointDir, {
|
|
2984
|
+
threadId,
|
|
2985
|
+
sourceRevision,
|
|
2986
|
+
metadata: capture.metadata,
|
|
2987
|
+
pages: capture.pages,
|
|
2988
|
+
lockHeld: true,
|
|
2989
|
+
});
|
|
2990
|
+
snapshot = {
|
|
2991
|
+
...compactSnapshot,
|
|
2992
|
+
version: 2,
|
|
2993
|
+
threadRead: capture.metadata,
|
|
2994
|
+
captureManifest: manifest,
|
|
2995
|
+
};
|
|
2996
|
+
} else if (Buffer.byteLength(JSON.stringify(compactSnapshot), 'utf8') > MAX_CAPTURE_BYTES) {
|
|
2997
|
+
throw new Error('capture snapshot exceeds bound');
|
|
2998
|
+
}
|
|
2999
|
+
const manifestForRollback = snapshot.version === 2 ? snapshot.captureManifest : null;
|
|
3000
|
+
const immutableCapturePath = resolve(
|
|
3001
|
+
checkpointDir,
|
|
3002
|
+
`${safeThreadName(threadId)}.${sourceRevision}.json`,
|
|
3003
|
+
);
|
|
3004
|
+
try {
|
|
3005
|
+
await atomicWriteImmutablePrivate(immutableCapturePath, snapshot, sourceRevision, true);
|
|
3006
|
+
} catch (error) {
|
|
3007
|
+
if (manifestForRollback?.segments) {
|
|
3008
|
+
for (const segment of manifestForRollback.segments)
|
|
3009
|
+
await unlink(resolve(checkpointDir, segment)).catch(() => undefined);
|
|
3010
|
+
}
|
|
3011
|
+
throw error;
|
|
3012
|
+
}
|
|
3013
|
+
await atomicWritePrivate(
|
|
3014
|
+
resolve(checkpointDir, `${safeThreadName(threadId)}.json`),
|
|
3015
|
+
snapshot,
|
|
3016
|
+
{ maxBytes: MAX_CAPTURE_BYTES, compact: true, capture: true, captureLockHeld: true },
|
|
3017
|
+
);
|
|
3018
|
+
return { capture, read, sourceRevision, snapshot, immutableCapturePath };
|
|
3019
|
+
});
|
|
3020
|
+
const { capture, read, sourceRevision, snapshot, immutableCapturePath } = publication;
|
|
3021
|
+
if (capture.pages !== null && recoveryEnabled && recoveryDisclosed) {
|
|
3022
|
+
const verificationWaiter = deferred();
|
|
3023
|
+
const verificationCapture = await readThreadCapture(threadId, pending, verificationWaiter, {
|
|
3024
|
+
verifyOnly: true,
|
|
3025
|
+
});
|
|
3026
|
+
try {
|
|
3027
|
+
if (captureSourceRevision(verificationCapture) !== sourceRevision)
|
|
3028
|
+
throw new Error('recovery source changed before provider admission');
|
|
3029
|
+
} finally {
|
|
3030
|
+
for (const descriptor of verificationCapture.pages ?? [])
|
|
3031
|
+
if (descriptor.stagedPath) await unlink(descriptor.stagedPath).catch(() => undefined);
|
|
3032
|
+
}
|
|
3033
|
+
}
|
|
3034
|
+
emitDiagnostic({
|
|
3035
|
+
phase: 'capture',
|
|
3036
|
+
stage: 'capture',
|
|
3037
|
+
reason: 'completed',
|
|
1115
3038
|
threadId,
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
3039
|
+
operationId: pending.operationId,
|
|
3040
|
+
counts: {
|
|
3041
|
+
item_count: read.thread.turns.reduce(
|
|
3042
|
+
(count, turn) => count + (turn.items?.length ?? 0),
|
|
3043
|
+
0,
|
|
3044
|
+
),
|
|
3045
|
+
...(capture.pages === null
|
|
3046
|
+
? {}
|
|
3047
|
+
: {
|
|
3048
|
+
page_count: capture.pages.length,
|
|
3049
|
+
capture_bytes: capture.pages.reduce(
|
|
3050
|
+
(count, descriptor) => count + descriptor.bytes,
|
|
3051
|
+
0,
|
|
3052
|
+
),
|
|
3053
|
+
}),
|
|
1126
3054
|
},
|
|
1127
|
-
};
|
|
1128
|
-
if (Buffer.byteLength(JSON.stringify(snapshot)) > MAX_CAPTURE_BYTES)
|
|
1129
|
-
throw new Error('thread snapshot exceeds bound');
|
|
1130
|
-
await atomicWritePrivate(
|
|
1131
|
-
resolve(checkpointDir, `${safeThreadName(threadId)}.json`),
|
|
1132
|
-
snapshot,
|
|
1133
|
-
);
|
|
1134
|
-
const immutableCapturePath = resolve(
|
|
1135
|
-
checkpointDir,
|
|
1136
|
-
`${safeThreadName(threadId)}.${sourceRevision}.json`,
|
|
1137
|
-
);
|
|
1138
|
-
await atomicWriteImmutablePrivate(immutableCapturePath, snapshot, sourceRevision);
|
|
3055
|
+
});
|
|
1139
3056
|
assertCurrent(threadId, state, pending, revision);
|
|
1140
3057
|
let recoveryNotes;
|
|
3058
|
+
let nativeFallback = false;
|
|
3059
|
+
let budgetReceipt = null;
|
|
1141
3060
|
if (recoveryEnabled && recoveryDisclosed) {
|
|
1142
3061
|
strictThreadName(threadId);
|
|
1143
3062
|
if (
|
|
@@ -1155,127 +3074,317 @@ export function createCodexContextProxy({
|
|
|
1155
3074
|
await persistRecoveryState(threadId, state.recovery).catch(() => undefined);
|
|
1156
3075
|
throw new Error('saved recovery selection is stale');
|
|
1157
3076
|
} else {
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
3077
|
+
let candidates;
|
|
3078
|
+
if (budgetEnabled) {
|
|
3079
|
+
pending.stage = 'budget';
|
|
3080
|
+
const budget = await requestBudget(threadId, pending);
|
|
3081
|
+
budgetReceipt = budget;
|
|
3082
|
+
if (budget.error?.quota === true) {
|
|
3083
|
+
nativeFallback = true;
|
|
3084
|
+
emitNativeFallback(threadId, pending, 'admission_denied', budget.error.correlationId);
|
|
3085
|
+
} else if (budget.snapshot === null) {
|
|
3086
|
+
nativeFallback = true;
|
|
3087
|
+
emitNativeFallback(threadId, pending, 'unavailable', null);
|
|
3088
|
+
} else if (!budgetSnapshotFresh(budget.snapshot, budget.receivedAt, now())) {
|
|
3089
|
+
nativeFallback = true;
|
|
3090
|
+
emitNativeFallback(
|
|
3091
|
+
threadId,
|
|
3092
|
+
pending,
|
|
3093
|
+
'stale_snapshot',
|
|
3094
|
+
budget.snapshot.correlation_id,
|
|
3095
|
+
);
|
|
3096
|
+
} else if (
|
|
3097
|
+
budget.snapshot.account_in_flight === true ||
|
|
3098
|
+
budget.snapshot.available_requests < 1 ||
|
|
3099
|
+
budget.snapshot.available_tokens < 1 ||
|
|
3100
|
+
budget.snapshot.max_batches_per_operation < 1
|
|
3101
|
+
) {
|
|
3102
|
+
nativeFallback = true;
|
|
3103
|
+
emitNativeFallback(
|
|
3104
|
+
threadId,
|
|
3105
|
+
pending,
|
|
3106
|
+
'admission_denied',
|
|
3107
|
+
budget.snapshot.correlation_id,
|
|
3108
|
+
);
|
|
3109
|
+
} else {
|
|
3110
|
+
try {
|
|
3111
|
+
candidates = buildRecoveryCandidates(read, sourceRevision, contextPolicy, {
|
|
3112
|
+
allowWindow: true,
|
|
3113
|
+
});
|
|
3114
|
+
const bounded = boundCandidatesToBudget(
|
|
3115
|
+
candidates,
|
|
3116
|
+
{
|
|
3117
|
+
tokens: budget.snapshot.available_tokens,
|
|
3118
|
+
requests: Math.min(
|
|
3119
|
+
budget.snapshot.available_requests,
|
|
3120
|
+
budget.snapshot.max_batches_per_operation,
|
|
3121
|
+
),
|
|
3122
|
+
maxRequestTokens: contextPolicy?.maxRequestTokens ?? 30_000,
|
|
3123
|
+
outputTokenAllowance: budget.snapshot.output_token_allowance,
|
|
3124
|
+
},
|
|
3125
|
+
contextPolicy,
|
|
3126
|
+
);
|
|
3127
|
+
if (bounded.bounded) {
|
|
3128
|
+
emitDiagnostic({
|
|
3129
|
+
phase: 'candidate_window_bounded',
|
|
3130
|
+
stage: 'selection',
|
|
3131
|
+
reason: 'candidate_window_bounded',
|
|
3132
|
+
threadId,
|
|
3133
|
+
operationId: pending.operationId,
|
|
3134
|
+
correlationId: budget.snapshot.correlation_id,
|
|
3135
|
+
counts: {
|
|
3136
|
+
candidate_count: bounded.items.length,
|
|
3137
|
+
deferred_count: Math.max(0, candidates.items.length - bounded.items.length),
|
|
3138
|
+
provider_batch_count: bounded.preflight.batches.length,
|
|
3139
|
+
provider_request_tokens: bounded.preflight.requestTokens,
|
|
3140
|
+
},
|
|
3141
|
+
});
|
|
3142
|
+
}
|
|
3143
|
+
candidates = bounded;
|
|
3144
|
+
} catch (error) {
|
|
3145
|
+
nativeFallback = true;
|
|
3146
|
+
emitNativeFallback(
|
|
3147
|
+
threadId,
|
|
3148
|
+
pending,
|
|
3149
|
+
error?.message?.includes('protected')
|
|
3150
|
+
? 'protected_over_bound'
|
|
3151
|
+
: 'candidate_over_bound',
|
|
3152
|
+
budget.snapshot.correlation_id,
|
|
3153
|
+
);
|
|
1171
3154
|
}
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
};
|
|
3155
|
+
}
|
|
3156
|
+
} else {
|
|
3157
|
+
pending.stage = 'selection';
|
|
3158
|
+
try {
|
|
3159
|
+
candidates = buildRecoveryCandidates(read, sourceRevision, contextPolicy);
|
|
3160
|
+
} catch {
|
|
3161
|
+
nativeFallback = true;
|
|
3162
|
+
emitNativeFallback(threadId, pending, 'candidate_over_bound');
|
|
3163
|
+
}
|
|
3164
|
+
}
|
|
1183
3165
|
if (
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
)
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
pending.auxIds.push(
|
|
1192
|
-
sendOwned(
|
|
1193
|
-
'mcpServer/tool/call',
|
|
1194
|
-
{
|
|
1195
|
-
threadId,
|
|
1196
|
-
server: brainServer,
|
|
1197
|
-
tool: 'brain_context_select',
|
|
1198
|
-
arguments: selectionInput,
|
|
1199
|
-
},
|
|
1200
|
-
'select',
|
|
3166
|
+
!nativeFallback &&
|
|
3167
|
+
budgetReceipt?.snapshot !== null &&
|
|
3168
|
+
budgetReceipt?.snapshot !== undefined &&
|
|
3169
|
+
!budgetSnapshotFresh(budgetReceipt.snapshot, budgetReceipt.receivedAt, now())
|
|
3170
|
+
) {
|
|
3171
|
+
nativeFallback = true;
|
|
3172
|
+
emitNativeFallback(
|
|
1201
3173
|
threadId,
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
const decisions = parseSelection(selectionPayload, selectionInput, candidates, now());
|
|
1210
|
-
if (selectionInput.schema_version === 2) {
|
|
3174
|
+
pending,
|
|
3175
|
+
'stale_snapshot',
|
|
3176
|
+
budgetReceipt.snapshot.correlation_id,
|
|
3177
|
+
);
|
|
3178
|
+
}
|
|
3179
|
+
if (nativeFallback) candidates = null;
|
|
3180
|
+
if (candidates === null || candidates === undefined) {
|
|
1211
3181
|
emitDiagnostic({
|
|
1212
|
-
phase: '
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
: null,
|
|
1218
|
-
fittingStage:
|
|
1219
|
-
isRecord(selectionPayload) && isRecord(selectionPayload.fitting)
|
|
1220
|
-
? selectionPayload.fitting.stage
|
|
1221
|
-
: null,
|
|
1222
|
-
batchCount:
|
|
1223
|
-
isRecord(selectionPayload) && isRecord(selectionPayload.stats)
|
|
1224
|
-
? selectionPayload.stats.batch_count
|
|
1225
|
-
: null,
|
|
1226
|
-
providerModel:
|
|
1227
|
-
isRecord(selectionPayload) &&
|
|
1228
|
-
(typeof selectionPayload.provider_model === 'string' ||
|
|
1229
|
-
selectionPayload.provider_model === null)
|
|
1230
|
-
? selectionPayload.provider_model
|
|
1231
|
-
: null,
|
|
3182
|
+
phase: 'selection',
|
|
3183
|
+
stage: 'selection',
|
|
3184
|
+
reason: 'unavailable',
|
|
3185
|
+
threadId,
|
|
3186
|
+
operationId: pending.operationId,
|
|
1232
3187
|
});
|
|
3188
|
+
} else {
|
|
3189
|
+
pending.stage = 'selection';
|
|
3190
|
+
const selectionInput = contextPolicy
|
|
3191
|
+
? {
|
|
3192
|
+
schema_version: 2,
|
|
3193
|
+
client: 'codex_app_server',
|
|
3194
|
+
client_version: clientVersion,
|
|
3195
|
+
session_ref: threadId,
|
|
3196
|
+
revision: sourceRevision,
|
|
3197
|
+
observed_at: new Date(now()).toISOString(),
|
|
3198
|
+
request_id: pending.operationId,
|
|
3199
|
+
policy_version: 'context-recovery-v2',
|
|
3200
|
+
options: selectionOptions(contextPolicy),
|
|
3201
|
+
projection: candidates.projection,
|
|
3202
|
+
}
|
|
3203
|
+
: {
|
|
3204
|
+
schema_version: 1,
|
|
3205
|
+
client: 'codex_app_server',
|
|
3206
|
+
client_version: clientVersion,
|
|
3207
|
+
session_ref: threadId,
|
|
3208
|
+
revision: sourceRevision,
|
|
3209
|
+
observed_at: new Date(now()).toISOString(),
|
|
3210
|
+
request_id: pending.operationId,
|
|
3211
|
+
policy_version: 'context-recovery-v1',
|
|
3212
|
+
projection: candidates.projection,
|
|
3213
|
+
};
|
|
3214
|
+
if (
|
|
3215
|
+
typeof clientVersion !== 'string' ||
|
|
3216
|
+
clientVersion.length < 1 ||
|
|
3217
|
+
clientVersion.length > 64 ||
|
|
3218
|
+
Buffer.byteLength(JSON.stringify(selectionInput), 'utf8') > MAX_SELECTION_BYTES
|
|
3219
|
+
)
|
|
3220
|
+
throw new Error('recovery selection input exceeds the contract bound');
|
|
3221
|
+
const selectionWaiter = deferred();
|
|
3222
|
+
pending.auxIds.push(
|
|
3223
|
+
sendOwned(
|
|
3224
|
+
'mcpServer/tool/call',
|
|
3225
|
+
{
|
|
3226
|
+
threadId,
|
|
3227
|
+
server: brainServer,
|
|
3228
|
+
tool: 'brain_context_select',
|
|
3229
|
+
arguments: selectionInput,
|
|
3230
|
+
},
|
|
3231
|
+
'select',
|
|
3232
|
+
threadId,
|
|
3233
|
+
selectionWaiter,
|
|
3234
|
+
),
|
|
3235
|
+
);
|
|
3236
|
+
const selectionResponse = await selectionWaiter.promise;
|
|
3237
|
+
const selectionError = parseContextError(selectionResponse);
|
|
3238
|
+
if (!isRecord(selectionResponse) || selectionResponse.isError === true) {
|
|
3239
|
+
if (selectionError?.quota === true) {
|
|
3240
|
+
emitNativeFallback(
|
|
3241
|
+
threadId,
|
|
3242
|
+
pending,
|
|
3243
|
+
'admission_denied',
|
|
3244
|
+
selectionError.correlationId,
|
|
3245
|
+
);
|
|
3246
|
+
nativeFallback = true;
|
|
3247
|
+
} else if (selectionError !== null) {
|
|
3248
|
+
emitNativeFallback(
|
|
3249
|
+
threadId,
|
|
3250
|
+
pending,
|
|
3251
|
+
'provider_unavailable',
|
|
3252
|
+
selectionError.correlationId,
|
|
3253
|
+
);
|
|
3254
|
+
nativeFallback = true;
|
|
3255
|
+
} else {
|
|
3256
|
+
emitDiagnostic({
|
|
3257
|
+
phase: 'selection',
|
|
3258
|
+
stage: 'selection',
|
|
3259
|
+
reason: 'request_failed',
|
|
3260
|
+
threadId,
|
|
3261
|
+
operationId: pending.operationId,
|
|
3262
|
+
correlationId: selectionError?.correlationId,
|
|
3263
|
+
});
|
|
3264
|
+
throw new Error('recovery selection MCP call failed');
|
|
3265
|
+
}
|
|
3266
|
+
}
|
|
3267
|
+
if (nativeFallback) {
|
|
3268
|
+
emitDiagnostic({
|
|
3269
|
+
phase: 'selection',
|
|
3270
|
+
stage: 'selection',
|
|
3271
|
+
reason: 'admission_denied',
|
|
3272
|
+
threadId,
|
|
3273
|
+
operationId: pending.operationId,
|
|
3274
|
+
});
|
|
3275
|
+
} else {
|
|
3276
|
+
const selectionPayload = responseStructuredContent(selectionResponse);
|
|
3277
|
+
const decisions = parseSelection(selectionPayload, selectionInput, candidates, now());
|
|
3278
|
+
const selectionStats =
|
|
3279
|
+
isRecord(selectionPayload) && isRecord(selectionPayload.stats)
|
|
3280
|
+
? selectionPayload.stats
|
|
3281
|
+
: {};
|
|
3282
|
+
telemetryFor(threadId).selectionCompleted({
|
|
3283
|
+
operationId: pending.operationId,
|
|
3284
|
+
counts: {
|
|
3285
|
+
eligible: Number.isSafeInteger(selectionStats.eligible_count)
|
|
3286
|
+
? selectionStats.eligible_count
|
|
3287
|
+
: candidates.items.filter((item) => !item.protected).length,
|
|
3288
|
+
protected: Number.isSafeInteger(selectionStats.protected_count)
|
|
3289
|
+
? selectionStats.protected_count
|
|
3290
|
+
: candidates.items.filter((item) => item.protected).length,
|
|
3291
|
+
selected: Number.isSafeInteger(selectionStats.selected_count)
|
|
3292
|
+
? selectionStats.selected_count
|
|
3293
|
+
: decisions.filter((item) => item.keep).length,
|
|
3294
|
+
},
|
|
3295
|
+
});
|
|
3296
|
+
if (selectionInput.schema_version === 2) {
|
|
3297
|
+
emitDiagnostic({
|
|
3298
|
+
phase: 'selection_accepted',
|
|
3299
|
+
schemaVersion: 2,
|
|
3300
|
+
optionsHash:
|
|
3301
|
+
isRecord(selectionPayload) && typeof selectionPayload.options_hash === 'string'
|
|
3302
|
+
? selectionPayload.options_hash
|
|
3303
|
+
: null,
|
|
3304
|
+
fittingStage:
|
|
3305
|
+
isRecord(selectionPayload) && isRecord(selectionPayload.fitting)
|
|
3306
|
+
? selectionPayload.fitting.stage
|
|
3307
|
+
: null,
|
|
3308
|
+
batchCount:
|
|
3309
|
+
isRecord(selectionPayload) && isRecord(selectionPayload.stats)
|
|
3310
|
+
? selectionPayload.stats.batch_count
|
|
3311
|
+
: null,
|
|
3312
|
+
providerModel:
|
|
3313
|
+
isRecord(selectionPayload) &&
|
|
3314
|
+
(typeof selectionPayload.provider_model === 'string' ||
|
|
3315
|
+
selectionPayload.provider_model === null)
|
|
3316
|
+
? selectionPayload.provider_model
|
|
3317
|
+
: null,
|
|
3318
|
+
counts: {
|
|
3319
|
+
provider_eligible_count: candidates.items.filter((item) => !item.protected)
|
|
3320
|
+
.length,
|
|
3321
|
+
protected_candidate_count: candidates.items.filter((item) => item.protected)
|
|
3322
|
+
.length,
|
|
3323
|
+
protected_bundle_source_count: candidates.items
|
|
3324
|
+
.filter((item) => Array.isArray(item.sourceRefs))
|
|
3325
|
+
.reduce((count, item) => count + item.sourceRefs.length, 0),
|
|
3326
|
+
},
|
|
3327
|
+
});
|
|
3328
|
+
}
|
|
3329
|
+
const rereadWaiter = deferred();
|
|
3330
|
+
const rereadCapture = await readThreadCapture(threadId, pending, rereadWaiter, {
|
|
3331
|
+
verifyOnly: true,
|
|
3332
|
+
});
|
|
3333
|
+
try {
|
|
3334
|
+
if (captureSourceRevision(rereadCapture) !== sourceRevision)
|
|
3335
|
+
throw new Error('recovery source changed during selection');
|
|
3336
|
+
assertCurrent(threadId, state, pending, revision);
|
|
3337
|
+
} finally {
|
|
3338
|
+
for (const descriptor of rereadCapture.pages ?? [])
|
|
3339
|
+
if (descriptor.stagedPath)
|
|
3340
|
+
await unlink(descriptor.stagedPath).catch(() => undefined);
|
|
3341
|
+
}
|
|
3342
|
+
recoveryNotes = recoveryNotesText(
|
|
3343
|
+
threadId,
|
|
3344
|
+
sourceRevision,
|
|
3345
|
+
selectionInput.request_id,
|
|
3346
|
+
snapshot.capturedAt,
|
|
3347
|
+
relative(recoveryDir, immutableCapturePath).replaceAll('\\', '/'),
|
|
3348
|
+
resolve(recoveryDir),
|
|
3349
|
+
candidates,
|
|
3350
|
+
decisions,
|
|
3351
|
+
);
|
|
3352
|
+
pending.stage = 'notes';
|
|
3353
|
+
await atomicWriteTextPrivate(
|
|
3354
|
+
resolve(recoveryDir, `NOTES.${strictThreadName(threadId)}.md`),
|
|
3355
|
+
recoveryNotes,
|
|
3356
|
+
);
|
|
3357
|
+
telemetryFor(threadId).notesWritten({ operationId: pending.operationId });
|
|
3358
|
+
state.recovery = {
|
|
3359
|
+
version: 1,
|
|
3360
|
+
status: 'awaiting_native',
|
|
3361
|
+
threadId,
|
|
3362
|
+
sourceTurnId,
|
|
3363
|
+
sourceRevision,
|
|
3364
|
+
requestId: selectionInput.request_id,
|
|
3365
|
+
notes: recoveryNotes,
|
|
3366
|
+
notesHash: hashText(recoveryNotes),
|
|
3367
|
+
notesFile: `NOTES.${strictThreadName(threadId)}.md`,
|
|
3368
|
+
itemId: `recovery-${sourceRevision}`,
|
|
3369
|
+
operationId: pending.operationId,
|
|
3370
|
+
createdAt: new Date(now()).toISOString(),
|
|
3371
|
+
};
|
|
3372
|
+
state.recoveryLoaded = true;
|
|
3373
|
+
await persistRecoveryState(threadId, state.recovery);
|
|
3374
|
+
emitDiagnostic({
|
|
3375
|
+
phase: 'selection',
|
|
3376
|
+
stage: 'selection',
|
|
3377
|
+
reason: 'completed',
|
|
3378
|
+
threadId,
|
|
3379
|
+
operationId: pending.operationId,
|
|
3380
|
+
correlationId: selectionPayload.correlation_id,
|
|
3381
|
+
});
|
|
3382
|
+
}
|
|
1233
3383
|
}
|
|
1234
|
-
const rereadWaiter = deferred();
|
|
1235
|
-
const rereadId = sendOwned(
|
|
1236
|
-
'thread/read',
|
|
1237
|
-
{ threadId, includeTurns: true },
|
|
1238
|
-
'read',
|
|
1239
|
-
threadId,
|
|
1240
|
-
rereadWaiter,
|
|
1241
|
-
);
|
|
1242
|
-
pending.auxIds.push(rereadId);
|
|
1243
|
-
const reread = validateThreadRead(await rereadWaiter.promise, threadId);
|
|
1244
|
-
if (hashJson(reread) !== sourceRevision)
|
|
1245
|
-
throw new Error('recovery source changed during selection');
|
|
1246
|
-
assertCurrent(threadId, state, pending, revision);
|
|
1247
|
-
recoveryNotes = recoveryNotesText(
|
|
1248
|
-
threadId,
|
|
1249
|
-
sourceRevision,
|
|
1250
|
-
selectionInput.request_id,
|
|
1251
|
-
snapshot.capturedAt,
|
|
1252
|
-
relative(recoveryDir, immutableCapturePath).replaceAll('\\', '/'),
|
|
1253
|
-
resolve(recoveryDir),
|
|
1254
|
-
candidates,
|
|
1255
|
-
decisions,
|
|
1256
|
-
);
|
|
1257
|
-
await atomicWriteTextPrivate(
|
|
1258
|
-
resolve(recoveryDir, `NOTES.${strictThreadName(threadId)}.md`),
|
|
1259
|
-
recoveryNotes,
|
|
1260
|
-
);
|
|
1261
|
-
state.recovery = {
|
|
1262
|
-
version: 1,
|
|
1263
|
-
status: 'awaiting_native',
|
|
1264
|
-
threadId,
|
|
1265
|
-
sourceTurnId,
|
|
1266
|
-
sourceRevision,
|
|
1267
|
-
requestId: selectionInput.request_id,
|
|
1268
|
-
notes: recoveryNotes,
|
|
1269
|
-
notesHash: hashText(recoveryNotes),
|
|
1270
|
-
notesFile: `NOTES.${strictThreadName(threadId)}.md`,
|
|
1271
|
-
itemId: `recovery-${sourceRevision}`,
|
|
1272
|
-
createdAt: new Date(now()).toISOString(),
|
|
1273
|
-
};
|
|
1274
|
-
state.recoveryLoaded = true;
|
|
1275
|
-
await persistRecoveryState(threadId, state.recovery);
|
|
1276
3384
|
}
|
|
1277
3385
|
assertCurrent(threadId, state, pending, revision);
|
|
1278
3386
|
}
|
|
3387
|
+
pending.stage = 'native';
|
|
1279
3388
|
pending.compactId = sendOwned('thread/compact/start', { threadId }, 'compact', threadId);
|
|
1280
3389
|
pending.lifecycleArmed = true;
|
|
1281
3390
|
await completion.promise;
|
|
@@ -1288,6 +3397,7 @@ export function createCodexContextProxy({
|
|
|
1288
3397
|
await atomicWritePrivate(
|
|
1289
3398
|
resolve(checkpointDir, `${safeThreadName(threadId)}.completed.json`),
|
|
1290
3399
|
completed,
|
|
3400
|
+
{ maxBytes: MAX_CAPTURE_BYTES, compact: true, capture: true },
|
|
1291
3401
|
);
|
|
1292
3402
|
if (recoveryNotes !== undefined) {
|
|
1293
3403
|
state.recovery = {
|
|
@@ -1297,18 +3407,36 @@ export function createCodexContextProxy({
|
|
|
1297
3407
|
completedAt: new Date(now()).toISOString(),
|
|
1298
3408
|
};
|
|
1299
3409
|
await persistRecoveryState(threadId, state.recovery);
|
|
3410
|
+
telemetryFor(threadId).nativeCompleted({ operationId: pending.operationId });
|
|
1300
3411
|
await attachRecovery(threadId, state);
|
|
1301
|
-
|
|
3412
|
+
emitOutcome({
|
|
1302
3413
|
threadId,
|
|
1303
3414
|
status: 'native_completed',
|
|
1304
3415
|
turnId: pending.turnId,
|
|
1305
|
-
|
|
3416
|
+
operationId: pending.operationId,
|
|
3417
|
+
recoveryStatus: 'pending_attachment',
|
|
3418
|
+
telemetryAlreadyEmitted: true,
|
|
1306
3419
|
});
|
|
1307
3420
|
} else {
|
|
1308
|
-
|
|
3421
|
+
emitOutcome({
|
|
3422
|
+
threadId,
|
|
3423
|
+
status: 'completed',
|
|
3424
|
+
operationId: pending.operationId,
|
|
3425
|
+
turnId: pending.turnId,
|
|
3426
|
+
});
|
|
1309
3427
|
}
|
|
1310
|
-
} catch {
|
|
1311
|
-
|
|
3428
|
+
} catch (error) {
|
|
3429
|
+
emitDiagnostic({
|
|
3430
|
+
phase: 'operation_failed',
|
|
3431
|
+
stage: pending.stage ?? 'native',
|
|
3432
|
+
reason: diagnosticReasonForError(error),
|
|
3433
|
+
threadId,
|
|
3434
|
+
operationId: pending.operationId,
|
|
3435
|
+
...(boundedFailureCounts(error) === undefined
|
|
3436
|
+
? {}
|
|
3437
|
+
: { counts: boundedFailureCounts(error) }),
|
|
3438
|
+
});
|
|
3439
|
+
fail(threadId, 'checkpoint, request, or native completion failed', error);
|
|
1312
3440
|
} finally {
|
|
1313
3441
|
if (pending.timer) pending.timer();
|
|
1314
3442
|
cleanupRequest(pending.readId);
|
|
@@ -1328,22 +3456,44 @@ export function createCodexContextProxy({
|
|
|
1328
3456
|
const owned = ownedRequests.get(message.id);
|
|
1329
3457
|
ownedRequests.delete(message.id);
|
|
1330
3458
|
retire(message.id);
|
|
1331
|
-
if (
|
|
3459
|
+
if (
|
|
3460
|
+
owned.kind === 'read' ||
|
|
3461
|
+
owned.kind === 'page' ||
|
|
3462
|
+
owned.kind === 'select' ||
|
|
3463
|
+
owned.kind === 'budget' ||
|
|
3464
|
+
owned.kind === 'inject' ||
|
|
3465
|
+
owned.kind === 'telemetry'
|
|
3466
|
+
) {
|
|
1332
3467
|
const waiter = internalReads.get(message.id);
|
|
1333
3468
|
internalReads.delete(message.id);
|
|
1334
3469
|
if (waiter) {
|
|
1335
|
-
if ('error' in message) waiter.reject(
|
|
3470
|
+
if ('error' in message) waiter.reject(rpcFailure(owned.kind, message.error));
|
|
1336
3471
|
else waiter.resolve(message.result);
|
|
1337
3472
|
}
|
|
1338
|
-
} else if ('error' in message)
|
|
1339
|
-
|
|
3473
|
+
} else if ('error' in message) {
|
|
3474
|
+
const failure = rpcFailure(owned.kind, message.error);
|
|
3475
|
+
fail(owned.threadId, 'app-server request rejected', failure);
|
|
3476
|
+
}
|
|
3477
|
+
if ('error' in message) {
|
|
3478
|
+
const failure = rpcFailure(owned.kind, message.error);
|
|
1340
3479
|
emitDiagnostic({
|
|
1341
3480
|
phase: owned.kind,
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
?
|
|
1345
|
-
: '
|
|
3481
|
+
stage:
|
|
3482
|
+
owned.kind === 'read' || owned.kind === 'page'
|
|
3483
|
+
? 'capture'
|
|
3484
|
+
: owned.kind === 'budget'
|
|
3485
|
+
? 'budget'
|
|
3486
|
+
: owned.kind === 'select'
|
|
3487
|
+
? 'selection'
|
|
3488
|
+
: owned.kind === 'inject'
|
|
3489
|
+
? 'recovery'
|
|
3490
|
+
: 'transport',
|
|
3491
|
+
reason: 'protocol_error',
|
|
3492
|
+
threadId: owned.threadId,
|
|
3493
|
+
operationId: threads.get(owned.threadId)?.pending?.operationId,
|
|
3494
|
+
...(safeRpcCode(failure.rpcCode) === null ? {} : { rpcCode: failure.rpcCode }),
|
|
1346
3495
|
});
|
|
3496
|
+
}
|
|
1347
3497
|
if (owned.kind === 'compact') {
|
|
1348
3498
|
const state = threads.get(owned.threadId);
|
|
1349
3499
|
if (state?.pending) state.pending.compactId = null;
|
|
@@ -1356,11 +3506,19 @@ export function createCodexContextProxy({
|
|
|
1356
3506
|
typeof message.params.threadId === 'string' &&
|
|
1357
3507
|
isRecord(message.params.status)
|
|
1358
3508
|
) {
|
|
1359
|
-
const state =
|
|
3509
|
+
const state = threads.get(message.params.threadId);
|
|
3510
|
+
if (!state) return true;
|
|
1360
3511
|
state.status =
|
|
1361
3512
|
typeof message.params.status.type === 'string' ? message.params.status.type : 'unknown';
|
|
1362
3513
|
state.idle = state.status === 'idle';
|
|
1363
3514
|
if (!state.idle) invalidateAttachment(state);
|
|
3515
|
+
if (!state.owned) return true;
|
|
3516
|
+
if (!state.telemetryConnected) {
|
|
3517
|
+
state.telemetryConnected = true;
|
|
3518
|
+
telemetryFor(message.params.threadId).connected({
|
|
3519
|
+
triggerPercent: effectiveThreshold,
|
|
3520
|
+
});
|
|
3521
|
+
}
|
|
1364
3522
|
if (state.idle) void requestCompaction(message.params.threadId, state);
|
|
1365
3523
|
}
|
|
1366
3524
|
if (
|
|
@@ -1381,7 +3539,8 @@ export function createCodexContextProxy({
|
|
|
1381
3539
|
stderr.write('codex-context: invalid usage sample skipped\n');
|
|
1382
3540
|
return true;
|
|
1383
3541
|
}
|
|
1384
|
-
const state =
|
|
3542
|
+
const state = threads.get(params.threadId);
|
|
3543
|
+
if (!state) return true;
|
|
1385
3544
|
state.usage = params.tokenUsage;
|
|
1386
3545
|
state.usageReceivedAt = now();
|
|
1387
3546
|
state.turnId = params.turnId;
|
|
@@ -1395,6 +3554,17 @@ export function createCodexContextProxy({
|
|
|
1395
3554
|
state.turnId !== state.recovery.sourceTurnId
|
|
1396
3555
|
)
|
|
1397
3556
|
invalidateAttachment(state);
|
|
3557
|
+
if (!state.owned) return true;
|
|
3558
|
+
telemetryFor(params.threadId).usage({
|
|
3559
|
+
measurement: {
|
|
3560
|
+
unit: 'tokens',
|
|
3561
|
+
used: params.tokenUsage.last.totalTokens,
|
|
3562
|
+
...(hasKnownContextWindow(params.tokenUsage)
|
|
3563
|
+
? { capacity: params.tokenUsage.modelContextWindow }
|
|
3564
|
+
: {}),
|
|
3565
|
+
},
|
|
3566
|
+
triggerPercent: effectiveThreshold,
|
|
3567
|
+
});
|
|
1398
3568
|
if (state.idle) void requestCompaction(params.threadId, state);
|
|
1399
3569
|
}
|
|
1400
3570
|
if (
|
|
@@ -1427,6 +3597,13 @@ export function createCodexContextProxy({
|
|
|
1427
3597
|
pending.turnId = typeof p.turnId === 'string' ? p.turnId : null;
|
|
1428
3598
|
pending.started = true;
|
|
1429
3599
|
pending.itemId = p.item.id ?? null;
|
|
3600
|
+
emitDiagnostic({
|
|
3601
|
+
phase: 'native_started',
|
|
3602
|
+
stage: 'native',
|
|
3603
|
+
reason: 'requested',
|
|
3604
|
+
threadId: p.threadId,
|
|
3605
|
+
operationId: pending.operationId,
|
|
3606
|
+
});
|
|
1430
3607
|
}
|
|
1431
3608
|
}
|
|
1432
3609
|
if (
|
|
@@ -1445,6 +3622,13 @@ export function createCodexContextProxy({
|
|
|
1445
3622
|
pending.itemId === (p.item.id ?? null)
|
|
1446
3623
|
) {
|
|
1447
3624
|
pending.itemCompleted = true;
|
|
3625
|
+
emitDiagnostic({
|
|
3626
|
+
phase: 'native_item_completed',
|
|
3627
|
+
stage: 'native',
|
|
3628
|
+
reason: 'completed',
|
|
3629
|
+
threadId: p.threadId,
|
|
3630
|
+
operationId: pending.operationId,
|
|
3631
|
+
});
|
|
1448
3632
|
if (pending.turnCompleted) pending.deferred.resolve();
|
|
1449
3633
|
}
|
|
1450
3634
|
}
|
|
@@ -1460,7 +3644,16 @@ export function createCodexContextProxy({
|
|
|
1460
3644
|
if (pending?.lifecycleArmed && pending.turnId === turn.id) {
|
|
1461
3645
|
if (turn.status === 'completed') pending.turnCompleted = true;
|
|
1462
3646
|
else fail(message.params.threadId, 'native compaction turn failed');
|
|
1463
|
-
if (pending.itemCompleted && pending.turnCompleted)
|
|
3647
|
+
if (pending.itemCompleted && pending.turnCompleted) {
|
|
3648
|
+
emitDiagnostic({
|
|
3649
|
+
phase: 'native_completed',
|
|
3650
|
+
stage: 'native',
|
|
3651
|
+
reason: 'completed',
|
|
3652
|
+
threadId: message.params.threadId,
|
|
3653
|
+
operationId: pending.operationId,
|
|
3654
|
+
});
|
|
3655
|
+
pending.deferred.resolve();
|
|
3656
|
+
}
|
|
1464
3657
|
}
|
|
1465
3658
|
}
|
|
1466
3659
|
if (
|
|
@@ -1475,11 +3668,86 @@ export function createCodexContextProxy({
|
|
|
1475
3668
|
typeof message.params.threadId === 'string'
|
|
1476
3669
|
) {
|
|
1477
3670
|
fail(message.params.threadId, 'thread closed');
|
|
3671
|
+
ownedThreadIds.delete(message.params.threadId);
|
|
1478
3672
|
threads.delete(message.params.threadId);
|
|
1479
3673
|
}
|
|
1480
3674
|
return true;
|
|
1481
3675
|
}
|
|
1482
3676
|
|
|
3677
|
+
/**
|
|
3678
|
+
* Observe TUI-owned requests without changing their wire representation.
|
|
3679
|
+
* The app-server may emit status/usage events before a resumed thread's
|
|
3680
|
+
* response arrives, so the adapter must defer owned work until that response
|
|
3681
|
+
* confirms that the thread is loaded.
|
|
3682
|
+
*/
|
|
3683
|
+
function observeClientRequest(message) {
|
|
3684
|
+
if (!isRecord(message)) return;
|
|
3685
|
+
const requestId = message.id;
|
|
3686
|
+
const validRequestId =
|
|
3687
|
+
(typeof requestId === 'string' && requestId.length > 0 && requestId.length <= 128) ||
|
|
3688
|
+
Number.isSafeInteger(requestId);
|
|
3689
|
+
if (!validRequestId) return;
|
|
3690
|
+
const requestKey = clientRequestKey(requestId);
|
|
3691
|
+
if (requestKey === null) return;
|
|
3692
|
+
if (['thread/start', 'thread/resume', 'thread/fork'].includes(message.method)) {
|
|
3693
|
+
const threadId =
|
|
3694
|
+
isRecord(message.params) && typeof message.params.threadId === 'string'
|
|
3695
|
+
? message.params.threadId
|
|
3696
|
+
: null;
|
|
3697
|
+
if (
|
|
3698
|
+
clientRequestBindings.has(requestKey) ||
|
|
3699
|
+
clientRequestBindings.size >= MAX_CLIENT_REQUEST_BINDINGS
|
|
3700
|
+
)
|
|
3701
|
+
return;
|
|
3702
|
+
clientRequestBindings.set(requestKey, { method: message.method, threadId });
|
|
3703
|
+
if (message.method === 'thread/resume') {
|
|
3704
|
+
if (threadId === null || threadId.length === 0 || threadId.length > 256) {
|
|
3705
|
+
clientRequestBindings.delete(requestKey);
|
|
3706
|
+
return;
|
|
3707
|
+
}
|
|
3708
|
+
const state = threadState(threadId);
|
|
3709
|
+
state.resumePending = true;
|
|
3710
|
+
state.resumeRetryRequested = false;
|
|
3711
|
+
state.resumeRequests.add(requestKey);
|
|
3712
|
+
}
|
|
3713
|
+
}
|
|
3714
|
+
}
|
|
3715
|
+
|
|
3716
|
+
function observeClientResponse(message) {
|
|
3717
|
+
if (!isRecord(message) || message.id === undefined) return;
|
|
3718
|
+
const requestKey = clientRequestKey(message.id);
|
|
3719
|
+
if (requestKey === null) return;
|
|
3720
|
+
const binding = clientRequestBindings.get(requestKey);
|
|
3721
|
+
if (binding) {
|
|
3722
|
+
clientRequestBindings.delete(requestKey);
|
|
3723
|
+
if (binding.method === 'thread/resume' && binding.threadId !== null) {
|
|
3724
|
+
const state = threadState(binding.threadId);
|
|
3725
|
+
state.resumeRequests.delete(requestKey);
|
|
3726
|
+
state.resumePending = state.resumeRequests.size > 0;
|
|
3727
|
+
state.resumeRetryRequested = false;
|
|
3728
|
+
const responseThreadId = message.result?.thread?.id;
|
|
3729
|
+
if (
|
|
3730
|
+
message.error === undefined &&
|
|
3731
|
+
typeof responseThreadId === 'string' &&
|
|
3732
|
+
responseThreadId === binding.threadId
|
|
3733
|
+
) {
|
|
3734
|
+
const claimed = claimOwnedThread(binding.threadId);
|
|
3735
|
+
if (claimed && !state.resumePending && claimed.idle)
|
|
3736
|
+
void requestCompaction(binding.threadId, claimed);
|
|
3737
|
+
} else if (!state.owned && state.resumeRequests.size === 0 && state.pending === null) {
|
|
3738
|
+
threads.delete(binding.threadId);
|
|
3739
|
+
}
|
|
3740
|
+
} else if (message.error === undefined) {
|
|
3741
|
+
const selectedId = message.result?.thread?.id;
|
|
3742
|
+
if (typeof selectedId === 'string' && selectedId.length > 0 && selectedId.length <= 256) {
|
|
3743
|
+
const selected = claimOwnedThread(selectedId);
|
|
3744
|
+
if (!selected) return;
|
|
3745
|
+
if (selected.idle) void requestCompaction(selectedId, selected);
|
|
3746
|
+
}
|
|
3747
|
+
}
|
|
3748
|
+
}
|
|
3749
|
+
}
|
|
3750
|
+
|
|
1483
3751
|
const decoder = new StringDecoder('utf8');
|
|
1484
3752
|
let inputBuffer = '';
|
|
1485
3753
|
let passthrough = false;
|
|
@@ -1496,44 +3764,100 @@ export function createCodexContextProxy({
|
|
|
1496
3764
|
}
|
|
1497
3765
|
};
|
|
1498
3766
|
const processLine = (line) => {
|
|
1499
|
-
if (Buffer.byteLength(line) > MAX_LINE_BYTES) {
|
|
1500
|
-
|
|
3767
|
+
if (Buffer.byteLength(line, 'utf8') + 1 > MAX_LINE_BYTES) {
|
|
3768
|
+
const ownedId = internalRequestId(line);
|
|
3769
|
+
if (ownedId !== null) {
|
|
1501
3770
|
stderr.write('codex-context: oversized internal response rejected\n');
|
|
1502
|
-
|
|
3771
|
+
failOversizedInternal(ownedId, Buffer.byteLength(line, 'utf8') + 1);
|
|
1503
3772
|
} else forward(`${line}\n`);
|
|
1504
3773
|
return;
|
|
1505
3774
|
}
|
|
1506
3775
|
let forwardLine = true;
|
|
1507
3776
|
try {
|
|
1508
|
-
|
|
3777
|
+
const message = JSON.parse(line.endsWith('\r') ? line.slice(0, -1) : line);
|
|
3778
|
+
observeClientResponse(message);
|
|
3779
|
+
forwardLine = handle(message);
|
|
1509
3780
|
} catch {
|
|
1510
3781
|
/* preserve malformed external protocol lines */
|
|
1511
3782
|
}
|
|
1512
3783
|
if (forwardLine) forward(`${line}\n`);
|
|
1513
3784
|
};
|
|
3785
|
+
function internalRequestId(line) {
|
|
3786
|
+
const match =
|
|
3787
|
+
/^\s*\{\s*(?:"id"\s*:\s*"(ai-brain-context-[A-Za-z0-9._:-]{1,128})"|"jsonrpc"\s*:\s*"2\.0"\s*,\s*"id"\s*:\s*"(ai-brain-context-[A-Za-z0-9._:-]{1,128})")/u.exec(
|
|
3788
|
+
line,
|
|
3789
|
+
);
|
|
3790
|
+
return match?.[1] ?? match?.[2] ?? null;
|
|
3791
|
+
}
|
|
3792
|
+
function failCompactions(reason) {
|
|
3793
|
+
for (const [threadId, state] of threads) {
|
|
3794
|
+
if (state.pending) fail(threadId, reason);
|
|
3795
|
+
}
|
|
3796
|
+
}
|
|
3797
|
+
function failOversizedInternal(id, responseBytes) {
|
|
3798
|
+
const owned = ownedRequests.get(id);
|
|
3799
|
+
if (!owned) {
|
|
3800
|
+
failCompactions('oversized internal response');
|
|
3801
|
+
return;
|
|
3802
|
+
}
|
|
3803
|
+
if (owned.kind === 'page') {
|
|
3804
|
+
ownedRequests.delete(id);
|
|
3805
|
+
const waiter = internalReads.get(id);
|
|
3806
|
+
internalReads.delete(id);
|
|
3807
|
+
retire(id);
|
|
3808
|
+
const failure = new Error('oversized internal page response');
|
|
3809
|
+
failure.diagnosticReason = 'capture_size_exceeded';
|
|
3810
|
+
failure.responseBytes = responseBytes;
|
|
3811
|
+
failure.responseLimitBytes = MAX_LINE_BYTES;
|
|
3812
|
+
waiter?.reject(failure);
|
|
3813
|
+
return;
|
|
3814
|
+
}
|
|
3815
|
+
cleanupRequest(id);
|
|
3816
|
+
if (owned.kind === 'telemetry') return;
|
|
3817
|
+
const failure = new Error('oversized internal response');
|
|
3818
|
+
failure.diagnosticReason = 'capture_size_exceeded';
|
|
3819
|
+
failure.responseBytes = responseBytes;
|
|
3820
|
+
failure.responseLimitBytes = MAX_LINE_BYTES;
|
|
3821
|
+
fail(owned.threadId, 'oversized internal response', failure);
|
|
3822
|
+
}
|
|
1514
3823
|
child.stdout.on('data', (chunk) => {
|
|
1515
3824
|
if (stopped) return;
|
|
1516
|
-
|
|
3825
|
+
const decoded = decoder.write(chunk);
|
|
3826
|
+
if (discardOversized) {
|
|
3827
|
+
const newline = decoded.indexOf('\n');
|
|
3828
|
+
if (newline < 0) return;
|
|
3829
|
+
discardOversized = false;
|
|
3830
|
+
inputBuffer = decoded.slice(newline + 1);
|
|
3831
|
+
} else inputBuffer += decoded;
|
|
1517
3832
|
while (true) {
|
|
1518
3833
|
const newline = inputBuffer.indexOf('\n');
|
|
1519
3834
|
if (newline < 0) break;
|
|
1520
3835
|
const line = inputBuffer.slice(0, newline);
|
|
1521
3836
|
inputBuffer = inputBuffer.slice(newline + 1);
|
|
1522
|
-
if (
|
|
3837
|
+
if (discardOversized) {
|
|
3838
|
+
discardOversized = false;
|
|
3839
|
+
} else if (passthrough) {
|
|
1523
3840
|
forward(`${line}\n`);
|
|
1524
3841
|
passthrough = false;
|
|
1525
|
-
} else if (discardOversized) {
|
|
1526
|
-
discardOversized = false;
|
|
1527
3842
|
} else processLine(line);
|
|
1528
3843
|
}
|
|
1529
|
-
if (Buffer.byteLength(inputBuffer) > MAX_LINE_BYTES) {
|
|
1530
|
-
if (
|
|
3844
|
+
if (Buffer.byteLength(inputBuffer, 'utf8') + 1 > MAX_LINE_BYTES) {
|
|
3845
|
+
if (passthrough) {
|
|
3846
|
+
forward(inputBuffer);
|
|
3847
|
+
inputBuffer = '';
|
|
3848
|
+
} else if (internalRequestId(inputBuffer) !== null) {
|
|
1531
3849
|
stderr.write('codex-context: oversized internal response rejected\n');
|
|
1532
|
-
|
|
3850
|
+
failOversizedInternal(
|
|
3851
|
+
internalRequestId(inputBuffer),
|
|
3852
|
+
Buffer.byteLength(inputBuffer, 'utf8') + 1,
|
|
3853
|
+
);
|
|
1533
3854
|
discardOversized = true;
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
3855
|
+
inputBuffer = '';
|
|
3856
|
+
} else {
|
|
3857
|
+
forward(inputBuffer);
|
|
3858
|
+
inputBuffer = '';
|
|
3859
|
+
passthrough = true;
|
|
3860
|
+
}
|
|
1537
3861
|
}
|
|
1538
3862
|
});
|
|
1539
3863
|
child.stdout.on('end', () => {
|
|
@@ -1553,17 +3877,49 @@ export function createCodexContextProxy({
|
|
|
1553
3877
|
failAll('app-server disconnected');
|
|
1554
3878
|
emitDiagnostic({ phase: 'transport_close' });
|
|
1555
3879
|
});
|
|
3880
|
+
// Native app-server stderr is operational logging and may contain private
|
|
3881
|
+
// paths, prompts or tool output. Drain the owned pipe without forwarding or
|
|
3882
|
+
// persisting its contents; protocol errors remain on stdout and process exit
|
|
3883
|
+
// status is still propagated by the launcher.
|
|
3884
|
+
if (child.stderr && typeof child.stderr.on === 'function') {
|
|
3885
|
+
child.stderr.on('data', () => undefined);
|
|
3886
|
+
child.stderr.resume?.();
|
|
3887
|
+
}
|
|
1556
3888
|
return {
|
|
1557
3889
|
threads,
|
|
1558
|
-
|
|
1559
|
-
|
|
3890
|
+
observeClientRequest,
|
|
3891
|
+
observeClientResponse,
|
|
3892
|
+
async stop() {
|
|
3893
|
+
if (stopped) {
|
|
3894
|
+
await Promise.all(
|
|
3895
|
+
[...diagnosticSinks.values()].map((sink) =>
|
|
3896
|
+
typeof sink.flush === 'function'
|
|
3897
|
+
? Promise.resolve(sink.flush()).catch(() => undefined)
|
|
3898
|
+
: undefined,
|
|
3899
|
+
),
|
|
3900
|
+
);
|
|
3901
|
+
return;
|
|
3902
|
+
}
|
|
3903
|
+
for (const reporter of telemetryByThread.values()) {
|
|
3904
|
+
reporter.disconnected();
|
|
3905
|
+
void reporter.flush();
|
|
3906
|
+
}
|
|
1560
3907
|
stopped = true;
|
|
3908
|
+
clientRequestBindings.clear();
|
|
3909
|
+
ownedThreadIds.clear();
|
|
1561
3910
|
failAll('adapter stopped');
|
|
3911
|
+
await Promise.all(
|
|
3912
|
+
[...diagnosticSinks.values()].map((sink) =>
|
|
3913
|
+
typeof sink.flush === 'function'
|
|
3914
|
+
? Promise.resolve(sink.flush()).catch(() => undefined)
|
|
3915
|
+
: undefined,
|
|
3916
|
+
),
|
|
3917
|
+
);
|
|
1562
3918
|
},
|
|
1563
3919
|
};
|
|
1564
3920
|
}
|
|
1565
3921
|
|
|
1566
|
-
function main() {
|
|
3922
|
+
async function main() {
|
|
1567
3923
|
if (process.argv.includes('--help')) {
|
|
1568
3924
|
process.stderr.write(
|
|
1569
3925
|
'Usage: node scripts/codex-context-compaction.mjs [app-server arguments...]\nStarts an owned Codex app-server and requests native compaction at CODEX_CONTEXT_COMPACT_AT (default 85).\n',
|
|
@@ -1572,19 +3928,87 @@ function main() {
|
|
|
1572
3928
|
}
|
|
1573
3929
|
const policy = parseContextPolicy();
|
|
1574
3930
|
const command = process.env.CODEX_BIN ?? 'codex';
|
|
1575
|
-
const
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
}
|
|
1579
|
-
|
|
3931
|
+
const profile = process.env.CODEX_APP_SERVER_PROFILE;
|
|
3932
|
+
if (profile !== undefined && !/^[A-Za-z0-9._-]{1,64}$/u.test(profile)) {
|
|
3933
|
+
throw new Error('CODEX_APP_SERVER_PROFILE is invalid');
|
|
3934
|
+
}
|
|
3935
|
+
const child = spawn(
|
|
3936
|
+
command,
|
|
3937
|
+
[
|
|
3938
|
+
...(profile === undefined ? [] : ['--profile', profile]),
|
|
3939
|
+
'app-server',
|
|
3940
|
+
...process.argv.slice(2),
|
|
3941
|
+
],
|
|
3942
|
+
{
|
|
3943
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
3944
|
+
env: codexChildEnvironment(),
|
|
3945
|
+
},
|
|
3946
|
+
);
|
|
3947
|
+
let diagnosticFactory = null;
|
|
3948
|
+
const diagnosticModule = process.env.CODEX_CONTEXT_DIAGNOSTICS_MODULE;
|
|
3949
|
+
if (
|
|
3950
|
+
process.env.CODEX_CONTEXT_DIAGNOSTICS_ENABLED === '1' &&
|
|
3951
|
+
typeof diagnosticModule === 'string' &&
|
|
3952
|
+
diagnosticModule.length > 0
|
|
3953
|
+
) {
|
|
3954
|
+
try {
|
|
3955
|
+
const module = await import(pathToFileURL(resolve(diagnosticModule)).href);
|
|
3956
|
+
if (typeof module.createCodexDiagnosticSink === 'function') {
|
|
3957
|
+
const stateDir = process.env.CODEX_CONTEXT_DIAGNOSTICS_DIR ?? process.cwd();
|
|
3958
|
+
diagnosticFactory = (threadId) =>
|
|
3959
|
+
module.createCodexDiagnosticSink({
|
|
3960
|
+
stateDir,
|
|
3961
|
+
sessionId: threadId,
|
|
3962
|
+
clientVersion: process.env.CODEX_CLIENT_VERSION ?? '0.161.0',
|
|
3963
|
+
enabled: true,
|
|
3964
|
+
});
|
|
3965
|
+
}
|
|
3966
|
+
} catch {
|
|
3967
|
+
diagnosticFactory = null;
|
|
3968
|
+
}
|
|
3969
|
+
}
|
|
3970
|
+
const proxy = createCodexContextProxy({
|
|
1580
3971
|
child,
|
|
1581
3972
|
threshold: numberEnv('CODEX_CONTEXT_COMPACT_AT', DEFAULT_THRESHOLD),
|
|
1582
3973
|
timeoutMs: numberEnv('CODEX_CONTEXT_COMPACT_TIMEOUT_MS', DEFAULT_TIMEOUT_MS),
|
|
1583
3974
|
policy,
|
|
3975
|
+
createDiagnosticSink: diagnosticFactory,
|
|
3976
|
+
});
|
|
3977
|
+
const clientDecoder = new StringDecoder('utf8');
|
|
3978
|
+
let clientInputBuffer = '';
|
|
3979
|
+
let discardOversizedClientFrame = false;
|
|
3980
|
+
process.stdin.on('data', (chunk) => {
|
|
3981
|
+
const decoded = clientDecoder.write(chunk);
|
|
3982
|
+
if (discardOversizedClientFrame) {
|
|
3983
|
+
const newline = decoded.indexOf('\n');
|
|
3984
|
+
if (newline < 0) return;
|
|
3985
|
+
discardOversizedClientFrame = false;
|
|
3986
|
+
clientInputBuffer = decoded.slice(newline + 1);
|
|
3987
|
+
} else clientInputBuffer += decoded;
|
|
3988
|
+
while (true) {
|
|
3989
|
+
const newline = clientInputBuffer.indexOf('\n');
|
|
3990
|
+
if (newline < 0) break;
|
|
3991
|
+
const line = clientInputBuffer.slice(0, newline).replace(/\r$/u, '');
|
|
3992
|
+
clientInputBuffer = clientInputBuffer.slice(newline + 1);
|
|
3993
|
+
if (!line) continue;
|
|
3994
|
+
if (Buffer.byteLength(line, 'utf8') + 1 > MAX_LINE_BYTES) continue;
|
|
3995
|
+
try {
|
|
3996
|
+
proxy.observeClientRequest(JSON.parse(line));
|
|
3997
|
+
} catch {
|
|
3998
|
+
/* Preserve malformed/native frames unchanged; the app-server owns validation. */
|
|
3999
|
+
}
|
|
4000
|
+
}
|
|
4001
|
+
if (Buffer.byteLength(clientInputBuffer, 'utf8') + 1 > MAX_LINE_BYTES) {
|
|
4002
|
+
clientInputBuffer = '';
|
|
4003
|
+
discardOversizedClientFrame = true;
|
|
4004
|
+
}
|
|
1584
4005
|
});
|
|
1585
4006
|
process.stdin.pipe(child.stdin);
|
|
1586
4007
|
process.stdin.on('end', () => child.stdin.end());
|
|
1587
|
-
child.on('exit', (code, signal) =>
|
|
4008
|
+
child.on('exit', async (code, signal) => {
|
|
4009
|
+
await proxy.stop();
|
|
4010
|
+
process.exitCode = code ?? (signal ? 1 : 0);
|
|
4011
|
+
});
|
|
1588
4012
|
}
|
|
1589
4013
|
if (process.argv[1] && resolve(process.argv[1]) === resolve(new URL(import.meta.url).pathname))
|
|
1590
|
-
main();
|
|
4014
|
+
void main();
|