@brunobracaioli/aibrain 0.1.0-beta.1 → 0.3.0-beta.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +87 -6
- package/dist/assets/claude/.claude-plugin/plugin.json +103 -1
- package/dist/assets/claude/hooks/context-compaction.ts +6263 -677
- package/dist/assets/claude/hooks/provider-window.js +263 -0
- package/dist/assets/claude/hooks/state-fitting.d.mts +47 -0
- package/dist/assets/claude/hooks/state-fitting.mjs +324 -0
- package/dist/assets/codex-context-compaction.mjs +2874 -262
- package/dist/assets/context-selection-fitting.mjs +194 -0
- package/dist/assets/context-state-fitting.mjs +324 -0
- package/dist/assets/context-telemetry-reporter.mjs +305 -0
- package/dist/assets/jev-paid-pilot-terminal.mjs +591 -125
- package/dist/claude-checkpoint.mjs +1338 -0
- package/dist/claude-observation.mjs +1336 -0
- package/dist/cli.mjs +396 -29
- package/dist/codex-compatibility.mjs +467 -0
- package/dist/codex-diagnostics.mjs +426 -0
- package/dist/codex-permissions.mjs +590 -0
- package/dist/codex-protocol-fingerprint.json +131 -0
- package/dist/context-telemetry.mjs +257 -0
- package/dist/launch.mjs +369 -23
- package/dist/mcp.mjs +577 -5
- package/dist/monitor.mjs +61 -1
- package/dist/package.json +1 -1
- package/dist/platform.mjs +33 -3
- package/dist/policy.mjs +218 -0
- package/dist/state.mjs +132 -25
- package/dist/telemetry-report.mjs +133 -0
- package/package.json +1 -1
|
@@ -1,23 +1,256 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
chmod,
|
|
5
|
+
lstat,
|
|
6
|
+
mkdir,
|
|
7
|
+
open,
|
|
8
|
+
readFile,
|
|
9
|
+
readdir,
|
|
10
|
+
rename,
|
|
11
|
+
unlink,
|
|
12
|
+
writeFile,
|
|
13
|
+
} from 'node:fs/promises';
|
|
4
14
|
import { spawn } from 'node:child_process';
|
|
5
15
|
import { createHash, randomUUID } from 'node:crypto';
|
|
6
16
|
import { dirname, relative, resolve } from 'node:path';
|
|
7
17
|
import { StringDecoder } from 'node:string_decoder';
|
|
18
|
+
import { pathToFileURL } from 'node:url';
|
|
19
|
+
import { createTelemetryReporter, TELEMETRY_TIMEOUT_MS } from './context-telemetry-reporter.mjs';
|
|
20
|
+
import { preflightSelectionState } from './context-selection-fitting.mjs';
|
|
21
|
+
|
|
22
|
+
const captureFs = {
|
|
23
|
+
chmod,
|
|
24
|
+
lstat,
|
|
25
|
+
mkdir,
|
|
26
|
+
open,
|
|
27
|
+
readFile,
|
|
28
|
+
readdir,
|
|
29
|
+
rename,
|
|
30
|
+
unlink,
|
|
31
|
+
writeFile,
|
|
32
|
+
};
|
|
8
33
|
|
|
9
34
|
const DEFAULT_THRESHOLD = 85;
|
|
10
35
|
const COOLDOWN_MS = 60_000;
|
|
11
36
|
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
12
|
-
const MAX_LINE_BYTES =
|
|
13
|
-
const MAX_CAPTURE_BYTES =
|
|
37
|
+
const MAX_LINE_BYTES = 8 * 1024 * 1024;
|
|
38
|
+
const MAX_CAPTURE_BYTES = 8 * 1024 * 1024;
|
|
39
|
+
const MAX_CAPTURE_PAGE_BYTES = 8 * 1024 * 1024;
|
|
40
|
+
const MAX_CAPTURE_TOTAL_BYTES = 128 * 1024 * 1024;
|
|
41
|
+
const MAX_CAPTURE_PAGES = 32;
|
|
42
|
+
const MAX_CAPTURE_ITEMS = 65_536;
|
|
43
|
+
const MAX_CAPTURE_DIRECTORY_BYTES = 256 * 1024 * 1024;
|
|
44
|
+
const MAX_CAPTURE_STAGING_HEADROOM_BYTES = 16 * 1024 * 1024;
|
|
45
|
+
const MAX_CAPTURE_FILES = 128;
|
|
46
|
+
const CAPTURE_LOCK_WAIT_MS = 1_000;
|
|
14
47
|
const MAX_OWNED_REQUESTS = 1_000;
|
|
15
48
|
const MAX_RETIRED_REQUESTS = 2_000;
|
|
49
|
+
const MAX_CLIENT_REQUEST_BINDINGS = 128;
|
|
50
|
+
const MAX_OWNED_THREAD_IDS = 256;
|
|
16
51
|
const MAX_RECOVERY_ITEMS = 128;
|
|
52
|
+
const MAX_RECOVERY_SOURCE_INDEX = 65_536;
|
|
17
53
|
const MAX_NOTE_BYTES = 128 * 1024;
|
|
18
54
|
const MAX_SELECTION_BYTES = 512 * 1024;
|
|
55
|
+
const MAX_POLICY_BYTES = 8 * 1024;
|
|
19
56
|
const MAX_NOTE_ITEM_CHARS = 16_384;
|
|
20
57
|
const USAGE_FRESHNESS_MS = 60_000;
|
|
58
|
+
const MAX_DIAGNOSTIC_STRING = 128;
|
|
59
|
+
const MAX_DIAGNOSTIC_BYTES = MAX_CAPTURE_DIRECTORY_BYTES;
|
|
60
|
+
const MAX_BUDGET_SNAPSHOT_AGE_MS = 15_000;
|
|
61
|
+
const CONTEXT_ERROR_CODES = new Set([
|
|
62
|
+
'INVALID_INPUT',
|
|
63
|
+
'FEATURE_DISABLED',
|
|
64
|
+
'PAID_PLAN_REQUIRED',
|
|
65
|
+
'ENTITLEMENT_UNAVAILABLE',
|
|
66
|
+
'RATE_LIMITED',
|
|
67
|
+
'PROVIDER_NOT_CONFIGURED',
|
|
68
|
+
'BUDGET_EXCEEDED',
|
|
69
|
+
'PROVIDER_TIMEOUT',
|
|
70
|
+
'PROVIDER_UNAVAILABLE',
|
|
71
|
+
'INVALID_PROVIDER_RESPONSE',
|
|
72
|
+
]);
|
|
73
|
+
const CONTEXT_FAILURE_DETAILS = new Set(['state_budget_exceeded', 'shared_admission_denied']);
|
|
74
|
+
const CONTEXT_ADMISSION_REASONS = new Set([
|
|
75
|
+
'request_replayed',
|
|
76
|
+
'account_request_limit',
|
|
77
|
+
'account_token_limit',
|
|
78
|
+
'global_request_limit',
|
|
79
|
+
'global_token_limit',
|
|
80
|
+
'account_in_flight',
|
|
81
|
+
'database_unavailable',
|
|
82
|
+
]);
|
|
83
|
+
const SHARED_QUOTA_ADMISSION_REASONS = new Set([
|
|
84
|
+
'account_request_limit',
|
|
85
|
+
'account_token_limit',
|
|
86
|
+
'global_request_limit',
|
|
87
|
+
'global_token_limit',
|
|
88
|
+
'account_in_flight',
|
|
89
|
+
]);
|
|
90
|
+
const DIAGNOSTIC_STAGES = new Set([
|
|
91
|
+
'capture',
|
|
92
|
+
'budget',
|
|
93
|
+
'selection',
|
|
94
|
+
'notes',
|
|
95
|
+
'native',
|
|
96
|
+
'recovery',
|
|
97
|
+
'transport',
|
|
98
|
+
]);
|
|
99
|
+
const DIAGNOSTIC_REASONS = new Set([
|
|
100
|
+
'requested',
|
|
101
|
+
'completed',
|
|
102
|
+
'candidate_window_bounded',
|
|
103
|
+
'candidate_over_bound',
|
|
104
|
+
'protected_over_bound',
|
|
105
|
+
'history_unavailable',
|
|
106
|
+
'stale_snapshot',
|
|
107
|
+
'unavailable',
|
|
108
|
+
'admission_denied',
|
|
109
|
+
'invalid_response',
|
|
110
|
+
'request_failed',
|
|
111
|
+
'source_changed',
|
|
112
|
+
'timeout',
|
|
113
|
+
'thread_closed',
|
|
114
|
+
'protocol_error',
|
|
115
|
+
'capture_size_exceeded',
|
|
116
|
+
'capture_item_count_exceeded',
|
|
117
|
+
'private_storage_failed',
|
|
118
|
+
'storage_capacity_exceeded',
|
|
119
|
+
'attached',
|
|
120
|
+
'uncertain',
|
|
121
|
+
'disconnected',
|
|
122
|
+
]);
|
|
123
|
+
const FITTING_STAGES = new Set([
|
|
124
|
+
'original',
|
|
125
|
+
'inputs_1000',
|
|
126
|
+
'inputs_200',
|
|
127
|
+
'inputs_60',
|
|
128
|
+
'text_head_tail',
|
|
129
|
+
'text_omitted',
|
|
130
|
+
'compact_calls',
|
|
131
|
+
'callless_omitted',
|
|
132
|
+
'runs_folded',
|
|
133
|
+
]);
|
|
134
|
+
|
|
135
|
+
function isFiniteNumber(value) {
|
|
136
|
+
return typeof value === 'number' && Number.isFinite(value);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function isSafeBoundedInteger(value, minimum, maximum) {
|
|
140
|
+
return Number.isSafeInteger(value) && value >= minimum && value <= maximum;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Parse the optional v2 policy without allowing partial or unknown settings.
|
|
145
|
+
* @param {string|undefined} [raw]
|
|
146
|
+
*/
|
|
147
|
+
export function parseContextPolicy(raw = process.env.CODEX_CONTEXT_POLICY_JSON) {
|
|
148
|
+
if (raw === undefined || raw.trim() === '') return null;
|
|
149
|
+
if (Buffer.byteLength(raw, 'utf8') > MAX_POLICY_BYTES)
|
|
150
|
+
throw new Error('context policy exceeds 8KiB');
|
|
151
|
+
let value;
|
|
152
|
+
try {
|
|
153
|
+
value = JSON.parse(raw);
|
|
154
|
+
} catch {
|
|
155
|
+
throw new Error('context policy is not valid JSON');
|
|
156
|
+
}
|
|
157
|
+
if (
|
|
158
|
+
!isRecord(value) ||
|
|
159
|
+
!hasOnlyKeys(value, [
|
|
160
|
+
'schemaVersion',
|
|
161
|
+
'compactAtPercent',
|
|
162
|
+
'keepThreshold',
|
|
163
|
+
'preserveRecentMessages',
|
|
164
|
+
'maxStateTokens',
|
|
165
|
+
'maxRequestTokens',
|
|
166
|
+
'goal',
|
|
167
|
+
]) ||
|
|
168
|
+
value.schemaVersion !== 2 ||
|
|
169
|
+
!isFiniteNumber(value.compactAtPercent) ||
|
|
170
|
+
value.compactAtPercent < 1 ||
|
|
171
|
+
value.compactAtPercent > 95 ||
|
|
172
|
+
!isFiniteNumber(value.keepThreshold) ||
|
|
173
|
+
value.keepThreshold < 0 ||
|
|
174
|
+
value.keepThreshold > 1 ||
|
|
175
|
+
!isSafeBoundedInteger(value.preserveRecentMessages, 1, 100) ||
|
|
176
|
+
!isSafeBoundedInteger(value.maxStateTokens, 1_000, 25_000) ||
|
|
177
|
+
!isSafeBoundedInteger(value.maxRequestTokens, 1_000, 30_000) ||
|
|
178
|
+
value.maxRequestTokens <= value.maxStateTokens + 256 ||
|
|
179
|
+
(value.goal !== undefined &&
|
|
180
|
+
(typeof value.goal !== 'string' || value.goal.length > 4_000 || value.goal.length === 0))
|
|
181
|
+
)
|
|
182
|
+
throw new Error('context policy v2 is invalid');
|
|
183
|
+
return Object.freeze({
|
|
184
|
+
schemaVersion: 2,
|
|
185
|
+
compactAtPercent: value.compactAtPercent,
|
|
186
|
+
keepThreshold: value.keepThreshold,
|
|
187
|
+
preserveRecentMessages: value.preserveRecentMessages,
|
|
188
|
+
maxStateTokens: value.maxStateTokens,
|
|
189
|
+
maxRequestTokens: value.maxRequestTokens,
|
|
190
|
+
...(value.goal === undefined ? {} : { goal: value.goal }),
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function normalizeContextPolicy(policy) {
|
|
195
|
+
if (policy === undefined) return parseContextPolicy();
|
|
196
|
+
if (policy === null) return null;
|
|
197
|
+
try {
|
|
198
|
+
const serialized = JSON.stringify(policy);
|
|
199
|
+
if (typeof serialized !== 'string') throw new Error('policy is not serializable');
|
|
200
|
+
return parseContextPolicy(serialized);
|
|
201
|
+
} catch {
|
|
202
|
+
throw new Error('context policy v2 is invalid');
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
function selectionOptions(policy) {
|
|
207
|
+
return {
|
|
208
|
+
keep_threshold: policy.keepThreshold,
|
|
209
|
+
preserve_recent_items: policy.preserveRecentMessages,
|
|
210
|
+
max_state_tokens: policy.maxStateTokens,
|
|
211
|
+
max_request_tokens: policy.maxRequestTokens,
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
function validSelectionOptions(value) {
|
|
216
|
+
return (
|
|
217
|
+
isRecord(value) &&
|
|
218
|
+
hasOnlyKeys(value, [
|
|
219
|
+
'keep_threshold',
|
|
220
|
+
'preserve_recent_items',
|
|
221
|
+
'max_state_tokens',
|
|
222
|
+
'max_request_tokens',
|
|
223
|
+
]) &&
|
|
224
|
+
isFiniteNumber(value.keep_threshold) &&
|
|
225
|
+
value.keep_threshold >= 0 &&
|
|
226
|
+
value.keep_threshold <= 1 &&
|
|
227
|
+
isSafeBoundedInteger(value.preserve_recent_items, 1, 100) &&
|
|
228
|
+
isSafeBoundedInteger(value.max_state_tokens, 1_000, 25_000) &&
|
|
229
|
+
isSafeBoundedInteger(value.max_request_tokens, 1_000, 30_000) &&
|
|
230
|
+
value.max_request_tokens > value.max_state_tokens + 256
|
|
231
|
+
);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function validateFitting(value, maxStateTokens = 25_000) {
|
|
235
|
+
if (value === null) return;
|
|
236
|
+
if (
|
|
237
|
+
!isRecord(value) ||
|
|
238
|
+
!hasOnlyKeys(value, [
|
|
239
|
+
'version',
|
|
240
|
+
'stage',
|
|
241
|
+
'estimator',
|
|
242
|
+
'before_estimated_state_tokens',
|
|
243
|
+
'after_estimated_state_tokens',
|
|
244
|
+
]) ||
|
|
245
|
+
value.version !== 'state-fit-v1' ||
|
|
246
|
+
!FITTING_STAGES.has(value.stage) ||
|
|
247
|
+
value.estimator !== 'utf8-bytes-v1' ||
|
|
248
|
+
!isSafeBoundedInteger(value.before_estimated_state_tokens, 0, MAX_SELECTION_BYTES) ||
|
|
249
|
+
!isSafeBoundedInteger(value.after_estimated_state_tokens, 0, maxStateTokens) ||
|
|
250
|
+
value.after_estimated_state_tokens > value.before_estimated_state_tokens
|
|
251
|
+
)
|
|
252
|
+
throw new Error('invalid v2 fitting metadata');
|
|
253
|
+
}
|
|
21
254
|
|
|
22
255
|
function numberEnv(name, fallback) {
|
|
23
256
|
const value = Number(process.env[name]);
|
|
@@ -101,10 +334,13 @@ function isValidUsage(value) {
|
|
|
101
334
|
isRecord(value) &&
|
|
102
335
|
isValidBreakdown(value.last) &&
|
|
103
336
|
isValidBreakdown(value.total) &&
|
|
104
|
-
|
|
105
|
-
|
|
337
|
+
(value.modelContextWindow === null ||
|
|
338
|
+
(isSafeCounter(value.modelContextWindow) && value.modelContextWindow > 0))
|
|
106
339
|
);
|
|
107
340
|
}
|
|
341
|
+
function hasKnownContextWindow(value) {
|
|
342
|
+
return isRecord(value) && isSafeCounter(value.modelContextWindow) && value.modelContextWindow > 0;
|
|
343
|
+
}
|
|
108
344
|
function redact(value, limit = 16_384) {
|
|
109
345
|
return String(value)
|
|
110
346
|
.replace(/(authorization\s*[:=]\s*bearer\s+)[^\s]+/gi, '$1[REDACTED]')
|
|
@@ -115,69 +351,978 @@ function redact(value, limit = 16_384) {
|
|
|
115
351
|
.slice(0, limit);
|
|
116
352
|
}
|
|
117
353
|
|
|
118
|
-
|
|
354
|
+
function isUuid(value) {
|
|
355
|
+
return (
|
|
356
|
+
typeof value === 'string' &&
|
|
357
|
+
/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu.test(value)
|
|
358
|
+
);
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
function boundedDiagnosticValue(value) {
|
|
362
|
+
return typeof value === 'string' && value.length > 0 && value.length <= MAX_DIAGNOSTIC_STRING
|
|
363
|
+
? value
|
|
364
|
+
: null;
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
function isSafeCorrelationId(value) {
|
|
368
|
+
return typeof value === 'string' && /^[A-Za-z0-9._:-]{1,128}$/u.test(value);
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
function boundedCorrelationValue(value) {
|
|
372
|
+
return typeof value === 'string' && value.length > 0 && value.length <= 256 ? value : null;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
function safeRpcCode(value) {
|
|
376
|
+
return Number.isSafeInteger(value) && value >= -32_768 && value <= 32_767 ? value : null;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
function clientRequestKey(value) {
|
|
380
|
+
if (typeof value === 'string' && value.length > 0 && value.length <= 128) return `s:${value}`;
|
|
381
|
+
if (Number.isSafeInteger(value)) return `n:${value}`;
|
|
382
|
+
return null;
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/**
|
|
386
|
+
* Closed private diagnostic contract consumed by the client sink. It contains
|
|
387
|
+
* stage/reason enums, pseudonymous session identity, operation/correlation
|
|
388
|
+
* identifiers and bounded counts only. It never accepts raw MCP/provider text,
|
|
389
|
+
* prompts, notes, paths or thread IDs.
|
|
390
|
+
*/
|
|
391
|
+
export function normalizeCodexDiagnostic(event, { threadId = null } = {}) {
|
|
392
|
+
if (!isRecord(event)) return null;
|
|
393
|
+
const stage = DIAGNOSTIC_STAGES.has(event.stage)
|
|
394
|
+
? event.stage
|
|
395
|
+
: DIAGNOSTIC_STAGES.has(event.phase)
|
|
396
|
+
? event.phase
|
|
397
|
+
: null;
|
|
398
|
+
const reason = DIAGNOSTIC_REASONS.has(event.reason)
|
|
399
|
+
? event.reason
|
|
400
|
+
: event.reasonCode === 'candidate_window_bounded'
|
|
401
|
+
? 'candidate_window_bounded'
|
|
402
|
+
: null;
|
|
403
|
+
if (stage === null || reason === null) return null;
|
|
404
|
+
const operationId = isUuid(event.operationId) ? event.operationId : undefined;
|
|
405
|
+
const correlationId = isSafeCorrelationId(event.correlationId) ? event.correlationId : null;
|
|
406
|
+
const rpcCode = safeRpcCode(event.rpcCode ?? event.rpc_code);
|
|
407
|
+
const counts = isRecord(event.counts)
|
|
408
|
+
? Object.fromEntries(
|
|
409
|
+
Object.entries(event.counts).filter(
|
|
410
|
+
([key, value]) =>
|
|
411
|
+
/^[a-z_]{1,48}$/u.test(key) &&
|
|
412
|
+
isSafeBoundedInteger(value, 0, /_bytes$/u.test(key) ? MAX_DIAGNOSTIC_BYTES : 1_000_000),
|
|
413
|
+
),
|
|
414
|
+
)
|
|
415
|
+
: undefined;
|
|
416
|
+
const result = {
|
|
417
|
+
schema_version: 1,
|
|
418
|
+
phase: boundedDiagnosticValue(event.phase) ?? `${stage}_${reason}`,
|
|
419
|
+
stage,
|
|
420
|
+
reason,
|
|
421
|
+
...(operationId === undefined ? {} : { operation_id: operationId }),
|
|
422
|
+
...(correlationId === null ? {} : { correlation_id: correlationId }),
|
|
423
|
+
...(rpcCode === null ? {} : { rpc_code: rpcCode }),
|
|
424
|
+
...(typeof threadId === 'string' && threadId.length > 0
|
|
425
|
+
? { session_key: hashText(threadId) }
|
|
426
|
+
: {}),
|
|
427
|
+
...(counts && Object.keys(counts).length > 0 ? { counts } : {}),
|
|
428
|
+
};
|
|
429
|
+
return result;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
function parseJsonText(value) {
|
|
433
|
+
if (typeof value !== 'string' || Buffer.byteLength(value, 'utf8') > MAX_SELECTION_BYTES)
|
|
434
|
+
return null;
|
|
435
|
+
try {
|
|
436
|
+
const parsed = JSON.parse(value);
|
|
437
|
+
return isRecord(parsed) ? parsed : null;
|
|
438
|
+
} catch {
|
|
439
|
+
return null;
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
function responseStructuredContent(response) {
|
|
444
|
+
if (!isRecord(response)) return null;
|
|
445
|
+
if (isRecord(response.structuredContent)) return response.structuredContent;
|
|
446
|
+
const content = Array.isArray(response.content) ? response.content : [];
|
|
447
|
+
const textPart = content.find((part) => isRecord(part) && part.type === 'text');
|
|
448
|
+
return textPart ? parseJsonText(textPart.text) : null;
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
/** Parse only the closed Context error envelope or known fixed-code text form. */
|
|
452
|
+
export function parseContextError(response) {
|
|
453
|
+
const structured = responseStructuredContent(response);
|
|
454
|
+
let value = structured;
|
|
455
|
+
if (value === null) {
|
|
456
|
+
const content = Array.isArray(response?.content) ? response.content : [];
|
|
457
|
+
const textPart = content.find((part) => isRecord(part) && part.type === 'text');
|
|
458
|
+
const text = typeof textPart?.text === 'string' ? textPart.text : '';
|
|
459
|
+
const fixed =
|
|
460
|
+
/^(INVALID_INPUT|FEATURE_DISABLED|PAID_PLAN_REQUIRED|ENTITLEMENT_UNAVAILABLE|RATE_LIMITED|PROVIDER_NOT_CONFIGURED|BUDGET_EXCEEDED|PROVIDER_TIMEOUT|PROVIDER_UNAVAILABLE|INVALID_PROVIDER_RESPONSE)(?::|$)/u.exec(
|
|
461
|
+
text,
|
|
462
|
+
);
|
|
463
|
+
if (!fixed) return null;
|
|
464
|
+
value = { error_code: fixed[1] };
|
|
465
|
+
}
|
|
466
|
+
if (
|
|
467
|
+
!CONTEXT_ERROR_CODES.has(value.error_code) ||
|
|
468
|
+
(value.failure_detail !== undefined && !CONTEXT_FAILURE_DETAILS.has(value.failure_detail)) ||
|
|
469
|
+
(value.admission_reason !== undefined &&
|
|
470
|
+
!CONTEXT_ADMISSION_REASONS.has(value.admission_reason)) ||
|
|
471
|
+
(value.correlation_id !== undefined && !isSafeCorrelationId(value.correlation_id)) ||
|
|
472
|
+
(value.request_id !== undefined && !isUuid(value.request_id)) ||
|
|
473
|
+
(value.retry_after_ms !== undefined &&
|
|
474
|
+
!isSafeBoundedInteger(value.retry_after_ms, 0, 3_600_000))
|
|
475
|
+
)
|
|
476
|
+
return null;
|
|
477
|
+
return {
|
|
478
|
+
errorCode: value.error_code,
|
|
479
|
+
failureDetail: value.failure_detail ?? null,
|
|
480
|
+
admissionReason: value.admission_reason ?? null,
|
|
481
|
+
correlationId: boundedCorrelationValue(value.correlation_id),
|
|
482
|
+
requestId: isUuid(value.request_id) ? value.request_id : null,
|
|
483
|
+
retryAfterMs: value.retry_after_ms ?? null,
|
|
484
|
+
quota:
|
|
485
|
+
value.error_code === 'RATE_LIMITED' ||
|
|
486
|
+
value.failure_detail === 'shared_admission_denied' ||
|
|
487
|
+
SHARED_QUOTA_ADMISSION_REASONS.has(value.admission_reason),
|
|
488
|
+
};
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
export function parseBudgetSnapshot(value) {
|
|
492
|
+
if (
|
|
493
|
+
!isRecord(value) ||
|
|
494
|
+
!hasOnlyKeys(value, [
|
|
495
|
+
'schema_version',
|
|
496
|
+
'server_time',
|
|
497
|
+
'expires_at',
|
|
498
|
+
'window_end',
|
|
499
|
+
'available_requests',
|
|
500
|
+
'available_tokens',
|
|
501
|
+
'output_token_allowance',
|
|
502
|
+
'max_batches_per_operation',
|
|
503
|
+
'account_in_flight',
|
|
504
|
+
'retry_after_ms',
|
|
505
|
+
'provider_batches',
|
|
506
|
+
'provider_model',
|
|
507
|
+
'correlation_id',
|
|
508
|
+
]) ||
|
|
509
|
+
value.schema_version !== 1 ||
|
|
510
|
+
!isTimestamp(value.server_time) ||
|
|
511
|
+
!isTimestamp(value.expires_at) ||
|
|
512
|
+
!isTimestamp(value.window_end) ||
|
|
513
|
+
Date.parse(value.expires_at) <= Date.parse(value.server_time) ||
|
|
514
|
+
Date.parse(value.expires_at) - Date.parse(value.server_time) > MAX_BUDGET_SNAPSHOT_AGE_MS ||
|
|
515
|
+
Date.parse(value.expires_at) > Date.parse(value.window_end) ||
|
|
516
|
+
!isSafeBoundedInteger(value.available_requests, 0, 1_000_000) ||
|
|
517
|
+
!isSafeBoundedInteger(value.available_tokens, 0, Number.MAX_SAFE_INTEGER) ||
|
|
518
|
+
!isSafeBoundedInteger(value.output_token_allowance, 0, Number.MAX_SAFE_INTEGER) ||
|
|
519
|
+
!isSafeBoundedInteger(value.max_batches_per_operation, 0, 4) ||
|
|
520
|
+
typeof value.account_in_flight !== 'boolean' ||
|
|
521
|
+
!isSafeBoundedInteger(value.retry_after_ms, 0, 3_600_000) ||
|
|
522
|
+
value.provider_batches !== 0 ||
|
|
523
|
+
value.provider_model !== null ||
|
|
524
|
+
!isSafeCorrelationId(value.correlation_id)
|
|
525
|
+
)
|
|
526
|
+
return null;
|
|
527
|
+
return value;
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
function budgetSnapshotFresh(snapshot, receivedAt, currentTime) {
|
|
531
|
+
if (!isRecord(snapshot) || !Number.isSafeInteger(receivedAt) || currentTime < receivedAt)
|
|
532
|
+
return false;
|
|
533
|
+
const lifetime = Date.parse(snapshot.expires_at) - Date.parse(snapshot.server_time);
|
|
534
|
+
return (
|
|
535
|
+
lifetime > 0 && lifetime <= MAX_BUDGET_SNAPSHOT_AGE_MS && currentTime - receivedAt <= lifetime
|
|
536
|
+
);
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
function diagnosticReasonForError(error) {
|
|
540
|
+
if (DIAGNOSTIC_REASONS.has(error?.diagnosticReason)) return error.diagnosticReason;
|
|
541
|
+
if (safeRpcCode(error?.rpcCode) !== null) return 'protocol_error';
|
|
542
|
+
const message = error instanceof Error ? error.message : String(error ?? '');
|
|
543
|
+
if (
|
|
544
|
+
/snapshot exceeds bound|capture exceeds|capture size|oversized internal response/iu.test(
|
|
545
|
+
message,
|
|
546
|
+
)
|
|
547
|
+
)
|
|
548
|
+
return 'capture_size_exceeded';
|
|
549
|
+
if (/capture item count|source index exceeds bound/iu.test(message))
|
|
550
|
+
return 'capture_item_count_exceeded';
|
|
551
|
+
if (/capture storage capacity|storage capacity|capture directory/iu.test(message))
|
|
552
|
+
return 'storage_capacity_exceeded';
|
|
553
|
+
if (/capture source identity validation/iu.test(message)) return 'source_changed';
|
|
554
|
+
if (/capture candidate|checkpoint target/iu.test(message)) return 'private_storage_failed';
|
|
555
|
+
if (/thread\/(?:read|turns\/list)|capture history|page unavailable/iu.test(message))
|
|
556
|
+
return 'history_unavailable';
|
|
557
|
+
if (/checkpoint|recovery notes|private (?:file|storage)|storage|EACCES|ENOSPC/iu.test(message))
|
|
558
|
+
return 'private_storage_failed';
|
|
559
|
+
if (/tim(?:e|ed)[ -]?out/iu.test(message)) return 'timeout';
|
|
560
|
+
if (/closed|deleted/iu.test(message)) return 'thread_closed';
|
|
561
|
+
if (/source changed|state changed|stale/iu.test(message)) return 'source_changed';
|
|
562
|
+
if (/candidate|history|thread\/read/iu.test(message)) return 'history_unavailable';
|
|
563
|
+
return 'request_failed';
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
function rpcFailure(kind, error) {
|
|
567
|
+
const failure = new Error(`${kind} request rejected by the app-server`);
|
|
568
|
+
failure.name = 'CodexProtocolError';
|
|
569
|
+
failure.diagnosticReason = 'protocol_error';
|
|
570
|
+
failure.rpcCode = safeRpcCode(error?.code);
|
|
571
|
+
return failure;
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
function boundedFailureCounts(error) {
|
|
575
|
+
if (!isRecord(error)) return undefined;
|
|
576
|
+
const counts = {};
|
|
577
|
+
for (const [source, target] of [
|
|
578
|
+
['captureItemCount', 'capture_item_count'],
|
|
579
|
+
['captureItemLimit', 'capture_item_limit'],
|
|
580
|
+
['captureBytes', 'capture_bytes'],
|
|
581
|
+
['captureLimitBytes', 'capture_limit_bytes'],
|
|
582
|
+
['responseBytes', 'response_bytes'],
|
|
583
|
+
['responseLimitBytes', 'response_limit_bytes'],
|
|
584
|
+
['captureDirectoryBytes', 'capture_directory_bytes'],
|
|
585
|
+
['captureDirectoryLimitBytes', 'capture_directory_limit_bytes'],
|
|
586
|
+
['captureFileCount', 'capture_file_count'],
|
|
587
|
+
['captureFileLimit', 'capture_file_limit'],
|
|
588
|
+
]) {
|
|
589
|
+
const maximum = target.includes('directory_bytes')
|
|
590
|
+
? MAX_CAPTURE_DIRECTORY_BYTES
|
|
591
|
+
: target.includes('bytes')
|
|
592
|
+
? MAX_DIAGNOSTIC_BYTES
|
|
593
|
+
: 1_000_000;
|
|
594
|
+
if (isSafeBoundedInteger(error[source], 0, maximum)) counts[target] = error[source];
|
|
595
|
+
}
|
|
596
|
+
return Object.keys(counts).length > 0 ? counts : undefined;
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
async function ensurePrivateDirectory(directory, fsOps = captureFs) {
|
|
119
600
|
const target = resolve(directory);
|
|
120
|
-
await mkdir(target, { recursive: true, mode: 0o700 });
|
|
601
|
+
await fsOps.mkdir(target, { recursive: true, mode: 0o700 });
|
|
121
602
|
const parts = target.split('/');
|
|
122
603
|
let current = parts[0] === '' ? '/' : parts[0];
|
|
123
604
|
for (const part of parts.slice(parts[0] === '' ? 1 : 1)) {
|
|
124
605
|
if (!part) continue;
|
|
125
606
|
current = current === '/' ? `/${part}` : `${current}/${part}`;
|
|
126
|
-
const info = await lstat(current);
|
|
607
|
+
const info = await fsOps.lstat(current);
|
|
127
608
|
if (info.isSymbolicLink() || !info.isDirectory())
|
|
128
609
|
throw new Error('checkpoint directory is unsafe');
|
|
129
610
|
}
|
|
130
|
-
await chmod(target, 0o700);
|
|
611
|
+
await fsOps.chmod(target, 0o700);
|
|
131
612
|
return target;
|
|
132
613
|
}
|
|
133
|
-
async function assertPrivateFile(file, label) {
|
|
134
|
-
const info = await lstat(file);
|
|
614
|
+
async function assertPrivateFile(file, label, fsOps = captureFs) {
|
|
615
|
+
const info = await fsOps.lstat(file);
|
|
135
616
|
if (info.isSymbolicLink() || !info.isFile() || (info.mode & 0o077) !== 0)
|
|
136
617
|
throw new Error(`${label} is not a private regular file`);
|
|
137
618
|
return info;
|
|
138
619
|
}
|
|
139
|
-
async function readPrivateFile(file, label, maxBytes = null) {
|
|
140
|
-
const info = await assertPrivateFile(file, label);
|
|
620
|
+
async function readPrivateFile(file, label, maxBytes = null, fsOps = captureFs) {
|
|
621
|
+
const info = await assertPrivateFile(file, label, fsOps);
|
|
141
622
|
if (maxBytes !== null && info.size > maxBytes) throw new Error(`${label} exceeds its bound`);
|
|
142
|
-
return readFile(file, 'utf8');
|
|
623
|
+
return fsOps.readFile(file, 'utf8');
|
|
143
624
|
}
|
|
144
625
|
|
|
145
|
-
|
|
146
|
-
const
|
|
626
|
+
function captureCapacityError(state) {
|
|
627
|
+
const error = new Error('capture storage capacity exceeded');
|
|
628
|
+
error.diagnosticReason = 'storage_capacity_exceeded';
|
|
629
|
+
error.captureDirectoryBytes = state?.totalBytes ?? MAX_CAPTURE_DIRECTORY_BYTES;
|
|
630
|
+
error.captureDirectoryLimitBytes = MAX_CAPTURE_DIRECTORY_BYTES;
|
|
631
|
+
error.captureFileCount = state?.managedFiles?.length ?? MAX_CAPTURE_FILES;
|
|
632
|
+
error.captureFileLimit = MAX_CAPTURE_FILES;
|
|
633
|
+
return error;
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
function validateCaptureSnapshot(value, expectedSourceRevision = null) {
|
|
637
|
+
if (
|
|
638
|
+
!isRecord(value) ||
|
|
639
|
+
![1, 2].includes(value.version) ||
|
|
640
|
+
typeof value.threadId !== 'string' ||
|
|
641
|
+
typeof value.sourceRevision !== 'string' ||
|
|
642
|
+
!/^[a-f0-9]{64}$/u.test(value.sourceRevision) ||
|
|
643
|
+
!isRecord(value.threadRead) ||
|
|
644
|
+
value.threadRead.thread?.id !== value.threadId ||
|
|
645
|
+
(value.version === 1 && hashJson(value.threadRead) !== value.sourceRevision) ||
|
|
646
|
+
(value.version === 2 &&
|
|
647
|
+
(!isRecord(value.captureManifest) ||
|
|
648
|
+
value.captureManifest.version !== 'codex-capture-manifest-v1' ||
|
|
649
|
+
value.captureManifest.sourceRevision !== value.sourceRevision ||
|
|
650
|
+
value.captureManifest.threadId !== value.threadId ||
|
|
651
|
+
!isSafeBoundedInteger(value.captureManifest.pageCount, 1, MAX_CAPTURE_PAGES) ||
|
|
652
|
+
!isSafeBoundedInteger(value.captureManifest.itemCount, 0, MAX_CAPTURE_ITEMS) ||
|
|
653
|
+
!isSafeBoundedInteger(value.captureManifest.totalBytes, 1, MAX_CAPTURE_TOTAL_BYTES) ||
|
|
654
|
+
!Array.isArray(value.captureManifest.segments) ||
|
|
655
|
+
value.captureManifest.segments.length !== value.captureManifest.pageCount ||
|
|
656
|
+
!Array.isArray(value.captureManifest.pageHashes) ||
|
|
657
|
+
value.captureManifest.pageHashes.length !== value.captureManifest.pageCount ||
|
|
658
|
+
value.captureManifest.pageHashes.some((hash) => !/^[a-f0-9]{64}$/u.test(hash)) ||
|
|
659
|
+
captureManifestRevision(value.threadRead.thread, value.captureManifest.pageHashes) !==
|
|
660
|
+
value.sourceRevision ||
|
|
661
|
+
value.captureManifest.segments.some(
|
|
662
|
+
(segment, index) =>
|
|
663
|
+
segment !== captureSegmentName(value.threadId, value.sourceRevision, index),
|
|
664
|
+
))) ||
|
|
665
|
+
(expectedSourceRevision !== null && value.sourceRevision !== expectedSourceRevision)
|
|
666
|
+
)
|
|
667
|
+
throw new Error('capture source identity validation failed');
|
|
668
|
+
strictThreadName(value.threadId);
|
|
669
|
+
return value;
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
function validateCaptureSegment(value, expectedThreadId = null, expectedSourceRevision = null) {
|
|
673
|
+
if (
|
|
674
|
+
!isRecord(value) ||
|
|
675
|
+
value.version !== 'codex-capture-segment-v1' ||
|
|
676
|
+
typeof value.threadId !== 'string' ||
|
|
677
|
+
(expectedThreadId !== null && value.threadId !== expectedThreadId) ||
|
|
678
|
+
typeof value.sourceRevision !== 'string' ||
|
|
679
|
+
!/^[a-f0-9]{64}$/u.test(value.sourceRevision) ||
|
|
680
|
+
(expectedSourceRevision !== null && value.sourceRevision !== expectedSourceRevision) ||
|
|
681
|
+
!isSafeBoundedInteger(value.pageIndex, 0, MAX_CAPTURE_PAGES - 1) ||
|
|
682
|
+
!isRecord(value.page) ||
|
|
683
|
+
value.page.threadId !== value.threadId ||
|
|
684
|
+
!Array.isArray(value.page.turns) ||
|
|
685
|
+
typeof value.pageHash !== 'string' ||
|
|
686
|
+
!/^[a-f0-9]{64}$/u.test(value.pageHash) ||
|
|
687
|
+
hashJson(value.page) !== value.pageHash ||
|
|
688
|
+
!isSafeBoundedInteger(value.itemCount, 0, MAX_CAPTURE_ITEMS)
|
|
689
|
+
)
|
|
690
|
+
throw new Error('capture segment identity validation failed');
|
|
691
|
+
return value;
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
function captureSegmentName(threadId, sourceRevision, pageIndex) {
|
|
695
|
+
return `${strictThreadName(threadId)}.${sourceRevision}.segment-${String(pageIndex).padStart(4, '0')}.json`;
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
function captureManifestRevision(metadataThread, pageHashes) {
|
|
699
|
+
const thread = Object.fromEntries(
|
|
700
|
+
Object.entries(metadataThread).filter(([key]) => key !== 'turns'),
|
|
701
|
+
);
|
|
702
|
+
return hashJson({ thread, pages: pageHashes });
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
function captureNameForSnapshot(value, suffix = '') {
|
|
706
|
+
return `${strictThreadName(value.threadId)}${suffix}.json`;
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
async function readCaptureDirectoryState(directory, fsOps = captureFs) {
|
|
710
|
+
const entries = await fsOps.readdir(directory, { withFileTypes: true });
|
|
711
|
+
if (entries.length > MAX_CAPTURE_FILES * 4)
|
|
712
|
+
throw captureCapacityError({ totalBytes: MAX_CAPTURE_DIRECTORY_BYTES, managedFiles: [] });
|
|
713
|
+
const state = {
|
|
714
|
+
entryCount: entries.length,
|
|
715
|
+
totalBytes: 0,
|
|
716
|
+
managedFiles: [],
|
|
717
|
+
referencedImmutableNames: new Set(),
|
|
718
|
+
protectedGroups: new Set(),
|
|
719
|
+
manifestRefs: [],
|
|
720
|
+
blockedEntries: false,
|
|
721
|
+
unsafeRecovery: false,
|
|
722
|
+
};
|
|
723
|
+
for (const entry of entries) {
|
|
724
|
+
if (entry.name === '.capture.lock') continue;
|
|
725
|
+
const file = resolve(directory, entry.name);
|
|
726
|
+
const info = await fsOps.lstat(file);
|
|
727
|
+
state.totalBytes += info.size;
|
|
728
|
+
if (
|
|
729
|
+
info.isSymbolicLink() ||
|
|
730
|
+
!info.isFile() ||
|
|
731
|
+
(typeof process.getuid === 'function' && info.uid !== process.getuid()) ||
|
|
732
|
+
(info.mode & 0o077) !== 0
|
|
733
|
+
) {
|
|
734
|
+
state.blockedEntries = true;
|
|
735
|
+
continue;
|
|
736
|
+
}
|
|
737
|
+
if (!entry.name.endsWith('.json')) continue;
|
|
738
|
+
let value;
|
|
739
|
+
try {
|
|
740
|
+
if (info.size > MAX_CAPTURE_BYTES) {
|
|
741
|
+
if (entry.name.endsWith('.recovery.json')) state.unsafeRecovery = true;
|
|
742
|
+
continue;
|
|
743
|
+
}
|
|
744
|
+
value = JSON.parse(await fsOps.readFile(file, 'utf8'));
|
|
745
|
+
} catch {
|
|
746
|
+
if (entry.name.endsWith('.recovery.json')) state.unsafeRecovery = true;
|
|
747
|
+
continue;
|
|
748
|
+
}
|
|
749
|
+
if (entry.name.endsWith('.recovery.json')) {
|
|
750
|
+
if (
|
|
751
|
+
isRecord(value) &&
|
|
752
|
+
typeof value.threadId === 'string' &&
|
|
753
|
+
/^[a-f0-9]{64}$/u.test(value.sourceRevision) &&
|
|
754
|
+
typeof value.notes === 'string' &&
|
|
755
|
+
typeof value.notesHash === 'string' &&
|
|
756
|
+
value.notesHash === hashText(value.notes)
|
|
757
|
+
) {
|
|
758
|
+
try {
|
|
759
|
+
state.referencedImmutableNames.add(
|
|
760
|
+
`${strictThreadName(value.threadId)}.${value.sourceRevision}.json`,
|
|
761
|
+
);
|
|
762
|
+
state.protectedGroups.add(`${strictThreadName(value.threadId)}.${value.sourceRevision}`);
|
|
763
|
+
} catch {
|
|
764
|
+
state.unsafeRecovery = true;
|
|
765
|
+
}
|
|
766
|
+
} else state.unsafeRecovery = true;
|
|
767
|
+
if (isRecord(value) && typeof value.notes === 'string') {
|
|
768
|
+
const references = [
|
|
769
|
+
...value.notes.matchAll(/capture_file:\s+.*?([A-Za-z0-9._-]+\.[a-f0-9]{64}\.json)/gu),
|
|
770
|
+
];
|
|
771
|
+
for (const match of references) state.referencedImmutableNames.add(match[1]);
|
|
772
|
+
}
|
|
773
|
+
continue;
|
|
774
|
+
}
|
|
775
|
+
if (isRecord(value) && value.version === 'codex-capture-segment-v1') {
|
|
776
|
+
try {
|
|
777
|
+
validateCaptureSegment(value);
|
|
778
|
+
const expectedName = captureSegmentName(
|
|
779
|
+
value.threadId,
|
|
780
|
+
value.sourceRevision,
|
|
781
|
+
value.pageIndex,
|
|
782
|
+
);
|
|
783
|
+
if (entry.name !== expectedName) continue;
|
|
784
|
+
state.managedFiles.push({
|
|
785
|
+
file,
|
|
786
|
+
name: entry.name,
|
|
787
|
+
size: info.size,
|
|
788
|
+
mtimeMs: info.mtimeMs,
|
|
789
|
+
immutable: true,
|
|
790
|
+
group: `${value.threadId}.${value.sourceRevision}`,
|
|
791
|
+
});
|
|
792
|
+
} catch {
|
|
793
|
+
state.blockedEntries = true;
|
|
794
|
+
}
|
|
795
|
+
continue;
|
|
796
|
+
}
|
|
797
|
+
try {
|
|
798
|
+
validateCaptureSnapshot(value);
|
|
799
|
+
} catch {
|
|
800
|
+
continue;
|
|
801
|
+
}
|
|
802
|
+
const sourceName = `${strictThreadName(value.threadId)}.${value.sourceRevision}.json`;
|
|
803
|
+
if (
|
|
804
|
+
!new Set([
|
|
805
|
+
captureNameForSnapshot(value),
|
|
806
|
+
captureNameForSnapshot(value, '.completed'),
|
|
807
|
+
sourceName,
|
|
808
|
+
]).has(entry.name)
|
|
809
|
+
)
|
|
810
|
+
continue;
|
|
811
|
+
state.managedFiles.push({
|
|
812
|
+
file,
|
|
813
|
+
name: entry.name,
|
|
814
|
+
size: info.size,
|
|
815
|
+
mtimeMs: info.mtimeMs,
|
|
816
|
+
immutable: entry.name === sourceName,
|
|
817
|
+
group: `${value.threadId}.${value.sourceRevision}`,
|
|
818
|
+
});
|
|
819
|
+
if (!entry.name.endsWith('.' + value.sourceRevision + '.json'))
|
|
820
|
+
state.referencedImmutableNames.add(sourceName);
|
|
821
|
+
if (value.version === 2 && entry.name !== sourceName) {
|
|
822
|
+
for (const segment of value.captureManifest.segments) {
|
|
823
|
+
if (typeof segment === 'string') state.referencedImmutableNames.add(segment);
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
if (value.version === 2) state.manifestRefs.push(value.captureManifest);
|
|
827
|
+
if (
|
|
828
|
+
value.version === 2 &&
|
|
829
|
+
state.protectedGroups.has(`${value.threadId}.${value.sourceRevision}`)
|
|
830
|
+
) {
|
|
831
|
+
for (const segment of value.captureManifest.segments)
|
|
832
|
+
state.referencedImmutableNames.add(segment);
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
const managedNames = new Set(state.managedFiles.map((entry) => entry.name));
|
|
836
|
+
for (const manifest of state.manifestRefs) {
|
|
837
|
+
if (manifest.segments.some((segment) => !managedNames.has(segment)))
|
|
838
|
+
state.blockedEntries = true;
|
|
839
|
+
}
|
|
840
|
+
for (const entry of state.managedFiles) {
|
|
841
|
+
if (entry.group && state.referencedImmutableNames.has(entry.name))
|
|
842
|
+
state.protectedGroups.add(entry.group);
|
|
843
|
+
}
|
|
844
|
+
return state;
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
async function withCaptureStoreLock(
|
|
848
|
+
directory,
|
|
849
|
+
task,
|
|
850
|
+
fsOps = captureFs,
|
|
851
|
+
{
|
|
852
|
+
now = Date.now,
|
|
853
|
+
sleep = (milliseconds) =>
|
|
854
|
+
new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds)),
|
|
855
|
+
} = {},
|
|
856
|
+
) {
|
|
857
|
+
const lockPath = resolve(directory, '.capture.lock');
|
|
858
|
+
const started = now();
|
|
859
|
+
while (true) {
|
|
860
|
+
let handle;
|
|
861
|
+
try {
|
|
862
|
+
handle = await fsOps.open(lockPath, 'wx', 0o600);
|
|
863
|
+
try {
|
|
864
|
+
return await task();
|
|
865
|
+
} finally {
|
|
866
|
+
await handle.close();
|
|
867
|
+
await fsOps.unlink(lockPath).catch(() => undefined);
|
|
868
|
+
}
|
|
869
|
+
} catch (error) {
|
|
870
|
+
if (handle) throw error;
|
|
871
|
+
if (error?.code !== 'EEXIST') throw error;
|
|
872
|
+
if (now() - started >= CAPTURE_LOCK_WAIT_MS)
|
|
873
|
+
throw captureCapacityError({ totalBytes: 0, managedFiles: [] });
|
|
874
|
+
await sleep(10);
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
async function writePrivateFile(file, serialized, fsOps = captureFs) {
|
|
880
|
+
const directory = await ensurePrivateDirectory(dirname(file), fsOps);
|
|
147
881
|
const target = resolve(
|
|
148
882
|
directory,
|
|
149
883
|
file.startsWith(directory) ? file.slice(directory.length + 1) : file,
|
|
150
884
|
);
|
|
151
885
|
try {
|
|
152
|
-
await assertPrivateFile(target, 'checkpoint target');
|
|
886
|
+
await assertPrivateFile(target, 'checkpoint target', fsOps);
|
|
153
887
|
} catch (error) {
|
|
154
888
|
if (error?.code !== 'ENOENT') throw error;
|
|
155
889
|
}
|
|
156
890
|
const temporary = `${target}.${process.pid}.${randomUUID()}.tmp`;
|
|
157
891
|
try {
|
|
158
|
-
await writeFile(temporary,
|
|
159
|
-
await rename(temporary, target);
|
|
892
|
+
await fsOps.writeFile(temporary, serialized, { flag: 'wx', mode: 0o600 });
|
|
893
|
+
await fsOps.rename(temporary, target);
|
|
160
894
|
} catch (error) {
|
|
895
|
+
await fsOps.unlink(temporary).catch(() => undefined);
|
|
896
|
+
throw error;
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
async function writeBoundedCapture(file, serialized, value, fsOps = captureFs, lockOptions = {}) {
|
|
901
|
+
const directory = await ensurePrivateDirectory(dirname(file), fsOps);
|
|
902
|
+
const target = resolve(
|
|
903
|
+
directory,
|
|
904
|
+
file.startsWith(directory) ? file.slice(directory.length + 1) : file,
|
|
905
|
+
);
|
|
906
|
+
const targetName = target.slice(directory.length + 1);
|
|
907
|
+
const operation = async () => {
|
|
908
|
+
const temporary = `${target}.${process.pid}.${randomUUID()}.tmp`;
|
|
909
|
+
const sourceName = `${strictThreadName(value.threadId)}.${value.sourceRevision}.json`;
|
|
910
|
+
const expectedNames = new Set([
|
|
911
|
+
captureNameForSnapshot(value),
|
|
912
|
+
captureNameForSnapshot(value, '.completed'),
|
|
913
|
+
sourceName,
|
|
914
|
+
]);
|
|
915
|
+
let existingTarget = null;
|
|
161
916
|
try {
|
|
162
|
-
await
|
|
163
|
-
|
|
164
|
-
|
|
917
|
+
existingTarget = await assertPrivateFile(target, 'capture target', fsOps);
|
|
918
|
+
const existing = JSON.parse(
|
|
919
|
+
await readPrivateFile(target, 'capture target', MAX_CAPTURE_BYTES, fsOps),
|
|
920
|
+
);
|
|
921
|
+
validateCaptureSnapshot(existing);
|
|
922
|
+
if (!expectedNames.has(targetName) || existing.threadId !== value.threadId)
|
|
923
|
+
throw new Error('capture target identity validation failed');
|
|
924
|
+
if (targetName === sourceName && existing.sourceRevision === value.sourceRevision) return;
|
|
925
|
+
} catch (error) {
|
|
926
|
+
if (error?.code !== 'ENOENT') throw error;
|
|
927
|
+
}
|
|
928
|
+
try {
|
|
929
|
+
await fsOps.writeFile(temporary, serialized, { flag: 'wx', mode: 0o600 });
|
|
930
|
+
const written = await assertPrivateFile(temporary, 'capture candidate', fsOps);
|
|
931
|
+
if (written.size !== Buffer.byteLength(serialized, 'utf8'))
|
|
932
|
+
throw new Error('capture candidate read-back size mismatch');
|
|
933
|
+
validateCaptureSnapshot(
|
|
934
|
+
JSON.parse(await fsOps.readFile(temporary, 'utf8')),
|
|
935
|
+
value.sourceRevision,
|
|
936
|
+
);
|
|
937
|
+
const state = await readCaptureDirectoryState(directory, fsOps);
|
|
938
|
+
if (state.unsafeRecovery || state.blockedEntries) throw captureCapacityError(state);
|
|
939
|
+
const protectedNames = new Set([targetName, sourceName]);
|
|
940
|
+
if (value.version === 2 && Array.isArray(value.captureManifest?.segments)) {
|
|
941
|
+
for (const segment of value.captureManifest.segments) protectedNames.add(segment);
|
|
942
|
+
}
|
|
943
|
+
const targetWasManaged = state.managedFiles.some((entry) => entry.name === targetName);
|
|
944
|
+
const eligibleGroups = new Map();
|
|
945
|
+
if (!state.blockedEntries) {
|
|
946
|
+
for (const entry of state.managedFiles) {
|
|
947
|
+
if (
|
|
948
|
+
!entry.immutable ||
|
|
949
|
+
protectedNames.has(entry.name) ||
|
|
950
|
+
state.referencedImmutableNames.has(entry.name) ||
|
|
951
|
+
state.protectedGroups.has(entry.group)
|
|
952
|
+
)
|
|
953
|
+
continue;
|
|
954
|
+
const group = entry.group ?? entry.name;
|
|
955
|
+
const members = eligibleGroups.get(group) ?? [];
|
|
956
|
+
members.push(entry);
|
|
957
|
+
eligibleGroups.set(group, members);
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
const eligible = [...eligibleGroups.values()].sort(
|
|
961
|
+
(left, right) =>
|
|
962
|
+
Math.min(...left.map((entry) => entry.mtimeMs)) -
|
|
963
|
+
Math.min(...right.map((entry) => entry.mtimeMs)),
|
|
964
|
+
);
|
|
965
|
+
const targetEntry = state.managedFiles.find((entry) => entry.name === targetName);
|
|
966
|
+
const replacedBytes = targetEntry?.size ?? 0;
|
|
967
|
+
const temporaryHeadroom = 2 * MAX_CAPTURE_BYTES;
|
|
968
|
+
let projectedPeakBytes = state.totalBytes;
|
|
969
|
+
let projectedSteadyBytes = state.totalBytes - replacedBytes;
|
|
970
|
+
let projectedFiles = state.managedFiles.length + (targetWasManaged ? 0 : 1);
|
|
971
|
+
const retire = [];
|
|
972
|
+
for (const group of eligible) {
|
|
973
|
+
if (
|
|
974
|
+
projectedPeakBytes <= MAX_CAPTURE_DIRECTORY_BYTES + temporaryHeadroom &&
|
|
975
|
+
projectedSteadyBytes <= MAX_CAPTURE_DIRECTORY_BYTES &&
|
|
976
|
+
projectedFiles <= MAX_CAPTURE_FILES
|
|
977
|
+
)
|
|
978
|
+
break;
|
|
979
|
+
const groupBytes = group.reduce((sum, entry) => sum + entry.size, 0);
|
|
980
|
+
projectedPeakBytes -= groupBytes;
|
|
981
|
+
projectedSteadyBytes -= groupBytes;
|
|
982
|
+
projectedFiles -= group.length;
|
|
983
|
+
retire.push(...group);
|
|
984
|
+
}
|
|
985
|
+
if (
|
|
986
|
+
projectedPeakBytes > MAX_CAPTURE_DIRECTORY_BYTES + temporaryHeadroom ||
|
|
987
|
+
projectedSteadyBytes > MAX_CAPTURE_DIRECTORY_BYTES ||
|
|
988
|
+
projectedFiles > MAX_CAPTURE_FILES
|
|
989
|
+
)
|
|
990
|
+
throw captureCapacityError(state);
|
|
991
|
+
const backup = existingTarget
|
|
992
|
+
? `${target}.${process.pid}.${randomUUID()}.capture-backup`
|
|
993
|
+
: null;
|
|
994
|
+
const staged = [];
|
|
995
|
+
let backupCreated = false;
|
|
996
|
+
let candidatePublished = false;
|
|
997
|
+
try {
|
|
998
|
+
if (backup !== null) {
|
|
999
|
+
await fsOps.rename(target, backup);
|
|
1000
|
+
backupCreated = true;
|
|
1001
|
+
}
|
|
1002
|
+
await fsOps.rename(temporary, target);
|
|
1003
|
+
candidatePublished = true;
|
|
1004
|
+
for (const candidate of retire) {
|
|
1005
|
+
const stagedPath = `${candidate.file}.${process.pid}.${randomUUID()}.capture-retire`;
|
|
1006
|
+
await fsOps.rename(candidate.file, stagedPath);
|
|
1007
|
+
staged.push({ original: candidate.file, staged: stagedPath });
|
|
1008
|
+
}
|
|
1009
|
+
if (backup !== null) await fsOps.unlink(backup).catch(() => undefined);
|
|
1010
|
+
for (const item of staged) await fsOps.unlink(item.staged).catch(() => undefined);
|
|
1011
|
+
} catch (error) {
|
|
1012
|
+
if (candidatePublished) await fsOps.unlink(target).catch(() => undefined);
|
|
1013
|
+
for (const item of staged)
|
|
1014
|
+
await fsOps.rename(item.staged, item.original).catch(() => undefined);
|
|
1015
|
+
if (backupCreated && backup !== null)
|
|
1016
|
+
await fsOps.rename(backup, target).catch(() => undefined);
|
|
1017
|
+
throw error;
|
|
1018
|
+
}
|
|
1019
|
+
} catch (error) {
|
|
1020
|
+
await fsOps.unlink(temporary).catch(() => undefined);
|
|
1021
|
+
throw error;
|
|
165
1022
|
}
|
|
1023
|
+
};
|
|
1024
|
+
return lockOptions.lockHeld
|
|
1025
|
+
? operation()
|
|
1026
|
+
: withCaptureStoreLock(directory, operation, fsOps, lockOptions);
|
|
1027
|
+
}
|
|
1028
|
+
|
|
1029
|
+
async function atomicWritePrivate(
|
|
1030
|
+
file,
|
|
1031
|
+
value,
|
|
1032
|
+
{
|
|
1033
|
+
maxBytes = null,
|
|
1034
|
+
compact = false,
|
|
1035
|
+
capture = false,
|
|
1036
|
+
captureLock = false,
|
|
1037
|
+
captureLockHeld = false,
|
|
1038
|
+
} = {},
|
|
1039
|
+
) {
|
|
1040
|
+
const serialized = `${compact ? JSON.stringify(value) : JSON.stringify(value, null, 2)}\n`;
|
|
1041
|
+
if (maxBytes !== null && Buffer.byteLength(serialized, 'utf8') > maxBytes) {
|
|
1042
|
+
const error = new Error('capture snapshot exceeds bound');
|
|
1043
|
+
error.diagnosticReason = 'capture_size_exceeded';
|
|
1044
|
+
error.captureBytes = Buffer.byteLength(serialized, 'utf8');
|
|
1045
|
+
error.captureLimitBytes = maxBytes;
|
|
166
1046
|
throw error;
|
|
167
1047
|
}
|
|
1048
|
+
if (capture) {
|
|
1049
|
+
const result = await writeBoundedCapture(file, serialized, value, captureFs, {
|
|
1050
|
+
lockHeld: captureLockHeld,
|
|
1051
|
+
});
|
|
1052
|
+
return result;
|
|
1053
|
+
}
|
|
1054
|
+
const directory = await ensurePrivateDirectory(dirname(file));
|
|
1055
|
+
if (captureLock) return withCaptureStoreLock(directory, () => writePrivateFile(file, serialized));
|
|
1056
|
+
return writePrivateFile(file, serialized);
|
|
168
1057
|
}
|
|
169
|
-
async function atomicWriteImmutablePrivate(file, value, sourceRevision) {
|
|
170
|
-
|
|
1058
|
+
async function atomicWriteImmutablePrivate(file, value, sourceRevision, captureLockHeld = false) {
|
|
1059
|
+
validateCaptureSnapshot(value, sourceRevision);
|
|
1060
|
+
await atomicWritePrivate(file, value, {
|
|
1061
|
+
maxBytes: MAX_CAPTURE_BYTES,
|
|
1062
|
+
compact: true,
|
|
1063
|
+
capture: true,
|
|
1064
|
+
captureLockHeld,
|
|
1065
|
+
});
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
async function writeSegmentedCapture(
|
|
1069
|
+
directory,
|
|
1070
|
+
{ threadId, sourceRevision, metadata, pages, lockHeld = false, fs = captureFs },
|
|
1071
|
+
) {
|
|
1072
|
+
if (!Array.isArray(pages) || pages.length < 1 || pages.length > MAX_CAPTURE_PAGES)
|
|
1073
|
+
throw new Error('capture page count exceeds bound');
|
|
1074
|
+
const segments = [];
|
|
1075
|
+
const prepared = [];
|
|
1076
|
+
const cleanupSource = async () => {
|
|
1077
|
+
for (const descriptor of pages) {
|
|
1078
|
+
if (descriptor?.stagedPath) await fs.unlink(descriptor.stagedPath).catch(() => undefined);
|
|
1079
|
+
}
|
|
1080
|
+
};
|
|
171
1081
|
try {
|
|
172
|
-
const
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
1082
|
+
for (const [descriptorPosition, descriptor] of pages.entries()) {
|
|
1083
|
+
if (
|
|
1084
|
+
!isRecord(descriptor) ||
|
|
1085
|
+
!isSafeBoundedInteger(descriptor.pageIndex, 0, MAX_CAPTURE_PAGES - 1) ||
|
|
1086
|
+
descriptor.pageIndex !== descriptorPosition ||
|
|
1087
|
+
typeof descriptor.stagedPath !== 'string' ||
|
|
1088
|
+
typeof descriptor.pageHash !== 'string' ||
|
|
1089
|
+
!/^[a-f0-9]{64}$/u.test(descriptor.pageHash)
|
|
1090
|
+
)
|
|
1091
|
+
throw new Error('capture staged page identity validation failed');
|
|
1092
|
+
const page = JSON.parse(
|
|
1093
|
+
await readPrivateFile(
|
|
1094
|
+
descriptor.stagedPath,
|
|
1095
|
+
'capture staged page',
|
|
1096
|
+
MAX_CAPTURE_PAGE_BYTES,
|
|
1097
|
+
fs,
|
|
1098
|
+
),
|
|
1099
|
+
);
|
|
1100
|
+
if (
|
|
1101
|
+
!isRecord(page) ||
|
|
1102
|
+
page.threadId !== threadId ||
|
|
1103
|
+
!Array.isArray(page.turns) ||
|
|
1104
|
+
hashJson(page) !== descriptor.pageHash
|
|
1105
|
+
)
|
|
1106
|
+
throw new Error('capture staged page integrity validation failed');
|
|
1107
|
+
const payload = {
|
|
1108
|
+
version: 'codex-capture-segment-v1',
|
|
1109
|
+
threadId,
|
|
1110
|
+
sourceRevision,
|
|
1111
|
+
pageIndex: descriptor.pageIndex,
|
|
1112
|
+
pageHash: descriptor.pageHash,
|
|
1113
|
+
itemCount: page.turns.reduce(
|
|
1114
|
+
(count, turn) => count + (Array.isArray(turn.items) ? turn.items.length : 0),
|
|
1115
|
+
0,
|
|
1116
|
+
),
|
|
1117
|
+
page,
|
|
1118
|
+
};
|
|
1119
|
+
validateCaptureSegment(payload, threadId, sourceRevision);
|
|
1120
|
+
const serialized = `${JSON.stringify(payload)}\n`;
|
|
1121
|
+
if (Buffer.byteLength(serialized, 'utf8') > MAX_CAPTURE_PAGE_BYTES)
|
|
1122
|
+
throw new Error('capture page exceeds bound');
|
|
1123
|
+
const stagedState = await readCaptureDirectoryState(directory, fs);
|
|
1124
|
+
if (
|
|
1125
|
+
stagedState.totalBytes + Buffer.byteLength(serialized, 'utf8') >
|
|
1126
|
+
MAX_CAPTURE_DIRECTORY_BYTES + MAX_CAPTURE_STAGING_HEADROOM_BYTES
|
|
1127
|
+
)
|
|
1128
|
+
throw captureCapacityError(stagedState);
|
|
1129
|
+
const stagedFinalPath = resolve(
|
|
1130
|
+
directory,
|
|
1131
|
+
`.capture-segment-stage.${safeThreadName(threadId)}.${randomUUID()}.${descriptor.pageIndex}.tmp`,
|
|
1132
|
+
);
|
|
1133
|
+
await fs.writeFile(stagedFinalPath, serialized, { flag: 'wx', mode: 0o600 });
|
|
1134
|
+
prepared.push(stagedFinalPath);
|
|
1135
|
+
segments.push({
|
|
1136
|
+
name: captureSegmentName(threadId, sourceRevision, descriptor.pageIndex),
|
|
1137
|
+
pageHash: descriptor.pageHash,
|
|
1138
|
+
itemCount: payload.itemCount,
|
|
1139
|
+
bytes: Buffer.byteLength(serialized, 'utf8'),
|
|
1140
|
+
stagedPath: descriptor.stagedPath,
|
|
1141
|
+
stagedFinalPath,
|
|
1142
|
+
});
|
|
1143
|
+
}
|
|
177
1144
|
} catch (error) {
|
|
178
|
-
|
|
1145
|
+
for (const file of prepared) await fs.unlink(file).catch(() => undefined);
|
|
1146
|
+
await cleanupSource();
|
|
1147
|
+
throw error;
|
|
1148
|
+
}
|
|
1149
|
+
const manifest = {
|
|
1150
|
+
version: 'codex-capture-manifest-v1',
|
|
1151
|
+
threadId,
|
|
1152
|
+
sourceRevision,
|
|
1153
|
+
pageCount: segments.length,
|
|
1154
|
+
itemCount: segments.reduce((count, segment) => count + segment.itemCount, 0),
|
|
1155
|
+
totalBytes: segments.reduce((count, segment) => count + segment.bytes, 0),
|
|
1156
|
+
segments: segments.map((segment) => segment.name),
|
|
1157
|
+
pageHashes: segments.map((segment) => segment.pageHash),
|
|
1158
|
+
};
|
|
1159
|
+
if (
|
|
1160
|
+
isRecord(metadata?.thread) &&
|
|
1161
|
+
captureManifestRevision(metadata.thread, manifest.pageHashes) !== sourceRevision
|
|
1162
|
+
) {
|
|
1163
|
+
await cleanupSource();
|
|
1164
|
+
for (const file of prepared) await fs.unlink(file).catch(() => undefined);
|
|
1165
|
+
throw new Error('capture manifest revision does not match page hashes');
|
|
1166
|
+
}
|
|
1167
|
+
if (manifest.totalBytes > MAX_CAPTURE_TOTAL_BYTES || manifest.itemCount > MAX_CAPTURE_ITEMS) {
|
|
1168
|
+
await cleanupSource();
|
|
1169
|
+
for (const file of prepared) await fs.unlink(file).catch(() => undefined);
|
|
1170
|
+
throw new Error('capture exceeds segmented bound');
|
|
1171
|
+
}
|
|
1172
|
+
const operation = async () => {
|
|
1173
|
+
const state = await readCaptureDirectoryState(directory, fs);
|
|
1174
|
+
if (state.unsafeRecovery || state.blockedEntries) {
|
|
1175
|
+
await cleanupSource();
|
|
1176
|
+
for (const file of prepared) await fs.unlink(file).catch(() => undefined);
|
|
1177
|
+
throw captureCapacityError(state);
|
|
1178
|
+
}
|
|
1179
|
+
const staged = [];
|
|
1180
|
+
const publishedTargets = [];
|
|
1181
|
+
try {
|
|
1182
|
+
for (const segment of segments) {
|
|
1183
|
+
const target = resolve(directory, segment.name);
|
|
1184
|
+
try {
|
|
1185
|
+
await assertPrivateFile(target, 'capture segment', fs);
|
|
1186
|
+
const existing = JSON.parse(
|
|
1187
|
+
await readPrivateFile(target, 'capture segment', MAX_CAPTURE_PAGE_BYTES, fs),
|
|
1188
|
+
);
|
|
1189
|
+
validateCaptureSegment(existing, threadId, sourceRevision);
|
|
1190
|
+
if (existing.pageHash !== segment.pageHash) throw new Error('capture segment collision');
|
|
1191
|
+
await fs.unlink(segment.stagedFinalPath).catch(() => undefined);
|
|
1192
|
+
continue;
|
|
1193
|
+
} catch (error) {
|
|
1194
|
+
if (error?.code !== 'ENOENT') throw error;
|
|
1195
|
+
}
|
|
1196
|
+
staged.push({ temporary: segment.stagedFinalPath, target, bytes: segment.bytes });
|
|
1197
|
+
}
|
|
1198
|
+
const protectedNames = new Set(segments.map((segment) => segment.name));
|
|
1199
|
+
const groups = new Map();
|
|
1200
|
+
for (const entry of state.managedFiles) {
|
|
1201
|
+
if (
|
|
1202
|
+
!entry.immutable ||
|
|
1203
|
+
protectedNames.has(entry.name) ||
|
|
1204
|
+
state.referencedImmutableNames.has(entry.name) ||
|
|
1205
|
+
state.protectedGroups.has(entry.group)
|
|
1206
|
+
)
|
|
1207
|
+
continue;
|
|
1208
|
+
const members = groups.get(entry.group ?? entry.name) ?? [];
|
|
1209
|
+
members.push(entry);
|
|
1210
|
+
groups.set(entry.group ?? entry.name, members);
|
|
1211
|
+
}
|
|
1212
|
+
const retired = [];
|
|
1213
|
+
let projected = state.totalBytes;
|
|
1214
|
+
let projectedFiles = state.managedFiles.length + staged.length;
|
|
1215
|
+
for (const members of [...groups.values()].sort(
|
|
1216
|
+
(left, right) =>
|
|
1217
|
+
Math.min(...left.map((entry) => entry.mtimeMs)) -
|
|
1218
|
+
Math.min(...right.map((entry) => entry.mtimeMs)),
|
|
1219
|
+
)) {
|
|
1220
|
+
if (projected <= MAX_CAPTURE_DIRECTORY_BYTES && projectedFiles <= MAX_CAPTURE_FILES) break;
|
|
1221
|
+
const bytes = members.reduce((sum, entry) => sum + entry.size, 0);
|
|
1222
|
+
projected -= bytes;
|
|
1223
|
+
projectedFiles -= members.length;
|
|
1224
|
+
retired.push(...members);
|
|
1225
|
+
}
|
|
1226
|
+
if (projected > MAX_CAPTURE_DIRECTORY_BYTES || projectedFiles > MAX_CAPTURE_FILES)
|
|
1227
|
+
throw captureCapacityError({ totalBytes: projected, managedFiles: state.managedFiles });
|
|
1228
|
+
const retiredPaths = [];
|
|
1229
|
+
try {
|
|
1230
|
+
for (const entry of retired) {
|
|
1231
|
+
const stagedPath = `${entry.file}.${process.pid}.${randomUUID()}.capture-retire`;
|
|
1232
|
+
await fs.rename(entry.file, stagedPath);
|
|
1233
|
+
retiredPaths.push({ original: entry.file, staged: stagedPath });
|
|
1234
|
+
}
|
|
1235
|
+
for (const item of staged) await fs.rename(item.temporary, item.target);
|
|
1236
|
+
publishedTargets.push(...staged.map((item) => item.target));
|
|
1237
|
+
for (const segment of segments) {
|
|
1238
|
+
const verify = JSON.parse(
|
|
1239
|
+
await readPrivateFile(
|
|
1240
|
+
resolve(directory, segment.name),
|
|
1241
|
+
'capture segment',
|
|
1242
|
+
MAX_CAPTURE_PAGE_BYTES,
|
|
1243
|
+
fs,
|
|
1244
|
+
),
|
|
1245
|
+
);
|
|
1246
|
+
validateCaptureSegment(verify, threadId, sourceRevision);
|
|
1247
|
+
}
|
|
1248
|
+
for (const segment of segments) await fs.unlink(segment.stagedPath).catch(() => undefined);
|
|
1249
|
+
for (const file of prepared) await fs.unlink(file).catch(() => undefined);
|
|
1250
|
+
for (const item of retiredPaths) await fs.unlink(item.staged).catch(() => undefined);
|
|
1251
|
+
return manifest;
|
|
1252
|
+
} catch (error) {
|
|
1253
|
+
for (const item of retiredPaths)
|
|
1254
|
+
await fs.rename(item.staged, item.original).catch(() => undefined);
|
|
1255
|
+
if (error?.code === 'ENOENT') throw captureCapacityError(state);
|
|
1256
|
+
throw error;
|
|
1257
|
+
}
|
|
1258
|
+
} catch (error) {
|
|
1259
|
+
for (const item of staged) {
|
|
1260
|
+
await fs.unlink(item.temporary).catch(() => undefined);
|
|
1261
|
+
await fs.unlink(item.target).catch(() => undefined);
|
|
1262
|
+
}
|
|
1263
|
+
for (const target of publishedTargets) await fs.unlink(target).catch(() => undefined);
|
|
1264
|
+
for (const segment of segments) await fs.unlink(segment.stagedPath).catch(() => undefined);
|
|
1265
|
+
for (const file of prepared) await fs.unlink(file).catch(() => undefined);
|
|
1266
|
+
throw error;
|
|
1267
|
+
}
|
|
1268
|
+
};
|
|
1269
|
+
return lockHeld ? operation() : withCaptureStoreLock(directory, operation, fs);
|
|
1270
|
+
}
|
|
1271
|
+
|
|
1272
|
+
/**
|
|
1273
|
+
* Test seam for the segmented capture store. Pages are written as private
|
|
1274
|
+
* staging files first, then published as one lock-bound capture group.
|
|
1275
|
+
*/
|
|
1276
|
+
export async function __testWriteSegmentedCapture(
|
|
1277
|
+
directory,
|
|
1278
|
+
{ threadId, sourceRevision, metadata = { thread: { id: threadId } }, pages },
|
|
1279
|
+
{ fs = captureFs } = {},
|
|
1280
|
+
) {
|
|
1281
|
+
await ensurePrivateDirectory(directory, fs);
|
|
1282
|
+
const staged = [];
|
|
1283
|
+
try {
|
|
1284
|
+
for (const [pageIndex, page] of pages.entries()) {
|
|
1285
|
+
const serialized = `${JSON.stringify(page)}\n`;
|
|
1286
|
+
const state = await readCaptureDirectoryState(directory, fs);
|
|
1287
|
+
if (state.totalBytes + Buffer.byteLength(serialized, 'utf8') > MAX_CAPTURE_DIRECTORY_BYTES)
|
|
1288
|
+
throw captureCapacityError(state);
|
|
1289
|
+
const stagedPath = resolve(
|
|
1290
|
+
directory,
|
|
1291
|
+
`.capture-stage.test.${safeThreadName(threadId)}.${randomUUID()}.${pageIndex}.tmp`,
|
|
1292
|
+
);
|
|
1293
|
+
await fs.writeFile(stagedPath, serialized, { flag: 'wx', mode: 0o600 });
|
|
1294
|
+
staged.push({
|
|
1295
|
+
pageIndex,
|
|
1296
|
+
stagedPath,
|
|
1297
|
+
pageHash: hashJson(page),
|
|
1298
|
+
bytes: Buffer.byteLength(serialized, 'utf8'),
|
|
1299
|
+
});
|
|
1300
|
+
}
|
|
1301
|
+
return await writeSegmentedCapture(directory, {
|
|
1302
|
+
threadId,
|
|
1303
|
+
sourceRevision,
|
|
1304
|
+
metadata,
|
|
1305
|
+
pages: staged,
|
|
1306
|
+
fs,
|
|
1307
|
+
});
|
|
1308
|
+
} catch (error) {
|
|
1309
|
+
for (const page of staged) await fs.unlink(page.stagedPath).catch(() => undefined);
|
|
1310
|
+
throw error;
|
|
179
1311
|
}
|
|
180
|
-
|
|
1312
|
+
}
|
|
1313
|
+
|
|
1314
|
+
// Test-only seam for deterministic capture-store fault injection. Production
|
|
1315
|
+
// code continues to use the immutable writer above with the real filesystem.
|
|
1316
|
+
export async function __testWriteBoundedCapture(
|
|
1317
|
+
file,
|
|
1318
|
+
value,
|
|
1319
|
+
{ fs = captureFs, now = Date.now, sleep } = {},
|
|
1320
|
+
) {
|
|
1321
|
+
const serialized = `${JSON.stringify(value)}\n`;
|
|
1322
|
+
if (Buffer.byteLength(serialized, 'utf8') > MAX_CAPTURE_BYTES)
|
|
1323
|
+
throw new Error('capture snapshot exceeds bound');
|
|
1324
|
+
validateCaptureSnapshot(value, value.sourceRevision);
|
|
1325
|
+
return writeBoundedCapture(file, serialized, value, fs, { now, sleep });
|
|
181
1326
|
}
|
|
182
1327
|
async function atomicWriteTextPrivate(file, text) {
|
|
183
1328
|
if (Buffer.byteLength(text) > MAX_NOTE_BYTES) throw new Error('recovery notes exceed bound');
|
|
@@ -219,17 +1364,156 @@ function validateThreadRead(value, threadId) {
|
|
|
219
1364
|
return value;
|
|
220
1365
|
}
|
|
221
1366
|
|
|
222
|
-
|
|
1367
|
+
function validateThreadMetadata(value, threadId) {
|
|
1368
|
+
if (!isRecord(value) || !isRecord(value.thread) || value.thread.id !== threadId)
|
|
1369
|
+
throw new Error('thread/read returned invalid thread metadata');
|
|
1370
|
+
return value;
|
|
1371
|
+
}
|
|
1372
|
+
|
|
1373
|
+
export function parseTurnsPage(value, threadId, pageIndex) {
|
|
1374
|
+
if (!isRecord(value)) throw new Error('thread/turns/list returned invalid page');
|
|
1375
|
+
const data = Array.isArray(value.data)
|
|
1376
|
+
? value.data
|
|
1377
|
+
: Array.isArray(value.turns)
|
|
1378
|
+
? value.turns
|
|
1379
|
+
: null;
|
|
1380
|
+
if (data === null || data.some((turn) => !isRecord(turn) || typeof turn.id !== 'string'))
|
|
1381
|
+
throw new Error('thread/turns/list returned invalid turns');
|
|
1382
|
+
const page = { threadId, turns: data };
|
|
1383
|
+
const serializedBytes = Buffer.byteLength(JSON.stringify(page), 'utf8');
|
|
1384
|
+
if (serializedBytes > MAX_CAPTURE_PAGE_BYTES) {
|
|
1385
|
+
const error = new Error('capture page exceeds bound');
|
|
1386
|
+
error.diagnosticReason = 'capture_size_exceeded';
|
|
1387
|
+
error.captureBytes = serializedBytes;
|
|
1388
|
+
error.captureLimitBytes = MAX_CAPTURE_PAGE_BYTES;
|
|
1389
|
+
throw error;
|
|
1390
|
+
}
|
|
1391
|
+
const cursor = value.nextCursor ?? value.next_cursor ?? null;
|
|
1392
|
+
if (cursor !== null && (typeof cursor !== 'string' || cursor.length > 512))
|
|
1393
|
+
throw new Error('thread/turns/list returned invalid cursor');
|
|
1394
|
+
return { page, pageIndex, cursor };
|
|
1395
|
+
}
|
|
1396
|
+
|
|
1397
|
+
function summarizeCaptureItem(item) {
|
|
1398
|
+
if (!isRecord(item)) return { type: 'unknown' };
|
|
1399
|
+
const summary = { type: typeof item.type === 'string' ? item.type : 'unknown' };
|
|
1400
|
+
if (typeof item.id === 'string') summary.id = item.id;
|
|
1401
|
+
if (summary.type === 'userMessage') {
|
|
1402
|
+
const parts = Array.isArray(item.content)
|
|
1403
|
+
? item.content.filter((part) => isRecord(part) && typeof part.text === 'string')
|
|
1404
|
+
: [];
|
|
1405
|
+
const contentChars = parts.reduce((total, part) => total + part.text.length, 0);
|
|
1406
|
+
summary.content = parts.slice(0, 64).map((part) => ({
|
|
1407
|
+
type: part.type,
|
|
1408
|
+
text: part.text.slice(0, MAX_NOTE_ITEM_CHARS),
|
|
1409
|
+
}));
|
|
1410
|
+
summary.contentChars = contentChars;
|
|
1411
|
+
summary.bodyComplete =
|
|
1412
|
+
parts.length <= 64 && parts.every((part) => part.text.length <= MAX_NOTE_ITEM_CHARS);
|
|
1413
|
+
} else if (summary.type === 'agentMessage' || summary.type === 'plan') {
|
|
1414
|
+
const text = typeof item.text === 'string' ? item.text : '';
|
|
1415
|
+
summary.text = text.slice(0, MAX_NOTE_ITEM_CHARS);
|
|
1416
|
+
summary.textChars = text.length;
|
|
1417
|
+
summary.bodyComplete = text.length <= MAX_NOTE_ITEM_CHARS;
|
|
1418
|
+
} else if (summary.type === 'commandExecution') {
|
|
1419
|
+
const command = typeof item.command === 'string' ? item.command : '';
|
|
1420
|
+
summary.command = command.slice(0, MAX_NOTE_ITEM_CHARS);
|
|
1421
|
+
summary.commandComplete = command.length <= MAX_NOTE_ITEM_CHARS;
|
|
1422
|
+
for (const key of ['status', 'exitCode']) {
|
|
1423
|
+
if (key in item && (typeof item[key] === 'string' || typeof item[key] === 'number'))
|
|
1424
|
+
summary[key] = item[key];
|
|
1425
|
+
}
|
|
1426
|
+
summary.aggregatedOutput = String(item.aggregatedOutput ?? item.output ?? '').slice(0, 1024);
|
|
1427
|
+
summary.outputChars = String(item.aggregatedOutput ?? item.output ?? '').length;
|
|
1428
|
+
}
|
|
1429
|
+
return summary;
|
|
1430
|
+
}
|
|
1431
|
+
|
|
1432
|
+
function summarizeCaptureTurn(turn) {
|
|
1433
|
+
return {
|
|
1434
|
+
...(typeof turn.id === 'string' ? { id: turn.id } : {}),
|
|
1435
|
+
...(typeof turn.status === 'string' ? { status: turn.status } : {}),
|
|
1436
|
+
items: turn.items.map(summarizeCaptureItem),
|
|
1437
|
+
};
|
|
1438
|
+
}
|
|
1439
|
+
|
|
1440
|
+
export function mergePaginatedTurns(metadata, pages, threadId) {
|
|
1441
|
+
validateThreadMetadata(metadata, threadId);
|
|
1442
|
+
if (!Array.isArray(pages) || pages.length < 1 || pages.length > MAX_CAPTURE_PAGES)
|
|
1443
|
+
throw new Error('capture page count exceeds bound');
|
|
1444
|
+
const turns = [];
|
|
1445
|
+
const ids = new Set();
|
|
1446
|
+
const itemIds = new Set();
|
|
1447
|
+
let itemCount = 0;
|
|
1448
|
+
for (const [pagePosition, entry] of pages.entries()) {
|
|
1449
|
+
const page = entry?.page;
|
|
1450
|
+
if (!isRecord(page) || page.threadId !== threadId || !Array.isArray(page.turns))
|
|
1451
|
+
throw new Error('capture page identity validation failed');
|
|
1452
|
+
if (entry.pageIndex !== pagePosition) throw new Error('capture pages are out of order');
|
|
1453
|
+
for (const turn of page.turns) {
|
|
1454
|
+
if (!isRecord(turn) || typeof turn.id !== 'string' || ids.has(turn.id))
|
|
1455
|
+
throw new Error('capture history contains duplicate turn');
|
|
1456
|
+
ids.add(turn.id);
|
|
1457
|
+
if (!Array.isArray(turn.items)) throw new Error('capture turn items are unavailable');
|
|
1458
|
+
for (const item of turn.items) {
|
|
1459
|
+
if (isRecord(item) && typeof item.id === 'string') {
|
|
1460
|
+
if (itemIds.has(item.id)) throw new Error('capture history contains duplicate item');
|
|
1461
|
+
itemIds.add(item.id);
|
|
1462
|
+
}
|
|
1463
|
+
}
|
|
1464
|
+
itemCount += turn.items.length;
|
|
1465
|
+
if (itemCount > MAX_CAPTURE_ITEMS) {
|
|
1466
|
+
const error = new Error('capture item count exceeds bound');
|
|
1467
|
+
error.captureItemCount = itemCount;
|
|
1468
|
+
error.captureItemLimit = MAX_CAPTURE_ITEMS;
|
|
1469
|
+
throw error;
|
|
1470
|
+
}
|
|
1471
|
+
turns.push(turn);
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1474
|
+
if (turns.length === 0) throw new Error('thread/turns/list returned no recoverable turns');
|
|
1475
|
+
return { thread: { ...metadata.thread, turns } };
|
|
1476
|
+
}
|
|
1477
|
+
|
|
1478
|
+
function captureSourceRevision(capture) {
|
|
1479
|
+
if (capture.pages === null) return hashJson(capture.read);
|
|
1480
|
+
return captureManifestRevision(
|
|
1481
|
+
capture.metadata.thread,
|
|
1482
|
+
capture.pages.map(({ pageHash }) => pageHash),
|
|
1483
|
+
);
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
export function buildRecoveryCandidates(read, revision, policy = null, options = {}) {
|
|
223
1487
|
const turns = read.thread.turns;
|
|
224
|
-
const rawItems = turns.flatMap((turn) =>
|
|
225
|
-
|
|
1488
|
+
const rawItems = turns.flatMap((turn, turnIndex) =>
|
|
1489
|
+
(Array.isArray(turn.items) ? turn.items : []).map((item, itemIndex) => ({
|
|
1490
|
+
item,
|
|
1491
|
+
turnIndex,
|
|
1492
|
+
itemIndex,
|
|
1493
|
+
})),
|
|
1494
|
+
);
|
|
1495
|
+
const maxItems = options.maxItems ?? MAX_RECOVERY_ITEMS;
|
|
1496
|
+
if (!Number.isSafeInteger(maxItems) || maxItems < 1 || maxItems > MAX_RECOVERY_ITEMS)
|
|
1497
|
+
throw new Error('recovery candidate bound is invalid');
|
|
1498
|
+
if (rawItems.length > maxItems && options.allowWindow !== true)
|
|
226
1499
|
throw new Error('recovery candidate count exceeds bound');
|
|
227
|
-
|
|
1500
|
+
if (rawItems.length > MAX_RECOVERY_SOURCE_INDEX) {
|
|
1501
|
+
const error = new Error('recovery source index exceeds bound');
|
|
1502
|
+
error.captureItemCount = rawItems.length;
|
|
1503
|
+
error.captureItemLimit = MAX_RECOVERY_SOURCE_INDEX;
|
|
1504
|
+
throw error;
|
|
1505
|
+
}
|
|
1506
|
+
let items = rawItems.map(({ item: rawItem, turnIndex, itemIndex }, index) => {
|
|
228
1507
|
const item = isRecord(rawItem) ? rawItem : {};
|
|
229
1508
|
const type = typeof item.type === 'string' ? item.type : 'unknown';
|
|
230
1509
|
const commandOutput =
|
|
231
1510
|
type === 'commandExecution' ? String(item.aggregatedOutput ?? item.output ?? '') : '';
|
|
1511
|
+
const sourceOutputChars =
|
|
1512
|
+
type === 'commandExecution' && isSafeCounter(item.outputChars)
|
|
1513
|
+
? item.outputChars
|
|
1514
|
+
: commandOutput.length;
|
|
232
1515
|
const command = typeof item.command === 'string' ? item.command : 'unknown';
|
|
1516
|
+
const bodyComplete = item.bodyComplete !== false;
|
|
233
1517
|
const status = typeof item.status === 'string' ? item.status : 'unknown';
|
|
234
1518
|
const exitCode = typeof item.exitCode === 'number' ? item.exitCode : null;
|
|
235
1519
|
const localText =
|
|
@@ -275,9 +1559,11 @@ export function buildRecoveryCandidates(read, revision) {
|
|
|
275
1559
|
) &&
|
|
276
1560
|
(commandName !== 'git' ||
|
|
277
1561
|
['status', 'diff', 'log', 'show', 'rev-parse'].includes(commandWords[1])));
|
|
278
|
-
const unknownCommand =
|
|
1562
|
+
const unknownCommand =
|
|
1563
|
+
type === 'commandExecution' && (!readOnlyCommand || item.commandComplete === false);
|
|
279
1564
|
const unknown = !['userMessage', 'agentMessage', 'plan', 'commandExecution'].includes(type);
|
|
280
|
-
const
|
|
1565
|
+
const recentCount = policy?.preserveRecentMessages ?? 6;
|
|
1566
|
+
const recent = index >= Math.max(0, rawItems.length - recentCount);
|
|
281
1567
|
const protectedItem =
|
|
282
1568
|
kind === 'user_message' ||
|
|
283
1569
|
kind === 'plan' ||
|
|
@@ -296,7 +1582,14 @@ export function buildRecoveryCandidates(read, revision) {
|
|
|
296
1582
|
if (unknownCommand) pinReasons.push('unknown');
|
|
297
1583
|
if (unknown) pinReasons.push('unknown');
|
|
298
1584
|
if (recent && pinReasons.length === 0) pinReasons.push('recent');
|
|
299
|
-
const localChars =
|
|
1585
|
+
const localChars =
|
|
1586
|
+
type === 'commandExecution'
|
|
1587
|
+
? sourceOutputChars
|
|
1588
|
+
: type === 'userMessage' && isSafeCounter(item.contentChars)
|
|
1589
|
+
? item.contentChars
|
|
1590
|
+
: (type === 'agentMessage' || type === 'plan') && isSafeCounter(item.textChars)
|
|
1591
|
+
? item.textChars
|
|
1592
|
+
: localText.length;
|
|
300
1593
|
return {
|
|
301
1594
|
id: `recovery-${index + 1}`,
|
|
302
1595
|
text: redact(localText, 16_384),
|
|
@@ -306,19 +1599,102 @@ export function buildRecoveryCandidates(read, revision) {
|
|
|
306
1599
|
pinReasons,
|
|
307
1600
|
source: rawItem,
|
|
308
1601
|
sourceItemId: typeof item.id === 'string' ? item.id : `item-${index + 1}`,
|
|
309
|
-
|
|
1602
|
+
sourceLocation: { flatIndex: index, turnIndex, itemIndex },
|
|
1603
|
+
sourceRefs: null,
|
|
1604
|
+
sourceBodyChars: sourceOutputChars,
|
|
1605
|
+
bodyComplete,
|
|
310
1606
|
kind,
|
|
311
1607
|
};
|
|
312
1608
|
});
|
|
1609
|
+
if (items.length > maxItems) {
|
|
1610
|
+
const protectedItems = items.filter((item) => item.protected);
|
|
1611
|
+
const anchorIds = new Set();
|
|
1612
|
+
const first = items[0];
|
|
1613
|
+
if (first?.protected) anchorIds.add(first.id);
|
|
1614
|
+
const currentGoal = items.findLast((item) => item.kind === 'user_message');
|
|
1615
|
+
if (currentGoal?.protected) anchorIds.add(currentGoal.id);
|
|
1616
|
+
const recentCount = policy?.preserveRecentMessages ?? 6;
|
|
1617
|
+
for (const item of items.slice(-recentCount)) {
|
|
1618
|
+
if (item.protected) anchorIds.add(item.id);
|
|
1619
|
+
}
|
|
1620
|
+
const individualProtected = protectedItems.filter((item) => anchorIds.has(item.id));
|
|
1621
|
+
const bundledProtected = protectedItems.filter((item) => !anchorIds.has(item.id));
|
|
1622
|
+
if (individualProtected.length >= maxItems)
|
|
1623
|
+
throw new Error('recovery protected anchors exceed bound');
|
|
1624
|
+
const selected = new Set(individualProtected.map((item) => item.id));
|
|
1625
|
+
if (bundledProtected.length > 0) {
|
|
1626
|
+
const bundle = {
|
|
1627
|
+
id: 'recovery-protected-bundle-1',
|
|
1628
|
+
text: `protected reference bundle: ${bundledProtected.length} items retained in the verified immutable capture`,
|
|
1629
|
+
localText: '',
|
|
1630
|
+
localChars: 0,
|
|
1631
|
+
protected: true,
|
|
1632
|
+
pinReasons: ['reference'],
|
|
1633
|
+
source: null,
|
|
1634
|
+
sourceItemId: null,
|
|
1635
|
+
sourceLocation: null,
|
|
1636
|
+
sourceRefs: bundledProtected.map((item) => ({
|
|
1637
|
+
...item.sourceLocation,
|
|
1638
|
+
sourceItemId: item.sourceItemId,
|
|
1639
|
+
})),
|
|
1640
|
+
sourceBodyChars: 0,
|
|
1641
|
+
kind: 'reference',
|
|
1642
|
+
};
|
|
1643
|
+
items = [...items, bundle];
|
|
1644
|
+
selected.add(bundle.id);
|
|
1645
|
+
}
|
|
1646
|
+
for (const item of items
|
|
1647
|
+
.filter((candidate) => !candidate.protected)
|
|
1648
|
+
.slice()
|
|
1649
|
+
.reverse()) {
|
|
1650
|
+
if (selected.size >= maxItems) break;
|
|
1651
|
+
selected.add(item.id);
|
|
1652
|
+
}
|
|
1653
|
+
items = items.filter((item) => selected.has(item.id));
|
|
1654
|
+
}
|
|
313
1655
|
const projection = {
|
|
314
|
-
goal:
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
1656
|
+
goal:
|
|
1657
|
+
policy?.goal !== undefined
|
|
1658
|
+
? redact(policy.goal, 4_000)
|
|
1659
|
+
: redact(
|
|
1660
|
+
items
|
|
1661
|
+
.filter((item) => item.kind === 'user_message')
|
|
1662
|
+
.slice(-3)
|
|
1663
|
+
.map((item) => item.text)
|
|
1664
|
+
.join('\n'),
|
|
1665
|
+
4_000,
|
|
1666
|
+
),
|
|
1667
|
+
items: items.map(({ id, kind, text, localChars, protected: protectedItem, pinReasons }) => ({
|
|
1668
|
+
id,
|
|
1669
|
+
kind,
|
|
1670
|
+
text,
|
|
1671
|
+
local_chars: localChars,
|
|
1672
|
+
protected: protectedItem,
|
|
1673
|
+
pin_reasons: pinReasons,
|
|
1674
|
+
})),
|
|
1675
|
+
};
|
|
1676
|
+
return { items, projection, projectionHash: hashJson(projection), revision };
|
|
1677
|
+
}
|
|
1678
|
+
|
|
1679
|
+
function selectionFittingProjection(candidates) {
|
|
1680
|
+
return {
|
|
1681
|
+
goal: candidates.projection.goal,
|
|
1682
|
+
items: candidates.items.map(
|
|
1683
|
+
({ id, kind, text, localChars, protected: protectedItem, pinReasons }) => ({
|
|
1684
|
+
id,
|
|
1685
|
+
kind,
|
|
1686
|
+
text,
|
|
1687
|
+
localChars,
|
|
1688
|
+
protected: protectedItem,
|
|
1689
|
+
pinReasons,
|
|
1690
|
+
}),
|
|
321
1691
|
),
|
|
1692
|
+
};
|
|
1693
|
+
}
|
|
1694
|
+
|
|
1695
|
+
function selectionWireProjection(goal, items) {
|
|
1696
|
+
return {
|
|
1697
|
+
goal,
|
|
322
1698
|
items: items.map(({ id, kind, text, localChars, protected: protectedItem, pinReasons }) => ({
|
|
323
1699
|
id,
|
|
324
1700
|
kind,
|
|
@@ -328,34 +1704,105 @@ export function buildRecoveryCandidates(read, revision) {
|
|
|
328
1704
|
pin_reasons: pinReasons,
|
|
329
1705
|
})),
|
|
330
1706
|
};
|
|
331
|
-
|
|
1707
|
+
}
|
|
1708
|
+
|
|
1709
|
+
function selectionFittingOptions(policy, maxRequestTokens) {
|
|
1710
|
+
return {
|
|
1711
|
+
preserveRecentItems: policy?.preserveRecentMessages ?? 6,
|
|
1712
|
+
maxStateTokens: policy?.maxStateTokens ?? 25_000,
|
|
1713
|
+
maxRequestTokens,
|
|
1714
|
+
};
|
|
1715
|
+
}
|
|
1716
|
+
|
|
1717
|
+
function fitWithinCapacity(fit, capacity) {
|
|
1718
|
+
if (!fit.ok) return false;
|
|
1719
|
+
const batchCount = fit.batches.length;
|
|
1720
|
+
return (
|
|
1721
|
+
batchCount <= capacity.requests &&
|
|
1722
|
+
batchCount * (fit.requestTokens + capacity.outputTokenAllowance) <= capacity.tokens
|
|
1723
|
+
);
|
|
1724
|
+
}
|
|
1725
|
+
|
|
1726
|
+
export function boundCandidatesToBudget(candidates, availableTokens, policy = null) {
|
|
1727
|
+
const capacity =
|
|
1728
|
+
typeof availableTokens === 'number'
|
|
1729
|
+
? {
|
|
1730
|
+
tokens: availableTokens,
|
|
1731
|
+
requests: Number.MAX_SAFE_INTEGER,
|
|
1732
|
+
maxRequestTokens: 30_000,
|
|
1733
|
+
outputTokenAllowance: 0,
|
|
1734
|
+
}
|
|
1735
|
+
: { outputTokenAllowance: 0, ...availableTokens };
|
|
1736
|
+
if (
|
|
1737
|
+
!isRecord(capacity) ||
|
|
1738
|
+
!isSafeBoundedInteger(capacity.tokens, 1, Number.MAX_SAFE_INTEGER) ||
|
|
1739
|
+
!isSafeBoundedInteger(capacity.requests, 1, Number.MAX_SAFE_INTEGER) ||
|
|
1740
|
+
!isSafeBoundedInteger(capacity.maxRequestTokens, 1_000, 30_000) ||
|
|
1741
|
+
!isSafeBoundedInteger(capacity.outputTokenAllowance, 0, Number.MAX_SAFE_INTEGER)
|
|
1742
|
+
)
|
|
1743
|
+
throw new Error('recovery budget has no usable token capacity');
|
|
1744
|
+
const protectedItems = candidates.items.filter((item) => item.protected);
|
|
1745
|
+
if (protectedItems.length > MAX_RECOVERY_ITEMS)
|
|
1746
|
+
throw new Error('recovery protected candidate count exceeds bound');
|
|
1747
|
+
const eligibleItems = candidates.items.filter((item) => !item.protected);
|
|
1748
|
+
const fitFor = (items) =>
|
|
1749
|
+
preflightSelectionState({
|
|
1750
|
+
projection: selectionFittingProjection({ ...candidates, items }),
|
|
1751
|
+
options: selectionFittingOptions(policy, capacity.maxRequestTokens),
|
|
1752
|
+
});
|
|
1753
|
+
for (let keepCount = eligibleItems.length; keepCount >= 0; keepCount -= 1) {
|
|
1754
|
+
const retainedEligible = new Set(
|
|
1755
|
+
(keepCount === 0 ? [] : eligibleItems.slice(-keepCount)).map((item) => item.id),
|
|
1756
|
+
);
|
|
1757
|
+
const items = candidates.items.filter(
|
|
1758
|
+
(item) => item.protected || retainedEligible.has(item.id),
|
|
1759
|
+
);
|
|
1760
|
+
const fit = fitFor(items);
|
|
1761
|
+
if (!fitWithinCapacity(fit, capacity)) continue;
|
|
1762
|
+
const projection = selectionFittingProjection({ ...candidates, items });
|
|
1763
|
+
const wireProjection = selectionWireProjection(candidates.projection.goal, items);
|
|
1764
|
+
return {
|
|
1765
|
+
items,
|
|
1766
|
+
projection: wireProjection,
|
|
1767
|
+
projectionHash: hashJson(wireProjection),
|
|
1768
|
+
revision: candidates.revision,
|
|
1769
|
+
bounded: items.length !== candidates.items.length,
|
|
1770
|
+
preflight: fit,
|
|
1771
|
+
providerProjection: projection,
|
|
1772
|
+
};
|
|
1773
|
+
}
|
|
1774
|
+
throw new Error('recovery protected candidate projection exceeds budget');
|
|
332
1775
|
}
|
|
333
1776
|
|
|
334
1777
|
export function parseSelection(value, input, candidates, currentTime = Date.now()) {
|
|
1778
|
+
const v2 = isRecord(input) && input.schema_version === 2;
|
|
1779
|
+
const baseKeys = [
|
|
1780
|
+
'schema_version',
|
|
1781
|
+
'status',
|
|
1782
|
+
'applied',
|
|
1783
|
+
'persisted',
|
|
1784
|
+
'request_id',
|
|
1785
|
+
'correlation_id',
|
|
1786
|
+
'session_ref',
|
|
1787
|
+
'revision',
|
|
1788
|
+
'projection_hash',
|
|
1789
|
+
'policy_version',
|
|
1790
|
+
'provider_model',
|
|
1791
|
+
'created_at',
|
|
1792
|
+
'expires_at',
|
|
1793
|
+
'decisions',
|
|
1794
|
+
'stats',
|
|
1795
|
+
];
|
|
1796
|
+
const expectedOptionsHash =
|
|
1797
|
+
v2 && validSelectionOptions(input.options) ? hashJson(input.options) : null;
|
|
335
1798
|
const expiresAt =
|
|
336
1799
|
isRecord(value) && isTimestamp(value.expires_at) ? Date.parse(value.expires_at) : Number.NaN;
|
|
337
1800
|
const createdAt =
|
|
338
1801
|
isRecord(value) && isTimestamp(value.created_at) ? Date.parse(value.created_at) : Number.NaN;
|
|
339
1802
|
if (
|
|
340
1803
|
!isRecord(value) ||
|
|
341
|
-
!hasOnlyKeys(value, [
|
|
342
|
-
|
|
343
|
-
'status',
|
|
344
|
-
'applied',
|
|
345
|
-
'persisted',
|
|
346
|
-
'request_id',
|
|
347
|
-
'correlation_id',
|
|
348
|
-
'session_ref',
|
|
349
|
-
'revision',
|
|
350
|
-
'projection_hash',
|
|
351
|
-
'policy_version',
|
|
352
|
-
'provider_model',
|
|
353
|
-
'created_at',
|
|
354
|
-
'expires_at',
|
|
355
|
-
'decisions',
|
|
356
|
-
'stats',
|
|
357
|
-
]) ||
|
|
358
|
-
value.schema_version !== 1 ||
|
|
1804
|
+
!hasOnlyKeys(value, v2 ? [...baseKeys, 'options_hash', 'fitting'] : baseKeys) ||
|
|
1805
|
+
value.schema_version !== (v2 ? 2 : 1) ||
|
|
359
1806
|
!['planned', 'noop'].includes(value.status) ||
|
|
360
1807
|
value.applied !== false ||
|
|
361
1808
|
value.persisted !== false ||
|
|
@@ -363,7 +1810,11 @@ export function parseSelection(value, input, candidates, currentTime = Date.now(
|
|
|
363
1810
|
value.session_ref !== input.session_ref ||
|
|
364
1811
|
value.revision !== input.revision ||
|
|
365
1812
|
value.projection_hash !== candidates.projectionHash ||
|
|
366
|
-
value.policy_version !== 'context-recovery-v1' ||
|
|
1813
|
+
value.policy_version !== (v2 ? 'context-recovery-v2' : 'context-recovery-v1') ||
|
|
1814
|
+
(v2 &&
|
|
1815
|
+
(!validSelectionOptions(input.options) ||
|
|
1816
|
+
typeof value.options_hash !== 'string' ||
|
|
1817
|
+
value.options_hash !== expectedOptionsHash)) ||
|
|
367
1818
|
typeof value.correlation_id !== 'string' ||
|
|
368
1819
|
value.correlation_id.length === 0 ||
|
|
369
1820
|
value.correlation_id.length > 128 ||
|
|
@@ -401,6 +1852,8 @@ export function parseSelection(value, input, candidates, currentTime = Date.now(
|
|
|
401
1852
|
!Array.isArray(value.decisions)
|
|
402
1853
|
)
|
|
403
1854
|
throw new Error('invalid recovery selection binding');
|
|
1855
|
+
if (v2) validateFitting(value.fitting, input.options.max_state_tokens);
|
|
1856
|
+
const keepThreshold = v2 ? input.options.keep_threshold : 0.2;
|
|
404
1857
|
const byId = new Map(candidates.items.map((item) => [item.id, item]));
|
|
405
1858
|
if (value.decisions.length !== candidates.items.length)
|
|
406
1859
|
throw new Error('incomplete recovery selection');
|
|
@@ -437,7 +1890,7 @@ export function parseSelection(value, input, candidates, currentTime = Date.now(
|
|
|
437
1890
|
(!candidate.protected && decision.reason === 'protected') ||
|
|
438
1891
|
(decision.reason === 'relevant' && decision.keep !== true) ||
|
|
439
1892
|
(decision.reason === 'below_threshold' && decision.keep !== false) ||
|
|
440
|
-
(!candidate.protected && decision.keep !== decision.probability >=
|
|
1893
|
+
(!candidate.protected && decision.keep !== decision.probability >= keepThreshold)
|
|
441
1894
|
)
|
|
442
1895
|
throw new Error('recovery decision semantics do not match the selection contract');
|
|
443
1896
|
decisions.push(decision);
|
|
@@ -464,6 +1917,44 @@ export function recoveryNotesText(
|
|
|
464
1917
|
candidates,
|
|
465
1918
|
decisions,
|
|
466
1919
|
) {
|
|
1920
|
+
const formatSourceLocations = (refs) => {
|
|
1921
|
+
const sorted = refs
|
|
1922
|
+
.filter(
|
|
1923
|
+
(ref) =>
|
|
1924
|
+
isRecord(ref) &&
|
|
1925
|
+
isSafeBoundedInteger(ref.flatIndex, 0, MAX_RECOVERY_SOURCE_INDEX) &&
|
|
1926
|
+
isSafeBoundedInteger(ref.turnIndex, 0, MAX_RECOVERY_SOURCE_INDEX) &&
|
|
1927
|
+
isSafeBoundedInteger(ref.itemIndex, 0, MAX_RECOVERY_SOURCE_INDEX),
|
|
1928
|
+
)
|
|
1929
|
+
.sort((left, right) => left.flatIndex - right.flatIndex);
|
|
1930
|
+
const unique = [];
|
|
1931
|
+
const seen = new Set();
|
|
1932
|
+
for (const ref of sorted) {
|
|
1933
|
+
const key = `${ref.flatIndex}:${ref.turnIndex}:${ref.itemIndex}:${String(ref.sourceItemId ?? '')}`;
|
|
1934
|
+
if (seen.has(key)) continue;
|
|
1935
|
+
seen.add(key);
|
|
1936
|
+
unique.push(ref);
|
|
1937
|
+
}
|
|
1938
|
+
const ranges = [];
|
|
1939
|
+
for (const ref of unique) {
|
|
1940
|
+
const previous = ranges.at(-1);
|
|
1941
|
+
if (previous && ref.flatIndex === previous.end + 1) previous.end = ref.flatIndex;
|
|
1942
|
+
else ranges.push({ start: ref.flatIndex, end: ref.flatIndex });
|
|
1943
|
+
}
|
|
1944
|
+
return {
|
|
1945
|
+
positions: ranges
|
|
1946
|
+
.map((range) =>
|
|
1947
|
+
range.start === range.end ? String(range.start) : `${range.start}-${range.end}`,
|
|
1948
|
+
)
|
|
1949
|
+
.join(','),
|
|
1950
|
+
locations: unique.map((ref) => `${ref.turnIndex}:${ref.itemIndex}`).join(','),
|
|
1951
|
+
ids: unique
|
|
1952
|
+
.map((ref) =>
|
|
1953
|
+
typeof ref.sourceItemId === 'string' ? escapeReference(ref.sourceItemId) : 'unknown',
|
|
1954
|
+
)
|
|
1955
|
+
.join(','),
|
|
1956
|
+
};
|
|
1957
|
+
};
|
|
467
1958
|
const selected = new Map(
|
|
468
1959
|
decisions.filter((decision) => decision.keep).map((decision) => [decision.id, decision]),
|
|
469
1960
|
);
|
|
@@ -477,6 +1968,7 @@ export function recoveryNotesText(
|
|
|
477
1968
|
`- capture_base: ${escapeReference(captureBase)}`,
|
|
478
1969
|
`- capture_file: ${escapeReference(captureFile)} (private checkpoint directory)`,
|
|
479
1970
|
`- capture_source_sha256: ${revision}`,
|
|
1971
|
+
'- capture_verified: immutable private capture read and revision checked before native compaction',
|
|
480
1972
|
`- source: private local capture`,
|
|
481
1973
|
'',
|
|
482
1974
|
'## Selected context',
|
|
@@ -485,10 +1977,16 @@ export function recoveryNotesText(
|
|
|
485
1977
|
let inlineBudget = Math.floor(MAX_NOTE_BYTES / 2);
|
|
486
1978
|
for (const item of candidates.items) {
|
|
487
1979
|
if (!selected.has(item.id)) continue;
|
|
488
|
-
const reference =
|
|
1980
|
+
const reference = item.sourceRefs
|
|
1981
|
+
? (() => {
|
|
1982
|
+
const source = formatSourceLocations(item.sourceRefs);
|
|
1983
|
+
return `[protected reference bundle retained in verified capture ${escapeReference(captureFile)}; source_positions=${source.positions}; thread_locations=${source.locations}; source_ids=${source.ids}]`;
|
|
1984
|
+
})()
|
|
1985
|
+
: `[selected local content retained in ${escapeReference(captureFile)}; source_item=${escapeReference(item.sourceItemId)}; source_location=${item.sourceLocation?.turnIndex ?? 'unknown'}:${item.sourceLocation?.itemIndex ?? 'unknown'}; local_chars=${item.localChars}]`;
|
|
489
1986
|
const inline =
|
|
490
1987
|
item.kind !== 'tool_result' &&
|
|
491
1988
|
item.kind !== 'reference' &&
|
|
1989
|
+
item.bodyComplete !== false &&
|
|
492
1990
|
item.localText.length > 0 &&
|
|
493
1991
|
item.localText.length <= MAX_NOTE_ITEM_CHARS &&
|
|
494
1992
|
Buffer.byteLength(item.localText, 'utf8') <= inlineBudget;
|
|
@@ -512,27 +2010,6 @@ export function recoveryNotesText(
|
|
|
512
2010
|
return `${lines.join('\n')}`;
|
|
513
2011
|
}
|
|
514
2012
|
|
|
515
|
-
function recoveryInjectionState(read, notes) {
|
|
516
|
-
const matches = read.thread.turns
|
|
517
|
-
.flatMap((turn) => (Array.isArray(turn.items) ? turn.items : []))
|
|
518
|
-
.filter(
|
|
519
|
-
(item) =>
|
|
520
|
-
isRecord(item) &&
|
|
521
|
-
((item.type === 'message' &&
|
|
522
|
-
item.role === 'user' &&
|
|
523
|
-
Array.isArray(item.content) &&
|
|
524
|
-
item.content.some(
|
|
525
|
-
(part) => isRecord(part) && part.type === 'input_text' && part.text === notes,
|
|
526
|
-
)) ||
|
|
527
|
-
(item.type === 'userMessage' &&
|
|
528
|
-
Array.isArray(item.content) &&
|
|
529
|
-
item.content.some(
|
|
530
|
-
(part) => isRecord(part) && part.type === 'text' && part.text === notes,
|
|
531
|
-
))),
|
|
532
|
-
);
|
|
533
|
-
return matches.length === 0 ? 'absent' : matches.length === 1 ? 'present' : 'ambiguous';
|
|
534
|
-
}
|
|
535
|
-
|
|
536
2013
|
/** Own a Codex app-server process and request native compaction after a fresh usage event. */
|
|
537
2014
|
export function createCodexContextProxy({
|
|
538
2015
|
child,
|
|
@@ -540,6 +2017,7 @@ export function createCodexContextProxy({
|
|
|
540
2017
|
stderr = process.stderr,
|
|
541
2018
|
now = () => Date.now(),
|
|
542
2019
|
threshold = DEFAULT_THRESHOLD,
|
|
2020
|
+
policy = undefined,
|
|
543
2021
|
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
544
2022
|
checkpointDir = process.env.CODEX_CONTEXT_CHECKPOINT_DIR ?? '.codex/context-checkpoints',
|
|
545
2023
|
recoveryDir = process.env.CODEX_CONTEXT_RECOVERY_DIR ?? '.notes/context',
|
|
@@ -551,17 +2029,104 @@ export function createCodexContextProxy({
|
|
|
551
2029
|
},
|
|
552
2030
|
recoveryEnabled = process.env.CODEX_CONTEXT_RECOVERY_ENABLED === '1',
|
|
553
2031
|
recoveryDisclosed = process.env.CODEX_CONTEXT_RECOVERY_DISCLOSED === '1',
|
|
2032
|
+
telemetryEnabled = process.env.CODEX_CONTEXT_TELEMETRY_ENABLED === '1',
|
|
2033
|
+
telemetryDisclosed = process.env.CODEX_CONTEXT_TELEMETRY_DISCLOSED === '1',
|
|
2034
|
+
telemetryTimeoutMs = TELEMETRY_TIMEOUT_MS,
|
|
2035
|
+
telemetrySetTimer = setTimeout,
|
|
2036
|
+
telemetryClearTimer = clearTimeout,
|
|
2037
|
+
budgetEnabled = process.env.CODEX_CONTEXT_BUDGET_ENABLED !== '0',
|
|
2038
|
+
createDiagnosticSink = null,
|
|
554
2039
|
brainServer = process.env.AI_BRAIN_MCP_SERVER ?? 'brain',
|
|
555
2040
|
clientVersion = process.env.CODEX_CLIENT_VERSION ?? '0.161.0',
|
|
556
2041
|
}) {
|
|
2042
|
+
const contextPolicy = normalizeContextPolicy(policy);
|
|
2043
|
+
const effectiveThreshold = contextPolicy?.compactAtPercent ?? threshold;
|
|
557
2044
|
const threads = new Map();
|
|
558
2045
|
const ownedRequests = new Map();
|
|
2046
|
+
const ownedThreadIds = new Set();
|
|
2047
|
+
const clientRequestBindings = new Map();
|
|
559
2048
|
const internalReads = new Map();
|
|
560
2049
|
const retiredRequests = new Set();
|
|
2050
|
+
const telemetryByThread = new Map();
|
|
2051
|
+
const diagnosticSinks = new Map();
|
|
561
2052
|
let stopped = false;
|
|
562
2053
|
function emitDiagnostic(event) {
|
|
563
2054
|
try {
|
|
564
|
-
|
|
2055
|
+
const phase = typeof event?.phase === 'string' ? event.phase : '';
|
|
2056
|
+
const stage =
|
|
2057
|
+
event?.stage ??
|
|
2058
|
+
(phase === 'selection_accepted' || phase === 'select' ? 'selection' : null) ??
|
|
2059
|
+
(phase === 'read' || phase === 'capture' ? 'capture' : null) ??
|
|
2060
|
+
(phase === 'compact' || phase === 'native' || phase === 'thread_failure'
|
|
2061
|
+
? 'native'
|
|
2062
|
+
: null) ??
|
|
2063
|
+
(phase === 'inject' || phase === 'recovery' ? 'recovery' : null) ??
|
|
2064
|
+
(phase === 'transport_end' || phase === 'transport_close' || phase === 'process_error'
|
|
2065
|
+
? 'transport'
|
|
2066
|
+
: null);
|
|
2067
|
+
const reason =
|
|
2068
|
+
event?.reason ??
|
|
2069
|
+
(DIAGNOSTIC_REASONS.has(event?.reasonCode) ? event.reasonCode : null) ??
|
|
2070
|
+
(phase === 'selection_accepted' ? 'completed' : null) ??
|
|
2071
|
+
(phase === 'transport_close' || phase === 'transport_end' ? 'disconnected' : null) ??
|
|
2072
|
+
(phase === 'thread_failure' ? 'protocol_error' : null) ??
|
|
2073
|
+
(event?.errorCode !== undefined ? 'protocol_error' : null);
|
|
2074
|
+
const normalized = normalizeCodexDiagnostic(
|
|
2075
|
+
{
|
|
2076
|
+
...event,
|
|
2077
|
+
stage,
|
|
2078
|
+
reason,
|
|
2079
|
+
operationId: event?.operationId ?? threads.get(event?.threadId)?.pending?.operationId,
|
|
2080
|
+
},
|
|
2081
|
+
{ threadId: event?.threadId },
|
|
2082
|
+
);
|
|
2083
|
+
if (normalized === null) return;
|
|
2084
|
+
if (
|
|
2085
|
+
typeof event?.schemaVersion === 'number' &&
|
|
2086
|
+
isSafeBoundedInteger(event.schemaVersion, 1, 2)
|
|
2087
|
+
)
|
|
2088
|
+
normalized.schemaVersion = event.schemaVersion;
|
|
2089
|
+
for (const [source, target] of [
|
|
2090
|
+
['optionsHash', 'options_hash'],
|
|
2091
|
+
['fittingStage', 'fitting_stage'],
|
|
2092
|
+
['batchCount', 'batch_count'],
|
|
2093
|
+
['providerModel', 'provider_model'],
|
|
2094
|
+
]) {
|
|
2095
|
+
if (
|
|
2096
|
+
(typeof event[source] === 'string' && event[source].length <= MAX_DIAGNOSTIC_STRING) ||
|
|
2097
|
+
(source === 'batchCount' && isSafeBoundedInteger(event[source], 0, 4)) ||
|
|
2098
|
+
(source === 'providerModel' && event[source] === null)
|
|
2099
|
+
)
|
|
2100
|
+
normalized[target] = event[source];
|
|
2101
|
+
if (source === 'optionsHash' && typeof event[source] === 'string')
|
|
2102
|
+
normalized.optionsHash = event[source];
|
|
2103
|
+
if (source === 'fittingStage' && typeof event[source] === 'string')
|
|
2104
|
+
normalized.fittingStage = event[source];
|
|
2105
|
+
if (source === 'batchCount' && isSafeBoundedInteger(event[source], 0, 4))
|
|
2106
|
+
normalized.batchCount = event[source];
|
|
2107
|
+
if (
|
|
2108
|
+
source === 'providerModel' &&
|
|
2109
|
+
(typeof event[source] === 'string' || event[source] === null)
|
|
2110
|
+
)
|
|
2111
|
+
normalized.providerModel = event[source];
|
|
2112
|
+
}
|
|
2113
|
+
onDiagnostic(normalized);
|
|
2114
|
+
if (typeof createDiagnosticSink === 'function' && typeof event?.threadId === 'string') {
|
|
2115
|
+
let sink = diagnosticSinks.get(event.threadId);
|
|
2116
|
+
if (!sink) {
|
|
2117
|
+
try {
|
|
2118
|
+
sink = createDiagnosticSink(event.threadId);
|
|
2119
|
+
if (typeof sink === 'function') diagnosticSinks.set(event.threadId, sink);
|
|
2120
|
+
} catch {
|
|
2121
|
+
sink = null;
|
|
2122
|
+
}
|
|
2123
|
+
}
|
|
2124
|
+
try {
|
|
2125
|
+
sink?.(normalized);
|
|
2126
|
+
} catch {
|
|
2127
|
+
/* private diagnostics cannot affect compaction */
|
|
2128
|
+
}
|
|
2129
|
+
}
|
|
565
2130
|
} catch {
|
|
566
2131
|
/* diagnostics must never affect compaction */
|
|
567
2132
|
}
|
|
@@ -589,9 +2154,10 @@ export function createCodexContextProxy({
|
|
|
589
2154
|
attachment.invalidated = true;
|
|
590
2155
|
cleanupRequest(attachment.readId);
|
|
591
2156
|
cleanupRequest(attachment.injectId);
|
|
2157
|
+
for (const id of attachment.auxIds ?? []) cleanupRequest(id);
|
|
592
2158
|
}
|
|
593
2159
|
|
|
594
|
-
function fail(threadId, reason) {
|
|
2160
|
+
function fail(threadId, reason, cause = null) {
|
|
595
2161
|
const state = threads.get(threadId);
|
|
596
2162
|
const pending = state?.pending;
|
|
597
2163
|
invalidateAttachment(state);
|
|
@@ -602,7 +2168,17 @@ export function createCodexContextProxy({
|
|
|
602
2168
|
cleanupRequest(pending.compactId);
|
|
603
2169
|
pending.deferred.reject(new Error(reason));
|
|
604
2170
|
state.pending = null;
|
|
605
|
-
|
|
2171
|
+
const operationId = pending.operationId;
|
|
2172
|
+
if (operationId) telemetryFor(threadId).failed({ operationId, reason: 'native_error' });
|
|
2173
|
+
emitDiagnostic({
|
|
2174
|
+
phase: 'thread_failure',
|
|
2175
|
+
stage: pending.stage ?? 'native',
|
|
2176
|
+
reason: diagnosticReasonForError(cause ?? new Error(reason)),
|
|
2177
|
+
threadId,
|
|
2178
|
+
operationId,
|
|
2179
|
+
...(boundedFailureCounts(cause) === undefined ? {} : { counts: boundedFailureCounts(cause) }),
|
|
2180
|
+
...(safeRpcCode(cause?.rpcCode) === null ? {} : { rpcCode: cause.rpcCode }),
|
|
2181
|
+
});
|
|
606
2182
|
stderr.write(`codex-context: native compaction preserved (${reason})\n`);
|
|
607
2183
|
}
|
|
608
2184
|
function failAll(reason) {
|
|
@@ -643,10 +2219,28 @@ export function createCodexContextProxy({
|
|
|
643
2219
|
recoveryLoaded: false,
|
|
644
2220
|
recoveryLoadPromise: null,
|
|
645
2221
|
recoveryAttach: null,
|
|
2222
|
+
telemetryConnected: false,
|
|
2223
|
+
owned: false,
|
|
2224
|
+
// Codex can publish an idle/usage snapshot before a resumed thread has
|
|
2225
|
+
// finished loading. Native MCP and compact requests sent in that gap
|
|
2226
|
+
// are rejected as invalid requests by the 0.163 app-server.
|
|
2227
|
+
resumePending: false,
|
|
2228
|
+
resumeRetryRequested: false,
|
|
2229
|
+
resumeRequests: new Set(),
|
|
646
2230
|
};
|
|
647
2231
|
threads.set(threadId, created);
|
|
648
2232
|
return created;
|
|
649
2233
|
}
|
|
2234
|
+
function claimOwnedThread(threadId) {
|
|
2235
|
+
const existing = threads.get(threadId);
|
|
2236
|
+
if (!existing && ownedThreadIds.size >= MAX_OWNED_THREAD_IDS) return null;
|
|
2237
|
+
const state = existing ?? threadState(threadId);
|
|
2238
|
+
if (state.owned) return state;
|
|
2239
|
+
if (ownedThreadIds.size >= MAX_OWNED_THREAD_IDS) return null;
|
|
2240
|
+
state.owned = true;
|
|
2241
|
+
ownedThreadIds.add(threadId);
|
|
2242
|
+
return state;
|
|
2243
|
+
}
|
|
650
2244
|
function sendOwned(method, params, kind, threadId, waiter) {
|
|
651
2245
|
if (ownedRequests.size >= MAX_OWNED_REQUESTS)
|
|
652
2246
|
throw new Error('too many pending adapter requests');
|
|
@@ -661,10 +2255,345 @@ export function createCodexContextProxy({
|
|
|
661
2255
|
}
|
|
662
2256
|
return id;
|
|
663
2257
|
}
|
|
2258
|
+
function telemetryFor(threadId) {
|
|
2259
|
+
const existing = telemetryByThread.get(threadId);
|
|
2260
|
+
if (existing) return existing;
|
|
2261
|
+
const reporter = createTelemetryReporter({
|
|
2262
|
+
client: 'codex-cli',
|
|
2263
|
+
clientVersion,
|
|
2264
|
+
sessionId: threadId,
|
|
2265
|
+
enabled: telemetryEnabled,
|
|
2266
|
+
disclosed: telemetryDisclosed,
|
|
2267
|
+
now,
|
|
2268
|
+
timeoutMs: telemetryTimeoutMs,
|
|
2269
|
+
setTimer: telemetrySetTimer,
|
|
2270
|
+
clearTimer: telemetryClearTimer,
|
|
2271
|
+
send: async (event) => {
|
|
2272
|
+
const waiter = deferred();
|
|
2273
|
+
void waiter.promise.catch(() => undefined);
|
|
2274
|
+
let id;
|
|
2275
|
+
try {
|
|
2276
|
+
id = sendOwned(
|
|
2277
|
+
'mcpServer/tool/call',
|
|
2278
|
+
{
|
|
2279
|
+
threadId,
|
|
2280
|
+
server: brainServer,
|
|
2281
|
+
tool: 'brain_context_report',
|
|
2282
|
+
arguments: event,
|
|
2283
|
+
},
|
|
2284
|
+
'telemetry',
|
|
2285
|
+
threadId,
|
|
2286
|
+
waiter,
|
|
2287
|
+
);
|
|
2288
|
+
let timer;
|
|
2289
|
+
try {
|
|
2290
|
+
const result = await Promise.race([
|
|
2291
|
+
waiter.promise,
|
|
2292
|
+
new Promise((_, reject) => {
|
|
2293
|
+
timer = telemetrySetTimer(
|
|
2294
|
+
() => reject(new Error('telemetry_timeout')),
|
|
2295
|
+
telemetryTimeoutMs,
|
|
2296
|
+
);
|
|
2297
|
+
}),
|
|
2298
|
+
]);
|
|
2299
|
+
if (!isRecord(result) || result.isError === true)
|
|
2300
|
+
throw new Error('telemetry_report_rejected');
|
|
2301
|
+
const structured = isRecord(result.structuredContent)
|
|
2302
|
+
? result.structuredContent
|
|
2303
|
+
: result;
|
|
2304
|
+
if (!isRecord(structured) || structured.accepted !== true)
|
|
2305
|
+
throw new Error('telemetry_report_rejected');
|
|
2306
|
+
return result;
|
|
2307
|
+
} finally {
|
|
2308
|
+
if (timer !== undefined) telemetryClearTimer(timer);
|
|
2309
|
+
}
|
|
2310
|
+
} finally {
|
|
2311
|
+
cleanupRequest(id);
|
|
2312
|
+
}
|
|
2313
|
+
},
|
|
2314
|
+
});
|
|
2315
|
+
telemetryByThread.set(threadId, reporter);
|
|
2316
|
+
return reporter;
|
|
2317
|
+
}
|
|
2318
|
+
async function requestBudget(threadId, pending) {
|
|
2319
|
+
const waiter = deferred();
|
|
2320
|
+
void waiter.promise.catch(() => undefined);
|
|
2321
|
+
const budgetRequestId = sendOwned(
|
|
2322
|
+
'mcpServer/tool/call',
|
|
2323
|
+
{
|
|
2324
|
+
threadId,
|
|
2325
|
+
server: brainServer,
|
|
2326
|
+
tool: 'brain_context_budget',
|
|
2327
|
+
arguments: { schema_version: 1 },
|
|
2328
|
+
},
|
|
2329
|
+
'budget',
|
|
2330
|
+
threadId,
|
|
2331
|
+
waiter,
|
|
2332
|
+
);
|
|
2333
|
+
pending.auxIds.push(budgetRequestId);
|
|
2334
|
+
emitDiagnostic({
|
|
2335
|
+
phase: 'budget',
|
|
2336
|
+
stage: 'budget',
|
|
2337
|
+
reason: 'requested',
|
|
2338
|
+
threadId,
|
|
2339
|
+
operationId: pending.operationId,
|
|
2340
|
+
});
|
|
2341
|
+
let response;
|
|
2342
|
+
try {
|
|
2343
|
+
response = await waiter.promise;
|
|
2344
|
+
} catch {
|
|
2345
|
+
emitDiagnostic({
|
|
2346
|
+
phase: 'budget',
|
|
2347
|
+
stage: 'budget',
|
|
2348
|
+
reason: 'unavailable',
|
|
2349
|
+
threadId,
|
|
2350
|
+
operationId: pending.operationId,
|
|
2351
|
+
});
|
|
2352
|
+
return { snapshot: null, error: null };
|
|
2353
|
+
}
|
|
2354
|
+
const error = parseContextError(response);
|
|
2355
|
+
if (error !== null && response?.isError === true) {
|
|
2356
|
+
emitDiagnostic({
|
|
2357
|
+
phase: 'budget',
|
|
2358
|
+
stage: 'budget',
|
|
2359
|
+
reason: error.quota ? 'admission_denied' : 'request_failed',
|
|
2360
|
+
threadId,
|
|
2361
|
+
operationId: pending.operationId,
|
|
2362
|
+
correlationId: error.correlationId,
|
|
2363
|
+
});
|
|
2364
|
+
return { snapshot: null, error };
|
|
2365
|
+
}
|
|
2366
|
+
const receivedAt = now();
|
|
2367
|
+
const snapshot = parseBudgetSnapshot(responseStructuredContent(response));
|
|
2368
|
+
if (snapshot === null) {
|
|
2369
|
+
emitDiagnostic({
|
|
2370
|
+
phase: 'budget',
|
|
2371
|
+
stage: 'budget',
|
|
2372
|
+
reason: 'stale_snapshot',
|
|
2373
|
+
threadId,
|
|
2374
|
+
operationId: pending.operationId,
|
|
2375
|
+
});
|
|
2376
|
+
return { snapshot: null, error: null };
|
|
2377
|
+
}
|
|
2378
|
+
emitDiagnostic({
|
|
2379
|
+
phase: 'budget',
|
|
2380
|
+
stage: 'budget',
|
|
2381
|
+
reason: 'completed',
|
|
2382
|
+
threadId,
|
|
2383
|
+
operationId: pending.operationId,
|
|
2384
|
+
correlationId: snapshot.correlation_id,
|
|
2385
|
+
counts: {
|
|
2386
|
+
available_requests: snapshot.available_requests,
|
|
2387
|
+
available_tokens: Math.min(snapshot.available_tokens, 1_000_000),
|
|
2388
|
+
},
|
|
2389
|
+
});
|
|
2390
|
+
return { snapshot, error: null, receivedAt };
|
|
2391
|
+
}
|
|
2392
|
+
function emitNativeFallback(threadId, pending, reason, correlationId = null) {
|
|
2393
|
+
telemetryFor(threadId).fallback({
|
|
2394
|
+
operationId: pending.operationId,
|
|
2395
|
+
reason:
|
|
2396
|
+
reason === 'admission_denied'
|
|
2397
|
+
? 'rate_limited'
|
|
2398
|
+
: reason === 'provider_unavailable'
|
|
2399
|
+
? 'provider_unavailable'
|
|
2400
|
+
: 'recovery_pending',
|
|
2401
|
+
});
|
|
2402
|
+
emitDiagnostic({
|
|
2403
|
+
phase: 'budget',
|
|
2404
|
+
stage: 'budget',
|
|
2405
|
+
reason: reason === 'provider_unavailable' ? 'request_failed' : reason,
|
|
2406
|
+
threadId,
|
|
2407
|
+
operationId: pending.operationId,
|
|
2408
|
+
correlationId,
|
|
2409
|
+
});
|
|
2410
|
+
}
|
|
2411
|
+
function emitOutcome(outcome) {
|
|
2412
|
+
try {
|
|
2413
|
+
onOutcome(outcome);
|
|
2414
|
+
} catch {
|
|
2415
|
+
/* lifecycle observers cannot affect compaction */
|
|
2416
|
+
}
|
|
2417
|
+
const reporter = telemetryFor(outcome.threadId);
|
|
2418
|
+
const operationId =
|
|
2419
|
+
outcome.operationId ??
|
|
2420
|
+
threads.get(outcome.threadId)?.recovery?.operationId ??
|
|
2421
|
+
threads.get(outcome.threadId)?.pending?.operationId;
|
|
2422
|
+
if (
|
|
2423
|
+
!outcome.telemetryAlreadyEmitted &&
|
|
2424
|
+
(outcome.status === 'native_completed' || outcome.status === 'completed') &&
|
|
2425
|
+
operationId
|
|
2426
|
+
)
|
|
2427
|
+
reporter.nativeCompleted({ operationId });
|
|
2428
|
+
else if (outcome.status === 'recovery_attached' && operationId)
|
|
2429
|
+
reporter.recoveryAttached({ operationId });
|
|
2430
|
+
else if (outcome.status === 'recovery_uncertain' && operationId)
|
|
2431
|
+
reporter.uncertain({ operationId, reason: 'recovery_uncertain' });
|
|
2432
|
+
else if (outcome.status === 'recovery_pending' && operationId)
|
|
2433
|
+
reporter.fallback({ operationId, reason: 'recovery_pending' });
|
|
2434
|
+
else if (outcome.status === 'recovery_abandoned' && operationId)
|
|
2435
|
+
reporter.fallback({ operationId, reason: 'stale_context' });
|
|
2436
|
+
else if (outcome.status === 'failed' && operationId)
|
|
2437
|
+
reporter.failed({ operationId, reason: 'native_error' });
|
|
2438
|
+
}
|
|
664
2439
|
function assertCurrent(threadId, state, pending, revision) {
|
|
665
2440
|
if (pending.canceled || state.pending !== pending || !state.idle || state.revision !== revision)
|
|
666
2441
|
throw new Error('Codex state changed during compaction');
|
|
667
2442
|
}
|
|
2443
|
+
async function readThreadCapture(
|
|
2444
|
+
threadId,
|
|
2445
|
+
pending,
|
|
2446
|
+
firstWaiter = null,
|
|
2447
|
+
{ onPage = null, readId: existingReadId = null, verifyOnly = false } = {},
|
|
2448
|
+
) {
|
|
2449
|
+
const waiter = firstWaiter ?? deferred();
|
|
2450
|
+
const readId =
|
|
2451
|
+
existingReadId ??
|
|
2452
|
+
sendOwned('thread/read', { threadId, includeTurns: false }, 'read', threadId, waiter);
|
|
2453
|
+
if (pending) {
|
|
2454
|
+
pending.readId = readId;
|
|
2455
|
+
pending.auxIds.push(readId);
|
|
2456
|
+
}
|
|
2457
|
+
const metadata = validateThreadMetadata(await waiter.promise, threadId);
|
|
2458
|
+
if (Array.isArray(metadata.thread.turns) && metadata.thread.turns.length > 0) {
|
|
2459
|
+
const legacy = validateThreadRead(metadata, threadId);
|
|
2460
|
+
await onPage?.(metadata.thread);
|
|
2461
|
+
return { read: legacy, metadata, pages: null };
|
|
2462
|
+
}
|
|
2463
|
+
const pages = [];
|
|
2464
|
+
const seenTurnIds = new Set();
|
|
2465
|
+
const seenItemIds = new Set();
|
|
2466
|
+
let totalItems = 0;
|
|
2467
|
+
if (!verifyOnly) await ensurePrivateDirectory(checkpointDir);
|
|
2468
|
+
const summaryTurns = [];
|
|
2469
|
+
const stagedFiles = [];
|
|
2470
|
+
let stagedBytes = 0;
|
|
2471
|
+
const cleanupStaged = async () => {
|
|
2472
|
+
for (const file of stagedFiles) await unlink(file).catch(() => undefined);
|
|
2473
|
+
};
|
|
2474
|
+
const seenCursors = new Set();
|
|
2475
|
+
let cursor = null;
|
|
2476
|
+
for (let pageIndex = 0; pageIndex < MAX_CAPTURE_PAGES; pageIndex += 1) {
|
|
2477
|
+
let pageResponse;
|
|
2478
|
+
let pageLimit = 16;
|
|
2479
|
+
for (let attempt = 0; attempt < 5; attempt += 1) {
|
|
2480
|
+
const pageWaiter = deferred();
|
|
2481
|
+
const params = {
|
|
2482
|
+
threadId,
|
|
2483
|
+
itemsView: 'full',
|
|
2484
|
+
sortDirection: 'asc',
|
|
2485
|
+
limit: pageLimit,
|
|
2486
|
+
};
|
|
2487
|
+
if (cursor !== null) params.cursor = cursor;
|
|
2488
|
+
const pageId = sendOwned('thread/turns/list', params, 'page', threadId, pageWaiter);
|
|
2489
|
+
if (pending) pending.auxIds.push(pageId);
|
|
2490
|
+
try {
|
|
2491
|
+
pageResponse = await pageWaiter.promise;
|
|
2492
|
+
break;
|
|
2493
|
+
} catch (error) {
|
|
2494
|
+
if (error?.diagnosticReason === 'capture_size_exceeded' && attempt < 4) {
|
|
2495
|
+
pageLimit = Math.max(1, Math.floor(pageLimit / 2));
|
|
2496
|
+
continue;
|
|
2497
|
+
}
|
|
2498
|
+
await cleanupStaged();
|
|
2499
|
+
if (error?.rpcCode === -32601 && pages.length === 0) {
|
|
2500
|
+
const legacyWaiter = deferred();
|
|
2501
|
+
const legacyId = sendOwned(
|
|
2502
|
+
'thread/read',
|
|
2503
|
+
{ threadId, includeTurns: true },
|
|
2504
|
+
'read',
|
|
2505
|
+
threadId,
|
|
2506
|
+
legacyWaiter,
|
|
2507
|
+
);
|
|
2508
|
+
if (pending) pending.auxIds.push(legacyId);
|
|
2509
|
+
const legacy = validateThreadRead(await legacyWaiter.promise, threadId);
|
|
2510
|
+
return { read: legacy, metadata, pages: null };
|
|
2511
|
+
}
|
|
2512
|
+
throw error;
|
|
2513
|
+
}
|
|
2514
|
+
}
|
|
2515
|
+
if (pageResponse === undefined) throw new Error('thread/turns/list page unavailable');
|
|
2516
|
+
try {
|
|
2517
|
+
const parsed = parseTurnsPage(pageResponse, threadId, pageIndex);
|
|
2518
|
+
if (parsed.page.turns.length === 0 && parsed.cursor !== null)
|
|
2519
|
+
throw new Error('thread/turns/list returned empty page with cursor');
|
|
2520
|
+
for (const turn of parsed.page.turns) {
|
|
2521
|
+
if (seenTurnIds.has(turn.id)) throw new Error('capture history contains duplicate turn');
|
|
2522
|
+
seenTurnIds.add(turn.id);
|
|
2523
|
+
if (!Array.isArray(turn.items)) throw new Error('capture turn items are unavailable');
|
|
2524
|
+
totalItems += turn.items.length;
|
|
2525
|
+
if (totalItems > MAX_CAPTURE_ITEMS) {
|
|
2526
|
+
const error = new Error('capture item count exceeds bound');
|
|
2527
|
+
error.captureItemCount = totalItems;
|
|
2528
|
+
error.captureItemLimit = MAX_CAPTURE_ITEMS;
|
|
2529
|
+
throw error;
|
|
2530
|
+
}
|
|
2531
|
+
for (const item of turn.items) {
|
|
2532
|
+
if (isRecord(item) && typeof item.id === 'string') {
|
|
2533
|
+
if (seenItemIds.has(item.id))
|
|
2534
|
+
throw new Error('capture history contains duplicate item');
|
|
2535
|
+
seenItemIds.add(item.id);
|
|
2536
|
+
}
|
|
2537
|
+
}
|
|
2538
|
+
}
|
|
2539
|
+
const serializedPage = `${JSON.stringify(parsed.page)}\n`;
|
|
2540
|
+
await onPage?.(parsed.page);
|
|
2541
|
+
const pageBytes = Buffer.byteLength(serializedPage, 'utf8');
|
|
2542
|
+
stagedBytes += pageBytes;
|
|
2543
|
+
if (stagedBytes > MAX_CAPTURE_TOTAL_BYTES)
|
|
2544
|
+
throw new Error('capture exceeds segmented bound');
|
|
2545
|
+
let stagedPath = null;
|
|
2546
|
+
if (!verifyOnly) {
|
|
2547
|
+
const storageState = await readCaptureDirectoryState(checkpointDir);
|
|
2548
|
+
if (
|
|
2549
|
+
storageState.totalBytes + pageBytes >
|
|
2550
|
+
MAX_CAPTURE_DIRECTORY_BYTES + MAX_CAPTURE_STAGING_HEADROOM_BYTES
|
|
2551
|
+
)
|
|
2552
|
+
throw captureCapacityError(storageState);
|
|
2553
|
+
stagedPath = resolve(
|
|
2554
|
+
checkpointDir,
|
|
2555
|
+
`.capture-stage.${safeThreadName(threadId)}.${randomUUID()}.${pageIndex}.tmp`,
|
|
2556
|
+
);
|
|
2557
|
+
await writeFile(stagedPath, serializedPage, { flag: 'wx', mode: 0o600 });
|
|
2558
|
+
stagedFiles.push(stagedPath);
|
|
2559
|
+
}
|
|
2560
|
+
pages.push({
|
|
2561
|
+
pageIndex,
|
|
2562
|
+
stagedPath,
|
|
2563
|
+
pageHash: hashJson(parsed.page),
|
|
2564
|
+
itemCount: parsed.page.turns.reduce(
|
|
2565
|
+
(count, turn) => count + (Array.isArray(turn.items) ? turn.items.length : 0),
|
|
2566
|
+
0,
|
|
2567
|
+
),
|
|
2568
|
+
bytes: pageBytes,
|
|
2569
|
+
cursor: parsed.cursor,
|
|
2570
|
+
});
|
|
2571
|
+
summaryTurns.push(...parsed.page.turns.map(summarizeCaptureTurn));
|
|
2572
|
+
if (parsed.cursor === null)
|
|
2573
|
+
return {
|
|
2574
|
+
read: {
|
|
2575
|
+
thread: {
|
|
2576
|
+
...metadata.thread,
|
|
2577
|
+
turns: summaryTurns,
|
|
2578
|
+
},
|
|
2579
|
+
},
|
|
2580
|
+
metadata,
|
|
2581
|
+
pages,
|
|
2582
|
+
stagedFiles,
|
|
2583
|
+
};
|
|
2584
|
+
if (seenCursors.has(parsed.cursor)) throw new Error('thread/turns/list cursor repeated');
|
|
2585
|
+
seenCursors.add(parsed.cursor);
|
|
2586
|
+
cursor = parsed.cursor;
|
|
2587
|
+
if (pageIndex === MAX_CAPTURE_PAGES - 1)
|
|
2588
|
+
throw new Error('thread/turns/list page count exceeds bound');
|
|
2589
|
+
} catch (error) {
|
|
2590
|
+
await cleanupStaged();
|
|
2591
|
+
throw error;
|
|
2592
|
+
}
|
|
2593
|
+
}
|
|
2594
|
+
await cleanupStaged();
|
|
2595
|
+
throw new Error('thread/turns/list page count exceeds bound');
|
|
2596
|
+
}
|
|
668
2597
|
async function loadRecoveryState(threadId, state) {
|
|
669
2598
|
if (state.recoveryLoaded) return state.recovery;
|
|
670
2599
|
if (!state.recoveryLoadPromise) {
|
|
@@ -702,14 +2631,22 @@ export function createCodexContextProxy({
|
|
|
702
2631
|
/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(
|
|
703
2632
|
value.requestId,
|
|
704
2633
|
) &&
|
|
2634
|
+
(value.operationId === undefined ||
|
|
2635
|
+
(typeof value.operationId === 'string' &&
|
|
2636
|
+
/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(
|
|
2637
|
+
value.operationId,
|
|
2638
|
+
))) &&
|
|
705
2639
|
typeof value.itemId === 'string' &&
|
|
706
2640
|
value.itemId.length <= 128 &&
|
|
707
2641
|
(!['pending_attachment', 'injecting', 'recovery_uncertain'].includes(value.status) ||
|
|
708
2642
|
typeof value.turnId === 'string')
|
|
709
2643
|
)
|
|
710
|
-
return
|
|
711
|
-
|
|
712
|
-
|
|
2644
|
+
return {
|
|
2645
|
+
...(value.status === 'injecting'
|
|
2646
|
+
? { ...value, status: 'recovery_uncertain' }
|
|
2647
|
+
: value),
|
|
2648
|
+
operationId: value.operationId ?? value.requestId,
|
|
2649
|
+
};
|
|
713
2650
|
} catch (error) {
|
|
714
2651
|
if (error?.code !== 'ENOENT') stderr.write('codex-context: recovery state unavailable\n');
|
|
715
2652
|
}
|
|
@@ -725,6 +2662,7 @@ export function createCodexContextProxy({
|
|
|
725
2662
|
await atomicWritePrivate(
|
|
726
2663
|
resolve(checkpointDir, `${safeThreadName(threadId)}.recovery.json`),
|
|
727
2664
|
record,
|
|
2665
|
+
{ captureLock: true },
|
|
728
2666
|
);
|
|
729
2667
|
}
|
|
730
2668
|
async function readRecoveryNotes(threadId, record) {
|
|
@@ -763,6 +2701,7 @@ export function createCodexContextProxy({
|
|
|
763
2701
|
injectSent: false,
|
|
764
2702
|
readId: null,
|
|
765
2703
|
injectId: null,
|
|
2704
|
+
auxIds: [],
|
|
766
2705
|
deferred: injectWaiter,
|
|
767
2706
|
timer: null,
|
|
768
2707
|
};
|
|
@@ -772,23 +2711,44 @@ export function createCodexContextProxy({
|
|
|
772
2711
|
attachment.canceled = true;
|
|
773
2712
|
cleanupRequest(attachment.readId);
|
|
774
2713
|
cleanupRequest(attachment.injectId);
|
|
2714
|
+
for (const id of attachment.auxIds) cleanupRequest(id);
|
|
775
2715
|
injectWaiter.reject(new Error('recovery attachment timed out'));
|
|
776
2716
|
}, timeoutMs);
|
|
777
2717
|
try {
|
|
778
2718
|
const notes = await readRecoveryNotes(threadId, record);
|
|
779
2719
|
if (attachment.invalidated || attachment.canceled)
|
|
780
2720
|
throw new Error('recovery attachment was invalidated before read');
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
2721
|
+
let latest = null;
|
|
2722
|
+
let injection = 'absent';
|
|
2723
|
+
const capture = await readThreadCapture(threadId, attachment, readWaiter, {
|
|
2724
|
+
verifyOnly: true,
|
|
2725
|
+
onPage: async (page) => {
|
|
2726
|
+
const pageLatest = page.turns?.at(-1);
|
|
2727
|
+
if (pageLatest) latest = pageLatest;
|
|
2728
|
+
const found = page.turns
|
|
2729
|
+
?.flatMap((turn) => (Array.isArray(turn.items) ? turn.items : []))
|
|
2730
|
+
.some(
|
|
2731
|
+
(item) =>
|
|
2732
|
+
isRecord(item) &&
|
|
2733
|
+
((item.type === 'message' &&
|
|
2734
|
+
item.role === 'user' &&
|
|
2735
|
+
Array.isArray(item.content) &&
|
|
2736
|
+
item.content.some(
|
|
2737
|
+
(part) => isRecord(part) && part.type === 'input_text' && part.text === notes,
|
|
2738
|
+
)) ||
|
|
2739
|
+
(item.type === 'userMessage' &&
|
|
2740
|
+
Array.isArray(item.content) &&
|
|
2741
|
+
item.content.some(
|
|
2742
|
+
(part) => isRecord(part) && part.type === 'text' && part.text === notes,
|
|
2743
|
+
))),
|
|
2744
|
+
);
|
|
2745
|
+
if (found) injection = injection === 'present' ? 'ambiguous' : 'present';
|
|
2746
|
+
},
|
|
2747
|
+
});
|
|
2748
|
+
for (const descriptor of capture.pages ?? [])
|
|
2749
|
+
if (descriptor.stagedPath) await unlink(descriptor.stagedPath).catch(() => undefined);
|
|
789
2750
|
if (attachment.invalidated || attachment.canceled)
|
|
790
2751
|
throw new Error('recovery attachment was invalidated during read');
|
|
791
|
-
const latest = read.thread.turns.at(-1);
|
|
792
2752
|
if (!latest || latest.id !== record.turnId || latest.status !== 'completed') {
|
|
793
2753
|
attachment.stale = true;
|
|
794
2754
|
throw new Error('recovery compaction turn is no longer current');
|
|
@@ -797,7 +2757,6 @@ export function createCodexContextProxy({
|
|
|
797
2757
|
attachment.stale = true;
|
|
798
2758
|
throw new Error('recovery attachment encountered a newer turn');
|
|
799
2759
|
}
|
|
800
|
-
const injection = recoveryInjectionState(read, notes);
|
|
801
2760
|
if (injection === 'ambiguous') {
|
|
802
2761
|
attachment.stale = true;
|
|
803
2762
|
throw new Error('recovery attachment has ambiguous prior injections');
|
|
@@ -813,7 +2772,14 @@ export function createCodexContextProxy({
|
|
|
813
2772
|
await persistRecoveryState(threadId, attached);
|
|
814
2773
|
if (attachment.invalidated || attachment.canceled) return;
|
|
815
2774
|
state.recovery = attached;
|
|
816
|
-
|
|
2775
|
+
emitDiagnostic({
|
|
2776
|
+
phase: 'recovery_attached',
|
|
2777
|
+
stage: 'recovery',
|
|
2778
|
+
reason: 'attached',
|
|
2779
|
+
threadId,
|
|
2780
|
+
operationId: record.operationId,
|
|
2781
|
+
});
|
|
2782
|
+
emitOutcome({ threadId, status: 'recovery_attached', turnId: record.turnId });
|
|
817
2783
|
return;
|
|
818
2784
|
}
|
|
819
2785
|
if (record.status === 'recovery_uncertain') {
|
|
@@ -824,7 +2790,14 @@ export function createCodexContextProxy({
|
|
|
824
2790
|
};
|
|
825
2791
|
await persistRecoveryState(threadId, uncertain).catch(() => undefined);
|
|
826
2792
|
state.recovery = uncertain;
|
|
827
|
-
|
|
2793
|
+
emitDiagnostic({
|
|
2794
|
+
phase: 'recovery_uncertain',
|
|
2795
|
+
stage: 'recovery',
|
|
2796
|
+
reason: 'uncertain',
|
|
2797
|
+
threadId,
|
|
2798
|
+
operationId: record.operationId,
|
|
2799
|
+
});
|
|
2800
|
+
emitOutcome({ threadId, status: 'recovery_uncertain', turnId: record.turnId });
|
|
828
2801
|
return;
|
|
829
2802
|
}
|
|
830
2803
|
if (!state.idle) throw new Error('thread became active before recovery attachment');
|
|
@@ -866,7 +2839,14 @@ export function createCodexContextProxy({
|
|
|
866
2839
|
};
|
|
867
2840
|
await persistRecoveryState(threadId, attached);
|
|
868
2841
|
state.recovery = attached;
|
|
869
|
-
|
|
2842
|
+
emitDiagnostic({
|
|
2843
|
+
phase: 'recovery_attached',
|
|
2844
|
+
stage: 'recovery',
|
|
2845
|
+
reason: 'attached',
|
|
2846
|
+
threadId,
|
|
2847
|
+
operationId: record.operationId,
|
|
2848
|
+
});
|
|
2849
|
+
emitOutcome({ threadId, status: 'recovery_attached', turnId: record.turnId });
|
|
870
2850
|
} catch {
|
|
871
2851
|
if (attachment.injectSent) {
|
|
872
2852
|
const uncertain = {
|
|
@@ -876,7 +2856,14 @@ export function createCodexContextProxy({
|
|
|
876
2856
|
};
|
|
877
2857
|
await persistRecoveryState(threadId, uncertain).catch(() => undefined);
|
|
878
2858
|
state.recovery = uncertain;
|
|
879
|
-
|
|
2859
|
+
emitDiagnostic({
|
|
2860
|
+
phase: 'recovery_uncertain',
|
|
2861
|
+
stage: 'recovery',
|
|
2862
|
+
reason: 'uncertain',
|
|
2863
|
+
threadId,
|
|
2864
|
+
operationId: record.operationId,
|
|
2865
|
+
});
|
|
2866
|
+
emitOutcome({ threadId, status: 'recovery_uncertain', turnId: record.turnId });
|
|
880
2867
|
} else if (!attachment.canceled) {
|
|
881
2868
|
if (attachment.stale) {
|
|
882
2869
|
state.recovery = {
|
|
@@ -885,18 +2872,24 @@ export function createCodexContextProxy({
|
|
|
885
2872
|
abandonedAt: new Date(now()).toISOString(),
|
|
886
2873
|
};
|
|
887
2874
|
await persistRecoveryState(threadId, state.recovery).catch(() => undefined);
|
|
888
|
-
|
|
889
|
-
} else
|
|
2875
|
+
emitOutcome({ threadId, status: 'recovery_abandoned', turnId: record.turnId });
|
|
2876
|
+
} else emitOutcome({ threadId, status: 'recovery_pending', turnId: record.turnId });
|
|
890
2877
|
}
|
|
891
2878
|
} finally {
|
|
892
2879
|
if (attachment.timer) attachment.timer();
|
|
893
2880
|
cleanupRequest(attachment.readId);
|
|
894
2881
|
cleanupRequest(attachment.injectId);
|
|
2882
|
+
for (const id of attachment.auxIds) cleanupRequest(id);
|
|
895
2883
|
if (state.recoveryAttach === attachment) state.recoveryAttach = null;
|
|
896
2884
|
}
|
|
897
2885
|
}
|
|
898
2886
|
async function requestCompaction(threadId, state) {
|
|
899
|
-
|
|
2887
|
+
if (!state.owned) return;
|
|
2888
|
+
if (state.resumePending) {
|
|
2889
|
+
state.resumeRetryRequested = true;
|
|
2890
|
+
return;
|
|
2891
|
+
}
|
|
2892
|
+
if (recoveryEnabled && recoveryDisclosed) await loadRecoveryState(threadId, state);
|
|
900
2893
|
if (
|
|
901
2894
|
state.recovery &&
|
|
902
2895
|
['pending_attachment', 'recovery_uncertain'].includes(state.recovery.status)
|
|
@@ -911,11 +2904,12 @@ export function createCodexContextProxy({
|
|
|
911
2904
|
state.pending ||
|
|
912
2905
|
!state.usage ||
|
|
913
2906
|
!isValidUsage(state.usage) ||
|
|
2907
|
+
!hasKnownContextWindow(state.usage) ||
|
|
914
2908
|
!Number.isFinite(state.usageReceivedAt) ||
|
|
915
2909
|
currentTime < state.usageReceivedAt ||
|
|
916
2910
|
currentTime - state.usageReceivedAt > USAGE_FRESHNESS_MS ||
|
|
917
2911
|
state.revision === null ||
|
|
918
|
-
(state.usage.last.totalTokens / state.usage.modelContextWindow) * 100 <
|
|
2912
|
+
(state.usage.last.totalTokens / state.usage.modelContextWindow) * 100 < effectiveThreshold
|
|
919
2913
|
)
|
|
920
2914
|
return;
|
|
921
2915
|
if (
|
|
@@ -943,52 +2937,126 @@ export function createCodexContextProxy({
|
|
|
943
2937
|
itemCompleted: false,
|
|
944
2938
|
turnCompleted: false,
|
|
945
2939
|
canceled: false,
|
|
2940
|
+
stage: 'capture',
|
|
2941
|
+
operationId: randomUUID(),
|
|
946
2942
|
deferred: completion,
|
|
947
2943
|
timer: null,
|
|
948
2944
|
};
|
|
949
2945
|
state.pending = pending;
|
|
950
2946
|
pending.timer = scheduleTimeout(() => fail(threadId, 'native compaction timed out'), timeoutMs);
|
|
951
2947
|
try {
|
|
2948
|
+
pending.stage = 'capture';
|
|
952
2949
|
pending.readId = sendOwned(
|
|
953
2950
|
'thread/read',
|
|
954
|
-
{ threadId, includeTurns:
|
|
2951
|
+
{ threadId, includeTurns: false },
|
|
955
2952
|
'read',
|
|
956
2953
|
threadId,
|
|
957
2954
|
readWaiter,
|
|
958
2955
|
);
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
2956
|
+
await ensurePrivateDirectory(checkpointDir);
|
|
2957
|
+
const publication = await withCaptureStoreLock(checkpointDir, async () => {
|
|
2958
|
+
const capture = await readThreadCapture(threadId, pending, readWaiter, {
|
|
2959
|
+
readId: pending.readId,
|
|
2960
|
+
});
|
|
2961
|
+
const read = capture.read;
|
|
2962
|
+
assertCurrent(threadId, state, pending, revision);
|
|
2963
|
+
const sourceRevision = captureSourceRevision(capture);
|
|
2964
|
+
const compactSnapshot = {
|
|
2965
|
+
version: 1,
|
|
2966
|
+
status: 'requested',
|
|
2967
|
+
capturedAt: new Date(now()).toISOString(),
|
|
2968
|
+
threadId,
|
|
2969
|
+
sourceTurnId,
|
|
2970
|
+
revision,
|
|
2971
|
+
sourceRevision,
|
|
2972
|
+
statusAtCheckpoint: state.status,
|
|
2973
|
+
usage: state.usage,
|
|
2974
|
+
threadRead: read,
|
|
2975
|
+
recovery: {
|
|
2976
|
+
mode: 'codex-native-compaction',
|
|
2977
|
+
action: 'resume-the-owned-app-server-session',
|
|
2978
|
+
historyMutation: false,
|
|
2979
|
+
},
|
|
2980
|
+
};
|
|
2981
|
+
let snapshot = compactSnapshot;
|
|
2982
|
+
if (capture.pages !== null) {
|
|
2983
|
+
const manifest = await writeSegmentedCapture(checkpointDir, {
|
|
2984
|
+
threadId,
|
|
2985
|
+
sourceRevision,
|
|
2986
|
+
metadata: capture.metadata,
|
|
2987
|
+
pages: capture.pages,
|
|
2988
|
+
lockHeld: true,
|
|
2989
|
+
});
|
|
2990
|
+
snapshot = {
|
|
2991
|
+
...compactSnapshot,
|
|
2992
|
+
version: 2,
|
|
2993
|
+
threadRead: capture.metadata,
|
|
2994
|
+
captureManifest: manifest,
|
|
2995
|
+
};
|
|
2996
|
+
} else if (Buffer.byteLength(JSON.stringify(compactSnapshot), 'utf8') > MAX_CAPTURE_BYTES) {
|
|
2997
|
+
throw new Error('capture snapshot exceeds bound');
|
|
2998
|
+
}
|
|
2999
|
+
const manifestForRollback = snapshot.version === 2 ? snapshot.captureManifest : null;
|
|
3000
|
+
const immutableCapturePath = resolve(
|
|
3001
|
+
checkpointDir,
|
|
3002
|
+
`${safeThreadName(threadId)}.${sourceRevision}.json`,
|
|
3003
|
+
);
|
|
3004
|
+
try {
|
|
3005
|
+
await atomicWriteImmutablePrivate(immutableCapturePath, snapshot, sourceRevision, true);
|
|
3006
|
+
} catch (error) {
|
|
3007
|
+
if (manifestForRollback?.segments) {
|
|
3008
|
+
for (const segment of manifestForRollback.segments)
|
|
3009
|
+
await unlink(resolve(checkpointDir, segment)).catch(() => undefined);
|
|
3010
|
+
}
|
|
3011
|
+
throw error;
|
|
3012
|
+
}
|
|
3013
|
+
await atomicWritePrivate(
|
|
3014
|
+
resolve(checkpointDir, `${safeThreadName(threadId)}.json`),
|
|
3015
|
+
snapshot,
|
|
3016
|
+
{ maxBytes: MAX_CAPTURE_BYTES, compact: true, capture: true, captureLockHeld: true },
|
|
3017
|
+
);
|
|
3018
|
+
return { capture, read, sourceRevision, snapshot, immutableCapturePath };
|
|
3019
|
+
});
|
|
3020
|
+
const { capture, read, sourceRevision, snapshot, immutableCapturePath } = publication;
|
|
3021
|
+
if (capture.pages !== null && recoveryEnabled && recoveryDisclosed) {
|
|
3022
|
+
const verificationWaiter = deferred();
|
|
3023
|
+
const verificationCapture = await readThreadCapture(threadId, pending, verificationWaiter, {
|
|
3024
|
+
verifyOnly: true,
|
|
3025
|
+
});
|
|
3026
|
+
try {
|
|
3027
|
+
if (captureSourceRevision(verificationCapture) !== sourceRevision)
|
|
3028
|
+
throw new Error('recovery source changed before provider admission');
|
|
3029
|
+
} finally {
|
|
3030
|
+
for (const descriptor of verificationCapture.pages ?? [])
|
|
3031
|
+
if (descriptor.stagedPath) await unlink(descriptor.stagedPath).catch(() => undefined);
|
|
3032
|
+
}
|
|
3033
|
+
}
|
|
3034
|
+
emitDiagnostic({
|
|
3035
|
+
phase: 'capture',
|
|
3036
|
+
stage: 'capture',
|
|
3037
|
+
reason: 'completed',
|
|
966
3038
|
threadId,
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
3039
|
+
operationId: pending.operationId,
|
|
3040
|
+
counts: {
|
|
3041
|
+
item_count: read.thread.turns.reduce(
|
|
3042
|
+
(count, turn) => count + (turn.items?.length ?? 0),
|
|
3043
|
+
0,
|
|
3044
|
+
),
|
|
3045
|
+
...(capture.pages === null
|
|
3046
|
+
? {}
|
|
3047
|
+
: {
|
|
3048
|
+
page_count: capture.pages.length,
|
|
3049
|
+
capture_bytes: capture.pages.reduce(
|
|
3050
|
+
(count, descriptor) => count + descriptor.bytes,
|
|
3051
|
+
0,
|
|
3052
|
+
),
|
|
3053
|
+
}),
|
|
977
3054
|
},
|
|
978
|
-
};
|
|
979
|
-
if (Buffer.byteLength(JSON.stringify(snapshot)) > MAX_CAPTURE_BYTES)
|
|
980
|
-
throw new Error('thread snapshot exceeds bound');
|
|
981
|
-
await atomicWritePrivate(
|
|
982
|
-
resolve(checkpointDir, `${safeThreadName(threadId)}.json`),
|
|
983
|
-
snapshot,
|
|
984
|
-
);
|
|
985
|
-
const immutableCapturePath = resolve(
|
|
986
|
-
checkpointDir,
|
|
987
|
-
`${safeThreadName(threadId)}.${sourceRevision}.json`,
|
|
988
|
-
);
|
|
989
|
-
await atomicWriteImmutablePrivate(immutableCapturePath, snapshot, sourceRevision);
|
|
3055
|
+
});
|
|
990
3056
|
assertCurrent(threadId, state, pending, revision);
|
|
991
3057
|
let recoveryNotes;
|
|
3058
|
+
let nativeFallback = false;
|
|
3059
|
+
let budgetReceipt = null;
|
|
992
3060
|
if (recoveryEnabled && recoveryDisclosed) {
|
|
993
3061
|
strictThreadName(threadId);
|
|
994
3062
|
if (
|
|
@@ -1006,90 +3074,317 @@ export function createCodexContextProxy({
|
|
|
1006
3074
|
await persistRecoveryState(threadId, state.recovery).catch(() => undefined);
|
|
1007
3075
|
throw new Error('saved recovery selection is stale');
|
|
1008
3076
|
} else {
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
clientVersion.length < 1 ||
|
|
1024
|
-
clientVersion.length > 64 ||
|
|
1025
|
-
Buffer.byteLength(JSON.stringify(selectionInput), 'utf8') > MAX_SELECTION_BYTES
|
|
1026
|
-
)
|
|
1027
|
-
throw new Error('recovery selection input exceeds the contract bound');
|
|
1028
|
-
const selectionWaiter = deferred();
|
|
1029
|
-
pending.auxIds.push(
|
|
1030
|
-
sendOwned(
|
|
1031
|
-
'mcpServer/tool/call',
|
|
1032
|
-
{
|
|
3077
|
+
let candidates;
|
|
3078
|
+
if (budgetEnabled) {
|
|
3079
|
+
pending.stage = 'budget';
|
|
3080
|
+
const budget = await requestBudget(threadId, pending);
|
|
3081
|
+
budgetReceipt = budget;
|
|
3082
|
+
if (budget.error?.quota === true) {
|
|
3083
|
+
nativeFallback = true;
|
|
3084
|
+
emitNativeFallback(threadId, pending, 'admission_denied', budget.error.correlationId);
|
|
3085
|
+
} else if (budget.snapshot === null) {
|
|
3086
|
+
nativeFallback = true;
|
|
3087
|
+
emitNativeFallback(threadId, pending, 'unavailable', null);
|
|
3088
|
+
} else if (!budgetSnapshotFresh(budget.snapshot, budget.receivedAt, now())) {
|
|
3089
|
+
nativeFallback = true;
|
|
3090
|
+
emitNativeFallback(
|
|
1033
3091
|
threadId,
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
3092
|
+
pending,
|
|
3093
|
+
'stale_snapshot',
|
|
3094
|
+
budget.snapshot.correlation_id,
|
|
3095
|
+
);
|
|
3096
|
+
} else if (
|
|
3097
|
+
budget.snapshot.account_in_flight === true ||
|
|
3098
|
+
budget.snapshot.available_requests < 1 ||
|
|
3099
|
+
budget.snapshot.available_tokens < 1 ||
|
|
3100
|
+
budget.snapshot.max_batches_per_operation < 1
|
|
3101
|
+
) {
|
|
3102
|
+
nativeFallback = true;
|
|
3103
|
+
emitNativeFallback(
|
|
3104
|
+
threadId,
|
|
3105
|
+
pending,
|
|
3106
|
+
'admission_denied',
|
|
3107
|
+
budget.snapshot.correlation_id,
|
|
3108
|
+
);
|
|
3109
|
+
} else {
|
|
3110
|
+
try {
|
|
3111
|
+
candidates = buildRecoveryCandidates(read, sourceRevision, contextPolicy, {
|
|
3112
|
+
allowWindow: true,
|
|
3113
|
+
});
|
|
3114
|
+
const bounded = boundCandidatesToBudget(
|
|
3115
|
+
candidates,
|
|
3116
|
+
{
|
|
3117
|
+
tokens: budget.snapshot.available_tokens,
|
|
3118
|
+
requests: Math.min(
|
|
3119
|
+
budget.snapshot.available_requests,
|
|
3120
|
+
budget.snapshot.max_batches_per_operation,
|
|
3121
|
+
),
|
|
3122
|
+
maxRequestTokens: contextPolicy?.maxRequestTokens ?? 30_000,
|
|
3123
|
+
outputTokenAllowance: budget.snapshot.output_token_allowance,
|
|
3124
|
+
},
|
|
3125
|
+
contextPolicy,
|
|
3126
|
+
);
|
|
3127
|
+
if (bounded.bounded) {
|
|
3128
|
+
emitDiagnostic({
|
|
3129
|
+
phase: 'candidate_window_bounded',
|
|
3130
|
+
stage: 'selection',
|
|
3131
|
+
reason: 'candidate_window_bounded',
|
|
3132
|
+
threadId,
|
|
3133
|
+
operationId: pending.operationId,
|
|
3134
|
+
correlationId: budget.snapshot.correlation_id,
|
|
3135
|
+
counts: {
|
|
3136
|
+
candidate_count: bounded.items.length,
|
|
3137
|
+
deferred_count: Math.max(0, candidates.items.length - bounded.items.length),
|
|
3138
|
+
provider_batch_count: bounded.preflight.batches.length,
|
|
3139
|
+
provider_request_tokens: bounded.preflight.requestTokens,
|
|
3140
|
+
},
|
|
3141
|
+
});
|
|
3142
|
+
}
|
|
3143
|
+
candidates = bounded;
|
|
3144
|
+
} catch (error) {
|
|
3145
|
+
nativeFallback = true;
|
|
3146
|
+
emitNativeFallback(
|
|
3147
|
+
threadId,
|
|
3148
|
+
pending,
|
|
3149
|
+
error?.message?.includes('protected')
|
|
3150
|
+
? 'protected_over_bound'
|
|
3151
|
+
: 'candidate_over_bound',
|
|
3152
|
+
budget.snapshot.correlation_id,
|
|
3153
|
+
);
|
|
3154
|
+
}
|
|
3155
|
+
}
|
|
3156
|
+
} else {
|
|
3157
|
+
pending.stage = 'selection';
|
|
3158
|
+
try {
|
|
3159
|
+
candidates = buildRecoveryCandidates(read, sourceRevision, contextPolicy);
|
|
3160
|
+
} catch {
|
|
3161
|
+
nativeFallback = true;
|
|
3162
|
+
emitNativeFallback(threadId, pending, 'candidate_over_bound');
|
|
3163
|
+
}
|
|
3164
|
+
}
|
|
3165
|
+
if (
|
|
3166
|
+
!nativeFallback &&
|
|
3167
|
+
budgetReceipt?.snapshot !== null &&
|
|
3168
|
+
budgetReceipt?.snapshot !== undefined &&
|
|
3169
|
+
!budgetSnapshotFresh(budgetReceipt.snapshot, budgetReceipt.receivedAt, now())
|
|
3170
|
+
) {
|
|
3171
|
+
nativeFallback = true;
|
|
3172
|
+
emitNativeFallback(
|
|
1039
3173
|
threadId,
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
3174
|
+
pending,
|
|
3175
|
+
'stale_snapshot',
|
|
3176
|
+
budgetReceipt.snapshot.correlation_id,
|
|
3177
|
+
);
|
|
3178
|
+
}
|
|
3179
|
+
if (nativeFallback) candidates = null;
|
|
3180
|
+
if (candidates === null || candidates === undefined) {
|
|
3181
|
+
emitDiagnostic({
|
|
3182
|
+
phase: 'selection',
|
|
3183
|
+
stage: 'selection',
|
|
3184
|
+
reason: 'unavailable',
|
|
3185
|
+
threadId,
|
|
3186
|
+
operationId: pending.operationId,
|
|
3187
|
+
});
|
|
3188
|
+
} else {
|
|
3189
|
+
pending.stage = 'selection';
|
|
3190
|
+
const selectionInput = contextPolicy
|
|
3191
|
+
? {
|
|
3192
|
+
schema_version: 2,
|
|
3193
|
+
client: 'codex_app_server',
|
|
3194
|
+
client_version: clientVersion,
|
|
3195
|
+
session_ref: threadId,
|
|
3196
|
+
revision: sourceRevision,
|
|
3197
|
+
observed_at: new Date(now()).toISOString(),
|
|
3198
|
+
request_id: pending.operationId,
|
|
3199
|
+
policy_version: 'context-recovery-v2',
|
|
3200
|
+
options: selectionOptions(contextPolicy),
|
|
3201
|
+
projection: candidates.projection,
|
|
3202
|
+
}
|
|
3203
|
+
: {
|
|
3204
|
+
schema_version: 1,
|
|
3205
|
+
client: 'codex_app_server',
|
|
3206
|
+
client_version: clientVersion,
|
|
3207
|
+
session_ref: threadId,
|
|
3208
|
+
revision: sourceRevision,
|
|
3209
|
+
observed_at: new Date(now()).toISOString(),
|
|
3210
|
+
request_id: pending.operationId,
|
|
3211
|
+
policy_version: 'context-recovery-v1',
|
|
3212
|
+
projection: candidates.projection,
|
|
3213
|
+
};
|
|
3214
|
+
if (
|
|
3215
|
+
typeof clientVersion !== 'string' ||
|
|
3216
|
+
clientVersion.length < 1 ||
|
|
3217
|
+
clientVersion.length > 64 ||
|
|
3218
|
+
Buffer.byteLength(JSON.stringify(selectionInput), 'utf8') > MAX_SELECTION_BYTES
|
|
3219
|
+
)
|
|
3220
|
+
throw new Error('recovery selection input exceeds the contract bound');
|
|
3221
|
+
const selectionWaiter = deferred();
|
|
3222
|
+
pending.auxIds.push(
|
|
3223
|
+
sendOwned(
|
|
3224
|
+
'mcpServer/tool/call',
|
|
3225
|
+
{
|
|
3226
|
+
threadId,
|
|
3227
|
+
server: brainServer,
|
|
3228
|
+
tool: 'brain_context_select',
|
|
3229
|
+
arguments: selectionInput,
|
|
3230
|
+
},
|
|
3231
|
+
'select',
|
|
3232
|
+
threadId,
|
|
3233
|
+
selectionWaiter,
|
|
3234
|
+
),
|
|
3235
|
+
);
|
|
3236
|
+
const selectionResponse = await selectionWaiter.promise;
|
|
3237
|
+
const selectionError = parseContextError(selectionResponse);
|
|
3238
|
+
if (!isRecord(selectionResponse) || selectionResponse.isError === true) {
|
|
3239
|
+
if (selectionError?.quota === true) {
|
|
3240
|
+
emitNativeFallback(
|
|
3241
|
+
threadId,
|
|
3242
|
+
pending,
|
|
3243
|
+
'admission_denied',
|
|
3244
|
+
selectionError.correlationId,
|
|
3245
|
+
);
|
|
3246
|
+
nativeFallback = true;
|
|
3247
|
+
} else if (selectionError !== null) {
|
|
3248
|
+
emitNativeFallback(
|
|
3249
|
+
threadId,
|
|
3250
|
+
pending,
|
|
3251
|
+
'provider_unavailable',
|
|
3252
|
+
selectionError.correlationId,
|
|
3253
|
+
);
|
|
3254
|
+
nativeFallback = true;
|
|
3255
|
+
} else {
|
|
3256
|
+
emitDiagnostic({
|
|
3257
|
+
phase: 'selection',
|
|
3258
|
+
stage: 'selection',
|
|
3259
|
+
reason: 'request_failed',
|
|
3260
|
+
threadId,
|
|
3261
|
+
operationId: pending.operationId,
|
|
3262
|
+
correlationId: selectionError?.correlationId,
|
|
3263
|
+
});
|
|
3264
|
+
throw new Error('recovery selection MCP call failed');
|
|
3265
|
+
}
|
|
3266
|
+
}
|
|
3267
|
+
if (nativeFallback) {
|
|
3268
|
+
emitDiagnostic({
|
|
3269
|
+
phase: 'selection',
|
|
3270
|
+
stage: 'selection',
|
|
3271
|
+
reason: 'admission_denied',
|
|
3272
|
+
threadId,
|
|
3273
|
+
operationId: pending.operationId,
|
|
3274
|
+
});
|
|
3275
|
+
} else {
|
|
3276
|
+
const selectionPayload = responseStructuredContent(selectionResponse);
|
|
3277
|
+
const decisions = parseSelection(selectionPayload, selectionInput, candidates, now());
|
|
3278
|
+
const selectionStats =
|
|
3279
|
+
isRecord(selectionPayload) && isRecord(selectionPayload.stats)
|
|
3280
|
+
? selectionPayload.stats
|
|
3281
|
+
: {};
|
|
3282
|
+
telemetryFor(threadId).selectionCompleted({
|
|
3283
|
+
operationId: pending.operationId,
|
|
3284
|
+
counts: {
|
|
3285
|
+
eligible: Number.isSafeInteger(selectionStats.eligible_count)
|
|
3286
|
+
? selectionStats.eligible_count
|
|
3287
|
+
: candidates.items.filter((item) => !item.protected).length,
|
|
3288
|
+
protected: Number.isSafeInteger(selectionStats.protected_count)
|
|
3289
|
+
? selectionStats.protected_count
|
|
3290
|
+
: candidates.items.filter((item) => item.protected).length,
|
|
3291
|
+
selected: Number.isSafeInteger(selectionStats.selected_count)
|
|
3292
|
+
? selectionStats.selected_count
|
|
3293
|
+
: decisions.filter((item) => item.keep).length,
|
|
3294
|
+
},
|
|
3295
|
+
});
|
|
3296
|
+
if (selectionInput.schema_version === 2) {
|
|
3297
|
+
emitDiagnostic({
|
|
3298
|
+
phase: 'selection_accepted',
|
|
3299
|
+
schemaVersion: 2,
|
|
3300
|
+
optionsHash:
|
|
3301
|
+
isRecord(selectionPayload) && typeof selectionPayload.options_hash === 'string'
|
|
3302
|
+
? selectionPayload.options_hash
|
|
3303
|
+
: null,
|
|
3304
|
+
fittingStage:
|
|
3305
|
+
isRecord(selectionPayload) && isRecord(selectionPayload.fitting)
|
|
3306
|
+
? selectionPayload.fitting.stage
|
|
3307
|
+
: null,
|
|
3308
|
+
batchCount:
|
|
3309
|
+
isRecord(selectionPayload) && isRecord(selectionPayload.stats)
|
|
3310
|
+
? selectionPayload.stats.batch_count
|
|
3311
|
+
: null,
|
|
3312
|
+
providerModel:
|
|
3313
|
+
isRecord(selectionPayload) &&
|
|
3314
|
+
(typeof selectionPayload.provider_model === 'string' ||
|
|
3315
|
+
selectionPayload.provider_model === null)
|
|
3316
|
+
? selectionPayload.provider_model
|
|
3317
|
+
: null,
|
|
3318
|
+
counts: {
|
|
3319
|
+
provider_eligible_count: candidates.items.filter((item) => !item.protected)
|
|
3320
|
+
.length,
|
|
3321
|
+
protected_candidate_count: candidates.items.filter((item) => item.protected)
|
|
3322
|
+
.length,
|
|
3323
|
+
protected_bundle_source_count: candidates.items
|
|
3324
|
+
.filter((item) => Array.isArray(item.sourceRefs))
|
|
3325
|
+
.reduce((count, item) => count + item.sourceRefs.length, 0),
|
|
3326
|
+
},
|
|
3327
|
+
});
|
|
3328
|
+
}
|
|
3329
|
+
const rereadWaiter = deferred();
|
|
3330
|
+
const rereadCapture = await readThreadCapture(threadId, pending, rereadWaiter, {
|
|
3331
|
+
verifyOnly: true,
|
|
3332
|
+
});
|
|
3333
|
+
try {
|
|
3334
|
+
if (captureSourceRevision(rereadCapture) !== sourceRevision)
|
|
3335
|
+
throw new Error('recovery source changed during selection');
|
|
3336
|
+
assertCurrent(threadId, state, pending, revision);
|
|
3337
|
+
} finally {
|
|
3338
|
+
for (const descriptor of rereadCapture.pages ?? [])
|
|
3339
|
+
if (descriptor.stagedPath)
|
|
3340
|
+
await unlink(descriptor.stagedPath).catch(() => undefined);
|
|
3341
|
+
}
|
|
3342
|
+
recoveryNotes = recoveryNotesText(
|
|
3343
|
+
threadId,
|
|
3344
|
+
sourceRevision,
|
|
3345
|
+
selectionInput.request_id,
|
|
3346
|
+
snapshot.capturedAt,
|
|
3347
|
+
relative(recoveryDir, immutableCapturePath).replaceAll('\\', '/'),
|
|
3348
|
+
resolve(recoveryDir),
|
|
3349
|
+
candidates,
|
|
3350
|
+
decisions,
|
|
3351
|
+
);
|
|
3352
|
+
pending.stage = 'notes';
|
|
3353
|
+
await atomicWriteTextPrivate(
|
|
3354
|
+
resolve(recoveryDir, `NOTES.${strictThreadName(threadId)}.md`),
|
|
3355
|
+
recoveryNotes,
|
|
3356
|
+
);
|
|
3357
|
+
telemetryFor(threadId).notesWritten({ operationId: pending.operationId });
|
|
3358
|
+
state.recovery = {
|
|
3359
|
+
version: 1,
|
|
3360
|
+
status: 'awaiting_native',
|
|
3361
|
+
threadId,
|
|
3362
|
+
sourceTurnId,
|
|
3363
|
+
sourceRevision,
|
|
3364
|
+
requestId: selectionInput.request_id,
|
|
3365
|
+
notes: recoveryNotes,
|
|
3366
|
+
notesHash: hashText(recoveryNotes),
|
|
3367
|
+
notesFile: `NOTES.${strictThreadName(threadId)}.md`,
|
|
3368
|
+
itemId: `recovery-${sourceRevision}`,
|
|
3369
|
+
operationId: pending.operationId,
|
|
3370
|
+
createdAt: new Date(now()).toISOString(),
|
|
3371
|
+
};
|
|
3372
|
+
state.recoveryLoaded = true;
|
|
3373
|
+
await persistRecoveryState(threadId, state.recovery);
|
|
3374
|
+
emitDiagnostic({
|
|
3375
|
+
phase: 'selection',
|
|
3376
|
+
stage: 'selection',
|
|
3377
|
+
reason: 'completed',
|
|
3378
|
+
threadId,
|
|
3379
|
+
operationId: pending.operationId,
|
|
3380
|
+
correlationId: selectionPayload.correlation_id,
|
|
3381
|
+
});
|
|
3382
|
+
}
|
|
3383
|
+
}
|
|
1090
3384
|
}
|
|
1091
3385
|
assertCurrent(threadId, state, pending, revision);
|
|
1092
3386
|
}
|
|
3387
|
+
pending.stage = 'native';
|
|
1093
3388
|
pending.compactId = sendOwned('thread/compact/start', { threadId }, 'compact', threadId);
|
|
1094
3389
|
pending.lifecycleArmed = true;
|
|
1095
3390
|
await completion.promise;
|
|
@@ -1102,6 +3397,7 @@ export function createCodexContextProxy({
|
|
|
1102
3397
|
await atomicWritePrivate(
|
|
1103
3398
|
resolve(checkpointDir, `${safeThreadName(threadId)}.completed.json`),
|
|
1104
3399
|
completed,
|
|
3400
|
+
{ maxBytes: MAX_CAPTURE_BYTES, compact: true, capture: true },
|
|
1105
3401
|
);
|
|
1106
3402
|
if (recoveryNotes !== undefined) {
|
|
1107
3403
|
state.recovery = {
|
|
@@ -1111,18 +3407,36 @@ export function createCodexContextProxy({
|
|
|
1111
3407
|
completedAt: new Date(now()).toISOString(),
|
|
1112
3408
|
};
|
|
1113
3409
|
await persistRecoveryState(threadId, state.recovery);
|
|
3410
|
+
telemetryFor(threadId).nativeCompleted({ operationId: pending.operationId });
|
|
1114
3411
|
await attachRecovery(threadId, state);
|
|
1115
|
-
|
|
3412
|
+
emitOutcome({
|
|
1116
3413
|
threadId,
|
|
1117
3414
|
status: 'native_completed',
|
|
1118
3415
|
turnId: pending.turnId,
|
|
1119
|
-
|
|
3416
|
+
operationId: pending.operationId,
|
|
3417
|
+
recoveryStatus: 'pending_attachment',
|
|
3418
|
+
telemetryAlreadyEmitted: true,
|
|
1120
3419
|
});
|
|
1121
3420
|
} else {
|
|
1122
|
-
|
|
3421
|
+
emitOutcome({
|
|
3422
|
+
threadId,
|
|
3423
|
+
status: 'completed',
|
|
3424
|
+
operationId: pending.operationId,
|
|
3425
|
+
turnId: pending.turnId,
|
|
3426
|
+
});
|
|
1123
3427
|
}
|
|
1124
|
-
} catch {
|
|
1125
|
-
|
|
3428
|
+
} catch (error) {
|
|
3429
|
+
emitDiagnostic({
|
|
3430
|
+
phase: 'operation_failed',
|
|
3431
|
+
stage: pending.stage ?? 'native',
|
|
3432
|
+
reason: diagnosticReasonForError(error),
|
|
3433
|
+
threadId,
|
|
3434
|
+
operationId: pending.operationId,
|
|
3435
|
+
...(boundedFailureCounts(error) === undefined
|
|
3436
|
+
? {}
|
|
3437
|
+
: { counts: boundedFailureCounts(error) }),
|
|
3438
|
+
});
|
|
3439
|
+
fail(threadId, 'checkpoint, request, or native completion failed', error);
|
|
1126
3440
|
} finally {
|
|
1127
3441
|
if (pending.timer) pending.timer();
|
|
1128
3442
|
cleanupRequest(pending.readId);
|
|
@@ -1142,22 +3456,44 @@ export function createCodexContextProxy({
|
|
|
1142
3456
|
const owned = ownedRequests.get(message.id);
|
|
1143
3457
|
ownedRequests.delete(message.id);
|
|
1144
3458
|
retire(message.id);
|
|
1145
|
-
if (
|
|
3459
|
+
if (
|
|
3460
|
+
owned.kind === 'read' ||
|
|
3461
|
+
owned.kind === 'page' ||
|
|
3462
|
+
owned.kind === 'select' ||
|
|
3463
|
+
owned.kind === 'budget' ||
|
|
3464
|
+
owned.kind === 'inject' ||
|
|
3465
|
+
owned.kind === 'telemetry'
|
|
3466
|
+
) {
|
|
1146
3467
|
const waiter = internalReads.get(message.id);
|
|
1147
3468
|
internalReads.delete(message.id);
|
|
1148
3469
|
if (waiter) {
|
|
1149
|
-
if ('error' in message) waiter.reject(
|
|
3470
|
+
if ('error' in message) waiter.reject(rpcFailure(owned.kind, message.error));
|
|
1150
3471
|
else waiter.resolve(message.result);
|
|
1151
3472
|
}
|
|
1152
|
-
} else if ('error' in message)
|
|
1153
|
-
|
|
3473
|
+
} else if ('error' in message) {
|
|
3474
|
+
const failure = rpcFailure(owned.kind, message.error);
|
|
3475
|
+
fail(owned.threadId, 'app-server request rejected', failure);
|
|
3476
|
+
}
|
|
3477
|
+
if ('error' in message) {
|
|
3478
|
+
const failure = rpcFailure(owned.kind, message.error);
|
|
1154
3479
|
emitDiagnostic({
|
|
1155
3480
|
phase: owned.kind,
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
?
|
|
1159
|
-
: '
|
|
3481
|
+
stage:
|
|
3482
|
+
owned.kind === 'read' || owned.kind === 'page'
|
|
3483
|
+
? 'capture'
|
|
3484
|
+
: owned.kind === 'budget'
|
|
3485
|
+
? 'budget'
|
|
3486
|
+
: owned.kind === 'select'
|
|
3487
|
+
? 'selection'
|
|
3488
|
+
: owned.kind === 'inject'
|
|
3489
|
+
? 'recovery'
|
|
3490
|
+
: 'transport',
|
|
3491
|
+
reason: 'protocol_error',
|
|
3492
|
+
threadId: owned.threadId,
|
|
3493
|
+
operationId: threads.get(owned.threadId)?.pending?.operationId,
|
|
3494
|
+
...(safeRpcCode(failure.rpcCode) === null ? {} : { rpcCode: failure.rpcCode }),
|
|
1160
3495
|
});
|
|
3496
|
+
}
|
|
1161
3497
|
if (owned.kind === 'compact') {
|
|
1162
3498
|
const state = threads.get(owned.threadId);
|
|
1163
3499
|
if (state?.pending) state.pending.compactId = null;
|
|
@@ -1170,11 +3506,19 @@ export function createCodexContextProxy({
|
|
|
1170
3506
|
typeof message.params.threadId === 'string' &&
|
|
1171
3507
|
isRecord(message.params.status)
|
|
1172
3508
|
) {
|
|
1173
|
-
const state =
|
|
3509
|
+
const state = threads.get(message.params.threadId);
|
|
3510
|
+
if (!state) return true;
|
|
1174
3511
|
state.status =
|
|
1175
3512
|
typeof message.params.status.type === 'string' ? message.params.status.type : 'unknown';
|
|
1176
3513
|
state.idle = state.status === 'idle';
|
|
1177
3514
|
if (!state.idle) invalidateAttachment(state);
|
|
3515
|
+
if (!state.owned) return true;
|
|
3516
|
+
if (!state.telemetryConnected) {
|
|
3517
|
+
state.telemetryConnected = true;
|
|
3518
|
+
telemetryFor(message.params.threadId).connected({
|
|
3519
|
+
triggerPercent: effectiveThreshold,
|
|
3520
|
+
});
|
|
3521
|
+
}
|
|
1178
3522
|
if (state.idle) void requestCompaction(message.params.threadId, state);
|
|
1179
3523
|
}
|
|
1180
3524
|
if (
|
|
@@ -1195,7 +3539,8 @@ export function createCodexContextProxy({
|
|
|
1195
3539
|
stderr.write('codex-context: invalid usage sample skipped\n');
|
|
1196
3540
|
return true;
|
|
1197
3541
|
}
|
|
1198
|
-
const state =
|
|
3542
|
+
const state = threads.get(params.threadId);
|
|
3543
|
+
if (!state) return true;
|
|
1199
3544
|
state.usage = params.tokenUsage;
|
|
1200
3545
|
state.usageReceivedAt = now();
|
|
1201
3546
|
state.turnId = params.turnId;
|
|
@@ -1209,6 +3554,17 @@ export function createCodexContextProxy({
|
|
|
1209
3554
|
state.turnId !== state.recovery.sourceTurnId
|
|
1210
3555
|
)
|
|
1211
3556
|
invalidateAttachment(state);
|
|
3557
|
+
if (!state.owned) return true;
|
|
3558
|
+
telemetryFor(params.threadId).usage({
|
|
3559
|
+
measurement: {
|
|
3560
|
+
unit: 'tokens',
|
|
3561
|
+
used: params.tokenUsage.last.totalTokens,
|
|
3562
|
+
...(hasKnownContextWindow(params.tokenUsage)
|
|
3563
|
+
? { capacity: params.tokenUsage.modelContextWindow }
|
|
3564
|
+
: {}),
|
|
3565
|
+
},
|
|
3566
|
+
triggerPercent: effectiveThreshold,
|
|
3567
|
+
});
|
|
1212
3568
|
if (state.idle) void requestCompaction(params.threadId, state);
|
|
1213
3569
|
}
|
|
1214
3570
|
if (
|
|
@@ -1241,6 +3597,13 @@ export function createCodexContextProxy({
|
|
|
1241
3597
|
pending.turnId = typeof p.turnId === 'string' ? p.turnId : null;
|
|
1242
3598
|
pending.started = true;
|
|
1243
3599
|
pending.itemId = p.item.id ?? null;
|
|
3600
|
+
emitDiagnostic({
|
|
3601
|
+
phase: 'native_started',
|
|
3602
|
+
stage: 'native',
|
|
3603
|
+
reason: 'requested',
|
|
3604
|
+
threadId: p.threadId,
|
|
3605
|
+
operationId: pending.operationId,
|
|
3606
|
+
});
|
|
1244
3607
|
}
|
|
1245
3608
|
}
|
|
1246
3609
|
if (
|
|
@@ -1259,6 +3622,13 @@ export function createCodexContextProxy({
|
|
|
1259
3622
|
pending.itemId === (p.item.id ?? null)
|
|
1260
3623
|
) {
|
|
1261
3624
|
pending.itemCompleted = true;
|
|
3625
|
+
emitDiagnostic({
|
|
3626
|
+
phase: 'native_item_completed',
|
|
3627
|
+
stage: 'native',
|
|
3628
|
+
reason: 'completed',
|
|
3629
|
+
threadId: p.threadId,
|
|
3630
|
+
operationId: pending.operationId,
|
|
3631
|
+
});
|
|
1262
3632
|
if (pending.turnCompleted) pending.deferred.resolve();
|
|
1263
3633
|
}
|
|
1264
3634
|
}
|
|
@@ -1274,7 +3644,16 @@ export function createCodexContextProxy({
|
|
|
1274
3644
|
if (pending?.lifecycleArmed && pending.turnId === turn.id) {
|
|
1275
3645
|
if (turn.status === 'completed') pending.turnCompleted = true;
|
|
1276
3646
|
else fail(message.params.threadId, 'native compaction turn failed');
|
|
1277
|
-
if (pending.itemCompleted && pending.turnCompleted)
|
|
3647
|
+
if (pending.itemCompleted && pending.turnCompleted) {
|
|
3648
|
+
emitDiagnostic({
|
|
3649
|
+
phase: 'native_completed',
|
|
3650
|
+
stage: 'native',
|
|
3651
|
+
reason: 'completed',
|
|
3652
|
+
threadId: message.params.threadId,
|
|
3653
|
+
operationId: pending.operationId,
|
|
3654
|
+
});
|
|
3655
|
+
pending.deferred.resolve();
|
|
3656
|
+
}
|
|
1278
3657
|
}
|
|
1279
3658
|
}
|
|
1280
3659
|
if (
|
|
@@ -1289,11 +3668,86 @@ export function createCodexContextProxy({
|
|
|
1289
3668
|
typeof message.params.threadId === 'string'
|
|
1290
3669
|
) {
|
|
1291
3670
|
fail(message.params.threadId, 'thread closed');
|
|
3671
|
+
ownedThreadIds.delete(message.params.threadId);
|
|
1292
3672
|
threads.delete(message.params.threadId);
|
|
1293
3673
|
}
|
|
1294
3674
|
return true;
|
|
1295
3675
|
}
|
|
1296
3676
|
|
|
3677
|
+
/**
|
|
3678
|
+
* Observe TUI-owned requests without changing their wire representation.
|
|
3679
|
+
* The app-server may emit status/usage events before a resumed thread's
|
|
3680
|
+
* response arrives, so the adapter must defer owned work until that response
|
|
3681
|
+
* confirms that the thread is loaded.
|
|
3682
|
+
*/
|
|
3683
|
+
function observeClientRequest(message) {
|
|
3684
|
+
if (!isRecord(message)) return;
|
|
3685
|
+
const requestId = message.id;
|
|
3686
|
+
const validRequestId =
|
|
3687
|
+
(typeof requestId === 'string' && requestId.length > 0 && requestId.length <= 128) ||
|
|
3688
|
+
Number.isSafeInteger(requestId);
|
|
3689
|
+
if (!validRequestId) return;
|
|
3690
|
+
const requestKey = clientRequestKey(requestId);
|
|
3691
|
+
if (requestKey === null) return;
|
|
3692
|
+
if (['thread/start', 'thread/resume', 'thread/fork'].includes(message.method)) {
|
|
3693
|
+
const threadId =
|
|
3694
|
+
isRecord(message.params) && typeof message.params.threadId === 'string'
|
|
3695
|
+
? message.params.threadId
|
|
3696
|
+
: null;
|
|
3697
|
+
if (
|
|
3698
|
+
clientRequestBindings.has(requestKey) ||
|
|
3699
|
+
clientRequestBindings.size >= MAX_CLIENT_REQUEST_BINDINGS
|
|
3700
|
+
)
|
|
3701
|
+
return;
|
|
3702
|
+
clientRequestBindings.set(requestKey, { method: message.method, threadId });
|
|
3703
|
+
if (message.method === 'thread/resume') {
|
|
3704
|
+
if (threadId === null || threadId.length === 0 || threadId.length > 256) {
|
|
3705
|
+
clientRequestBindings.delete(requestKey);
|
|
3706
|
+
return;
|
|
3707
|
+
}
|
|
3708
|
+
const state = threadState(threadId);
|
|
3709
|
+
state.resumePending = true;
|
|
3710
|
+
state.resumeRetryRequested = false;
|
|
3711
|
+
state.resumeRequests.add(requestKey);
|
|
3712
|
+
}
|
|
3713
|
+
}
|
|
3714
|
+
}
|
|
3715
|
+
|
|
3716
|
+
function observeClientResponse(message) {
|
|
3717
|
+
if (!isRecord(message) || message.id === undefined) return;
|
|
3718
|
+
const requestKey = clientRequestKey(message.id);
|
|
3719
|
+
if (requestKey === null) return;
|
|
3720
|
+
const binding = clientRequestBindings.get(requestKey);
|
|
3721
|
+
if (binding) {
|
|
3722
|
+
clientRequestBindings.delete(requestKey);
|
|
3723
|
+
if (binding.method === 'thread/resume' && binding.threadId !== null) {
|
|
3724
|
+
const state = threadState(binding.threadId);
|
|
3725
|
+
state.resumeRequests.delete(requestKey);
|
|
3726
|
+
state.resumePending = state.resumeRequests.size > 0;
|
|
3727
|
+
state.resumeRetryRequested = false;
|
|
3728
|
+
const responseThreadId = message.result?.thread?.id;
|
|
3729
|
+
if (
|
|
3730
|
+
message.error === undefined &&
|
|
3731
|
+
typeof responseThreadId === 'string' &&
|
|
3732
|
+
responseThreadId === binding.threadId
|
|
3733
|
+
) {
|
|
3734
|
+
const claimed = claimOwnedThread(binding.threadId);
|
|
3735
|
+
if (claimed && !state.resumePending && claimed.idle)
|
|
3736
|
+
void requestCompaction(binding.threadId, claimed);
|
|
3737
|
+
} else if (!state.owned && state.resumeRequests.size === 0 && state.pending === null) {
|
|
3738
|
+
threads.delete(binding.threadId);
|
|
3739
|
+
}
|
|
3740
|
+
} else if (message.error === undefined) {
|
|
3741
|
+
const selectedId = message.result?.thread?.id;
|
|
3742
|
+
if (typeof selectedId === 'string' && selectedId.length > 0 && selectedId.length <= 256) {
|
|
3743
|
+
const selected = claimOwnedThread(selectedId);
|
|
3744
|
+
if (!selected) return;
|
|
3745
|
+
if (selected.idle) void requestCompaction(selectedId, selected);
|
|
3746
|
+
}
|
|
3747
|
+
}
|
|
3748
|
+
}
|
|
3749
|
+
}
|
|
3750
|
+
|
|
1297
3751
|
const decoder = new StringDecoder('utf8');
|
|
1298
3752
|
let inputBuffer = '';
|
|
1299
3753
|
let passthrough = false;
|
|
@@ -1310,44 +3764,100 @@ export function createCodexContextProxy({
|
|
|
1310
3764
|
}
|
|
1311
3765
|
};
|
|
1312
3766
|
const processLine = (line) => {
|
|
1313
|
-
if (Buffer.byteLength(line) > MAX_LINE_BYTES) {
|
|
1314
|
-
|
|
3767
|
+
if (Buffer.byteLength(line, 'utf8') + 1 > MAX_LINE_BYTES) {
|
|
3768
|
+
const ownedId = internalRequestId(line);
|
|
3769
|
+
if (ownedId !== null) {
|
|
1315
3770
|
stderr.write('codex-context: oversized internal response rejected\n');
|
|
1316
|
-
|
|
3771
|
+
failOversizedInternal(ownedId, Buffer.byteLength(line, 'utf8') + 1);
|
|
1317
3772
|
} else forward(`${line}\n`);
|
|
1318
3773
|
return;
|
|
1319
3774
|
}
|
|
1320
3775
|
let forwardLine = true;
|
|
1321
3776
|
try {
|
|
1322
|
-
|
|
3777
|
+
const message = JSON.parse(line.endsWith('\r') ? line.slice(0, -1) : line);
|
|
3778
|
+
observeClientResponse(message);
|
|
3779
|
+
forwardLine = handle(message);
|
|
1323
3780
|
} catch {
|
|
1324
3781
|
/* preserve malformed external protocol lines */
|
|
1325
3782
|
}
|
|
1326
3783
|
if (forwardLine) forward(`${line}\n`);
|
|
1327
3784
|
};
|
|
3785
|
+
function internalRequestId(line) {
|
|
3786
|
+
const match =
|
|
3787
|
+
/^\s*\{\s*(?:"id"\s*:\s*"(ai-brain-context-[A-Za-z0-9._:-]{1,128})"|"jsonrpc"\s*:\s*"2\.0"\s*,\s*"id"\s*:\s*"(ai-brain-context-[A-Za-z0-9._:-]{1,128})")/u.exec(
|
|
3788
|
+
line,
|
|
3789
|
+
);
|
|
3790
|
+
return match?.[1] ?? match?.[2] ?? null;
|
|
3791
|
+
}
|
|
3792
|
+
function failCompactions(reason) {
|
|
3793
|
+
for (const [threadId, state] of threads) {
|
|
3794
|
+
if (state.pending) fail(threadId, reason);
|
|
3795
|
+
}
|
|
3796
|
+
}
|
|
3797
|
+
function failOversizedInternal(id, responseBytes) {
|
|
3798
|
+
const owned = ownedRequests.get(id);
|
|
3799
|
+
if (!owned) {
|
|
3800
|
+
failCompactions('oversized internal response');
|
|
3801
|
+
return;
|
|
3802
|
+
}
|
|
3803
|
+
if (owned.kind === 'page') {
|
|
3804
|
+
ownedRequests.delete(id);
|
|
3805
|
+
const waiter = internalReads.get(id);
|
|
3806
|
+
internalReads.delete(id);
|
|
3807
|
+
retire(id);
|
|
3808
|
+
const failure = new Error('oversized internal page response');
|
|
3809
|
+
failure.diagnosticReason = 'capture_size_exceeded';
|
|
3810
|
+
failure.responseBytes = responseBytes;
|
|
3811
|
+
failure.responseLimitBytes = MAX_LINE_BYTES;
|
|
3812
|
+
waiter?.reject(failure);
|
|
3813
|
+
return;
|
|
3814
|
+
}
|
|
3815
|
+
cleanupRequest(id);
|
|
3816
|
+
if (owned.kind === 'telemetry') return;
|
|
3817
|
+
const failure = new Error('oversized internal response');
|
|
3818
|
+
failure.diagnosticReason = 'capture_size_exceeded';
|
|
3819
|
+
failure.responseBytes = responseBytes;
|
|
3820
|
+
failure.responseLimitBytes = MAX_LINE_BYTES;
|
|
3821
|
+
fail(owned.threadId, 'oversized internal response', failure);
|
|
3822
|
+
}
|
|
1328
3823
|
child.stdout.on('data', (chunk) => {
|
|
1329
3824
|
if (stopped) return;
|
|
1330
|
-
|
|
3825
|
+
const decoded = decoder.write(chunk);
|
|
3826
|
+
if (discardOversized) {
|
|
3827
|
+
const newline = decoded.indexOf('\n');
|
|
3828
|
+
if (newline < 0) return;
|
|
3829
|
+
discardOversized = false;
|
|
3830
|
+
inputBuffer = decoded.slice(newline + 1);
|
|
3831
|
+
} else inputBuffer += decoded;
|
|
1331
3832
|
while (true) {
|
|
1332
3833
|
const newline = inputBuffer.indexOf('\n');
|
|
1333
3834
|
if (newline < 0) break;
|
|
1334
3835
|
const line = inputBuffer.slice(0, newline);
|
|
1335
3836
|
inputBuffer = inputBuffer.slice(newline + 1);
|
|
1336
|
-
if (
|
|
3837
|
+
if (discardOversized) {
|
|
3838
|
+
discardOversized = false;
|
|
3839
|
+
} else if (passthrough) {
|
|
1337
3840
|
forward(`${line}\n`);
|
|
1338
3841
|
passthrough = false;
|
|
1339
|
-
} else if (discardOversized) {
|
|
1340
|
-
discardOversized = false;
|
|
1341
3842
|
} else processLine(line);
|
|
1342
3843
|
}
|
|
1343
|
-
if (Buffer.byteLength(inputBuffer) > MAX_LINE_BYTES) {
|
|
1344
|
-
if (
|
|
3844
|
+
if (Buffer.byteLength(inputBuffer, 'utf8') + 1 > MAX_LINE_BYTES) {
|
|
3845
|
+
if (passthrough) {
|
|
3846
|
+
forward(inputBuffer);
|
|
3847
|
+
inputBuffer = '';
|
|
3848
|
+
} else if (internalRequestId(inputBuffer) !== null) {
|
|
1345
3849
|
stderr.write('codex-context: oversized internal response rejected\n');
|
|
1346
|
-
|
|
3850
|
+
failOversizedInternal(
|
|
3851
|
+
internalRequestId(inputBuffer),
|
|
3852
|
+
Buffer.byteLength(inputBuffer, 'utf8') + 1,
|
|
3853
|
+
);
|
|
1347
3854
|
discardOversized = true;
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
3855
|
+
inputBuffer = '';
|
|
3856
|
+
} else {
|
|
3857
|
+
forward(inputBuffer);
|
|
3858
|
+
inputBuffer = '';
|
|
3859
|
+
passthrough = true;
|
|
3860
|
+
}
|
|
1351
3861
|
}
|
|
1352
3862
|
});
|
|
1353
3863
|
child.stdout.on('end', () => {
|
|
@@ -1367,36 +3877,138 @@ export function createCodexContextProxy({
|
|
|
1367
3877
|
failAll('app-server disconnected');
|
|
1368
3878
|
emitDiagnostic({ phase: 'transport_close' });
|
|
1369
3879
|
});
|
|
3880
|
+
// Native app-server stderr is operational logging and may contain private
|
|
3881
|
+
// paths, prompts or tool output. Drain the owned pipe without forwarding or
|
|
3882
|
+
// persisting its contents; protocol errors remain on stdout and process exit
|
|
3883
|
+
// status is still propagated by the launcher.
|
|
3884
|
+
if (child.stderr && typeof child.stderr.on === 'function') {
|
|
3885
|
+
child.stderr.on('data', () => undefined);
|
|
3886
|
+
child.stderr.resume?.();
|
|
3887
|
+
}
|
|
1370
3888
|
return {
|
|
1371
3889
|
threads,
|
|
1372
|
-
|
|
1373
|
-
|
|
3890
|
+
observeClientRequest,
|
|
3891
|
+
observeClientResponse,
|
|
3892
|
+
async stop() {
|
|
3893
|
+
if (stopped) {
|
|
3894
|
+
await Promise.all(
|
|
3895
|
+
[...diagnosticSinks.values()].map((sink) =>
|
|
3896
|
+
typeof sink.flush === 'function'
|
|
3897
|
+
? Promise.resolve(sink.flush()).catch(() => undefined)
|
|
3898
|
+
: undefined,
|
|
3899
|
+
),
|
|
3900
|
+
);
|
|
3901
|
+
return;
|
|
3902
|
+
}
|
|
3903
|
+
for (const reporter of telemetryByThread.values()) {
|
|
3904
|
+
reporter.disconnected();
|
|
3905
|
+
void reporter.flush();
|
|
3906
|
+
}
|
|
1374
3907
|
stopped = true;
|
|
3908
|
+
clientRequestBindings.clear();
|
|
3909
|
+
ownedThreadIds.clear();
|
|
1375
3910
|
failAll('adapter stopped');
|
|
3911
|
+
await Promise.all(
|
|
3912
|
+
[...diagnosticSinks.values()].map((sink) =>
|
|
3913
|
+
typeof sink.flush === 'function'
|
|
3914
|
+
? Promise.resolve(sink.flush()).catch(() => undefined)
|
|
3915
|
+
: undefined,
|
|
3916
|
+
),
|
|
3917
|
+
);
|
|
1376
3918
|
},
|
|
1377
3919
|
};
|
|
1378
3920
|
}
|
|
1379
3921
|
|
|
1380
|
-
function main() {
|
|
3922
|
+
async function main() {
|
|
1381
3923
|
if (process.argv.includes('--help')) {
|
|
1382
3924
|
process.stderr.write(
|
|
1383
3925
|
'Usage: node scripts/codex-context-compaction.mjs [app-server arguments...]\nStarts an owned Codex app-server and requests native compaction at CODEX_CONTEXT_COMPACT_AT (default 85).\n',
|
|
1384
3926
|
);
|
|
1385
3927
|
return;
|
|
1386
3928
|
}
|
|
3929
|
+
const policy = parseContextPolicy();
|
|
1387
3930
|
const command = process.env.CODEX_BIN ?? 'codex';
|
|
1388
|
-
const
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
}
|
|
1392
|
-
|
|
3931
|
+
const profile = process.env.CODEX_APP_SERVER_PROFILE;
|
|
3932
|
+
if (profile !== undefined && !/^[A-Za-z0-9._-]{1,64}$/u.test(profile)) {
|
|
3933
|
+
throw new Error('CODEX_APP_SERVER_PROFILE is invalid');
|
|
3934
|
+
}
|
|
3935
|
+
const child = spawn(
|
|
3936
|
+
command,
|
|
3937
|
+
[
|
|
3938
|
+
...(profile === undefined ? [] : ['--profile', profile]),
|
|
3939
|
+
'app-server',
|
|
3940
|
+
...process.argv.slice(2),
|
|
3941
|
+
],
|
|
3942
|
+
{
|
|
3943
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
3944
|
+
env: codexChildEnvironment(),
|
|
3945
|
+
},
|
|
3946
|
+
);
|
|
3947
|
+
let diagnosticFactory = null;
|
|
3948
|
+
const diagnosticModule = process.env.CODEX_CONTEXT_DIAGNOSTICS_MODULE;
|
|
3949
|
+
if (
|
|
3950
|
+
process.env.CODEX_CONTEXT_DIAGNOSTICS_ENABLED === '1' &&
|
|
3951
|
+
typeof diagnosticModule === 'string' &&
|
|
3952
|
+
diagnosticModule.length > 0
|
|
3953
|
+
) {
|
|
3954
|
+
try {
|
|
3955
|
+
const module = await import(pathToFileURL(resolve(diagnosticModule)).href);
|
|
3956
|
+
if (typeof module.createCodexDiagnosticSink === 'function') {
|
|
3957
|
+
const stateDir = process.env.CODEX_CONTEXT_DIAGNOSTICS_DIR ?? process.cwd();
|
|
3958
|
+
diagnosticFactory = (threadId) =>
|
|
3959
|
+
module.createCodexDiagnosticSink({
|
|
3960
|
+
stateDir,
|
|
3961
|
+
sessionId: threadId,
|
|
3962
|
+
clientVersion: process.env.CODEX_CLIENT_VERSION ?? '0.161.0',
|
|
3963
|
+
enabled: true,
|
|
3964
|
+
});
|
|
3965
|
+
}
|
|
3966
|
+
} catch {
|
|
3967
|
+
diagnosticFactory = null;
|
|
3968
|
+
}
|
|
3969
|
+
}
|
|
3970
|
+
const proxy = createCodexContextProxy({
|
|
1393
3971
|
child,
|
|
1394
3972
|
threshold: numberEnv('CODEX_CONTEXT_COMPACT_AT', DEFAULT_THRESHOLD),
|
|
1395
3973
|
timeoutMs: numberEnv('CODEX_CONTEXT_COMPACT_TIMEOUT_MS', DEFAULT_TIMEOUT_MS),
|
|
3974
|
+
policy,
|
|
3975
|
+
createDiagnosticSink: diagnosticFactory,
|
|
3976
|
+
});
|
|
3977
|
+
const clientDecoder = new StringDecoder('utf8');
|
|
3978
|
+
let clientInputBuffer = '';
|
|
3979
|
+
let discardOversizedClientFrame = false;
|
|
3980
|
+
process.stdin.on('data', (chunk) => {
|
|
3981
|
+
const decoded = clientDecoder.write(chunk);
|
|
3982
|
+
if (discardOversizedClientFrame) {
|
|
3983
|
+
const newline = decoded.indexOf('\n');
|
|
3984
|
+
if (newline < 0) return;
|
|
3985
|
+
discardOversizedClientFrame = false;
|
|
3986
|
+
clientInputBuffer = decoded.slice(newline + 1);
|
|
3987
|
+
} else clientInputBuffer += decoded;
|
|
3988
|
+
while (true) {
|
|
3989
|
+
const newline = clientInputBuffer.indexOf('\n');
|
|
3990
|
+
if (newline < 0) break;
|
|
3991
|
+
const line = clientInputBuffer.slice(0, newline).replace(/\r$/u, '');
|
|
3992
|
+
clientInputBuffer = clientInputBuffer.slice(newline + 1);
|
|
3993
|
+
if (!line) continue;
|
|
3994
|
+
if (Buffer.byteLength(line, 'utf8') + 1 > MAX_LINE_BYTES) continue;
|
|
3995
|
+
try {
|
|
3996
|
+
proxy.observeClientRequest(JSON.parse(line));
|
|
3997
|
+
} catch {
|
|
3998
|
+
/* Preserve malformed/native frames unchanged; the app-server owns validation. */
|
|
3999
|
+
}
|
|
4000
|
+
}
|
|
4001
|
+
if (Buffer.byteLength(clientInputBuffer, 'utf8') + 1 > MAX_LINE_BYTES) {
|
|
4002
|
+
clientInputBuffer = '';
|
|
4003
|
+
discardOversizedClientFrame = true;
|
|
4004
|
+
}
|
|
1396
4005
|
});
|
|
1397
4006
|
process.stdin.pipe(child.stdin);
|
|
1398
4007
|
process.stdin.on('end', () => child.stdin.end());
|
|
1399
|
-
child.on('exit', (code, signal) =>
|
|
4008
|
+
child.on('exit', async (code, signal) => {
|
|
4009
|
+
await proxy.stop();
|
|
4010
|
+
process.exitCode = code ?? (signal ? 1 : 0);
|
|
4011
|
+
});
|
|
1400
4012
|
}
|
|
1401
4013
|
if (process.argv[1] && resolve(process.argv[1]) === resolve(new URL(import.meta.url).pathname))
|
|
1402
|
-
main();
|
|
4014
|
+
void main();
|