@ours.network/fleet 1.2.0-nightly.66 → 1.2.0-nightly.68
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -0
- package/dist/atomic-file.d.ts +17 -0
- package/dist/atomic-file.js +93 -18
- package/dist/build-info.json +6 -4
- package/dist/capabilities.d.ts +4 -1
- package/dist/capabilities.js +5 -0
- package/dist/cli.js +30 -0
- package/dist/docs.d.ts +1 -1
- package/dist/docs.js +13 -1
- package/dist/doctor.d.ts +3 -0
- package/dist/doctor.js +20 -0
- package/dist/harness/agent-session.d.ts +2 -0
- package/dist/harness/claude-code-session.js +1 -1
- package/dist/harness/codex-session.js +2 -2
- package/dist/process-incarnation.d.ts +23 -0
- package/dist/process-incarnation.js +42 -0
- package/dist/rooms-tasks/cowork-adapter.d.ts +10 -1
- package/dist/rooms-tasks/cowork-adapter.js +8 -0
- package/dist/rooms-tasks/cowork-http.js +29 -7
- package/dist/runner.d.ts +7 -0
- package/dist/runner.js +76 -12
- package/dist/session/acp.d.ts +14 -0
- package/dist/session/acp.js +22 -6
- package/dist/session/codex-app-server.d.ts +6 -0
- package/dist/session/codex-app-server.js +17 -4
- package/dist/session/types.d.ts +6 -0
- package/dist/startup-readiness.d.ts +19 -0
- package/dist/startup-readiness.js +63 -0
- package/dist/supervisor/registration-refresh.d.ts +45 -0
- package/dist/supervisor/registration-refresh.js +266 -0
- package/dist/workspace-enrollment.d.ts +3 -2
- package/dist/workspace-enrollment.js +19 -5
- package/package.json +1 -1
|
@@ -5,6 +5,16 @@ import { CoworkProtocolError, CoworkUnavailableError } from './cowork-adapter.js
|
|
|
5
5
|
const TRANSPORT_CODES = new Set(['EACCES', 'ENOENT', 'ECONNREFUSED', 'ECONNRESET',
|
|
6
6
|
'ETIMEDOUT', 'ENOTFOUND', 'EAI_AGAIN', 'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_SOCKET',
|
|
7
7
|
'CERT_HAS_EXPIRED', 'ERR_TLS_CERT_ALTNAME_INVALID']);
|
|
8
|
+
/** Failure kinds a dependency still starting (or restarting) produces. */
|
|
9
|
+
const RETRYABLE_KINDS = new Set(['ECONNREFUSED', 'ECONNRESET', 'ETIMEDOUT', 'EAI_AGAIN',
|
|
10
|
+
'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_SOCKET', 'timeout']);
|
|
11
|
+
/** Statuses a reverse proxy answers with while its upstream is not serving. */
|
|
12
|
+
const GATEWAY_UNAVAILABLE = new Set([502, 503, 504]);
|
|
13
|
+
/** Media type only, never parameters; anything unusual collapses to a fixed label. */
|
|
14
|
+
function safeContentType(response) {
|
|
15
|
+
const value = (response.headers.get('content-type') ?? '').split(';')[0].trim().toLowerCase();
|
|
16
|
+
return /^[a-z0-9.+-]{1,40}\/[a-z0-9.+-]{1,40}$/.test(value) ? value : 'unknown type';
|
|
17
|
+
}
|
|
8
18
|
function safeFailureKind(error, signal) {
|
|
9
19
|
if (signal.aborted)
|
|
10
20
|
return 'timeout';
|
|
@@ -22,6 +32,9 @@ function safeFailureKind(error, signal) {
|
|
|
22
32
|
}
|
|
23
33
|
return 'transport_error';
|
|
24
34
|
}
|
|
35
|
+
function safeOperation(method) {
|
|
36
|
+
return /^[a-z][a-z0-9_.-]{0,79}$/.test(method) ? method : 'management';
|
|
37
|
+
}
|
|
25
38
|
/** Same RPC protocol as the local socket; never replay a possibly delivered mutation. */
|
|
26
39
|
export async function coworkHttpCall(profile, method, params, timeoutMs) {
|
|
27
40
|
const id = randomUUID();
|
|
@@ -67,16 +80,25 @@ export async function coworkHttpCall(profile, method, params, timeoutMs) {
|
|
|
67
80
|
finally {
|
|
68
81
|
await reader.cancel();
|
|
69
82
|
}
|
|
83
|
+
// Status and media type, never the body: a proxy page may echo request data.
|
|
84
|
+
const shape = `HTTP ${response.status}, ${safeContentType(response)}`;
|
|
70
85
|
let rpc;
|
|
86
|
+
let malformed = false;
|
|
71
87
|
try {
|
|
72
88
|
rpc = JSON.parse(Buffer.concat(chunks).toString('utf8'));
|
|
73
89
|
}
|
|
74
90
|
catch {
|
|
75
|
-
|
|
91
|
+
malformed = true;
|
|
92
|
+
}
|
|
93
|
+
if (malformed || !rpc || Array.isArray(rpc) || rpc.version !== 1 || rpc.id !== id
|
|
94
|
+
|| Object.hasOwn(rpc, 'result') === Object.hasOwn(rpc, 'error')) {
|
|
95
|
+
// A gateway whose Cowork upstream is still starting answers with its own
|
|
96
|
+
// error page. That is unavailability, not a Cowork protocol violation; a
|
|
97
|
+
// well-formed RPC error under the same status keeps its meaning below.
|
|
98
|
+
if (GATEWAY_UNAVAILABLE.has(response.status))
|
|
99
|
+
throw new CoworkUnavailableError(`Cowork HTTP ${safeOperation(method)} failed: gateway answered ${shape} without an RPC response`, { retryable: true });
|
|
100
|
+
throw new CoworkProtocolError(method, `server returned ${malformed ? 'malformed JSON' : 'an invalid RPC response'} (${shape})`);
|
|
76
101
|
}
|
|
77
|
-
if (!rpc || Array.isArray(rpc) || rpc.version !== 1 || rpc.id !== id
|
|
78
|
-
|| Object.hasOwn(rpc, 'result') === Object.hasOwn(rpc, 'error'))
|
|
79
|
-
throw new CoworkProtocolError(method, 'server returned an invalid RPC response');
|
|
80
102
|
if (Object.hasOwn(rpc, 'error')) {
|
|
81
103
|
const error = rpc.error;
|
|
82
104
|
if (!error || typeof error.code !== 'string' || typeof error.message !== 'string')
|
|
@@ -88,11 +110,11 @@ export async function coworkHttpCall(profile, method, params, timeoutMs) {
|
|
|
88
110
|
return rpc.result;
|
|
89
111
|
}
|
|
90
112
|
catch (error) {
|
|
91
|
-
if (error instanceof CoworkProtocolError)
|
|
113
|
+
if (error instanceof CoworkProtocolError || error instanceof CoworkUnavailableError)
|
|
92
114
|
throw error;
|
|
93
115
|
// Durable cleanup records retain only .message. Keep the RPC step and a
|
|
94
116
|
// bounded allowlisted category there, never raw URLs, credentials or cause text.
|
|
95
|
-
const
|
|
96
|
-
throw new CoworkUnavailableError(`Cowork HTTP ${
|
|
117
|
+
const kind = safeFailureKind(error, signal);
|
|
118
|
+
throw new CoworkUnavailableError(`Cowork HTTP ${safeOperation(method)} failed during ${stage} (${kind})`, { cause: error, retryable: stage !== 'credential read' && RETRYABLE_KINDS.has(kind) });
|
|
97
119
|
}
|
|
98
120
|
}
|
package/dist/runner.d.ts
CHANGED
|
@@ -41,6 +41,8 @@ export interface RunnerDeps {
|
|
|
41
41
|
shouldStop?(): boolean;
|
|
42
42
|
/** Parent shutdown suspends a managed transient member; explicit stop still retires. */
|
|
43
43
|
suspendOnStop?: boolean;
|
|
44
|
+
/** One-shot timer for the sustained-health check; returns its cancel function. */
|
|
45
|
+
setTimer?(fn: () => void, ms: number): () => void;
|
|
44
46
|
}
|
|
45
47
|
export declare const SUPERVISOR_RECYCLE_REQUIRED = "OWNER_CHANNEL_SUPERVISOR_RECYCLE_REQUIRED";
|
|
46
48
|
export declare class SupervisorRecycleRequiredError extends Error {
|
|
@@ -177,6 +179,8 @@ export interface AttemptResult {
|
|
|
177
179
|
modelRecovery?: 'advance' | 'hold';
|
|
178
180
|
/** Present only when a temporary-role lifecycle signal ended the session. */
|
|
179
181
|
retirementReason?: 'identity-closed' | 'operator-stop' | 'supervisor-signal' | 'idle-timeout';
|
|
182
|
+
/** Seconds the session spent up after startup completed; absent if it never came up. */
|
|
183
|
+
healthySecs?: number;
|
|
180
184
|
}
|
|
181
185
|
/** Continuous authoritative absence required after an identity was observed. */
|
|
182
186
|
export declare const TEMP_IDENTITY_CLOSE_DEBOUNCE_MS = 5000;
|
|
@@ -190,6 +194,8 @@ export declare function runOnce(name: string, opts?: {
|
|
|
190
194
|
configPath?: string;
|
|
191
195
|
allowResumeRotation?: boolean;
|
|
192
196
|
identityLifetime?: 'permanent' | 'temporary';
|
|
197
|
+
/** Called once when the session is up, so the supervisor can time sustained health. */
|
|
198
|
+
onHealthy?: () => void;
|
|
193
199
|
}, partialDeps?: Partial<RunnerDeps>): Promise<AttemptResult>;
|
|
194
200
|
/**
|
|
195
201
|
* The persistent supervisor for one permanent role: run child sessions in a
|
|
@@ -208,6 +214,7 @@ export declare function runSupervised(name: string, opts?: {
|
|
|
208
214
|
configPath?: string;
|
|
209
215
|
allowResumeRotation?: boolean;
|
|
210
216
|
temp?: boolean;
|
|
217
|
+
onHealthy?: () => void;
|
|
211
218
|
}, d: Partial<RunnerDeps>) => Promise<AttemptResult>): Promise<RestartLedger>;
|
|
212
219
|
/** Temp-agent entrypoint: run once, journal why it ended, then archive its evidence. */
|
|
213
220
|
/** The parent uses the same adapter-relative sustained-uptime reset boundary. */
|
package/dist/runner.js
CHANGED
|
@@ -328,6 +328,11 @@ const carriedForward = (previous) => ({
|
|
|
328
328
|
...(previous.abruptTerminations ? { abruptTerminations: previous.abruptTerminations } : {}),
|
|
329
329
|
...(previous.supervisorStartedAt ? { supervisorStartedAt: previous.supervisorStartedAt } : {}),
|
|
330
330
|
});
|
|
331
|
+
function unrefTimer(fn, ms) {
|
|
332
|
+
const timer = setTimeout(fn, ms);
|
|
333
|
+
timer.unref?.();
|
|
334
|
+
return () => clearTimeout(timer);
|
|
335
|
+
}
|
|
331
336
|
/** Bounded exponential backoff for the nth consecutive immediate failure. */
|
|
332
337
|
export function backoffFor(consecutiveFailures) {
|
|
333
338
|
if (consecutiveFailures <= 0)
|
|
@@ -713,7 +718,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
713
718
|
managedOurs: managedHarness.ours,
|
|
714
719
|
role, prep,
|
|
715
720
|
launch: { ...launch, argv: wrappedArgv, env: managedHarness.env },
|
|
716
|
-
cwd: runCwd, stateDir: dir, mode, requireResume, permissions: perms,
|
|
721
|
+
cwd: runCwd, stateDir: dir, mode, requireResume, deferRecoveredPrompts: true, permissions: perms,
|
|
717
722
|
permissionMode: effectivePermissionMode(role), log: deps.log,
|
|
718
723
|
}));
|
|
719
724
|
managedService.setFileDelivery?.(agentSession.sendFileToChat?.bind(agentSession));
|
|
@@ -841,6 +846,11 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
841
846
|
}
|
|
842
847
|
return event;
|
|
843
848
|
});
|
|
849
|
+
// Recovered prompts may run the managed Fleet CLI immediately. Start them
|
|
850
|
+
// only now that the control socket, audit and spawn handlers are live, and
|
|
851
|
+
// never for a role being stopped: they stay durable for its next start.
|
|
852
|
+
if (!deps.shouldStop?.())
|
|
853
|
+
agentSession.activateRecoveredPrompts?.();
|
|
844
854
|
resolvedMonitorDeps.delivery = {
|
|
845
855
|
// A wake is only delivered when its turn TERMINATES successfully. A
|
|
846
856
|
// refusal or a cancellation reached the agent and was not acted on, so
|
|
@@ -1049,6 +1059,8 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
1049
1059
|
}
|
|
1050
1060
|
}
|
|
1051
1061
|
deps.log(`[${name}] up; pid=${pid} cwd=${runCwd} harness=${role.harness} session=${sessionBackend} mode=${mode}`);
|
|
1062
|
+
const healthyAt = deps.now();
|
|
1063
|
+
opts.onHealthy?.();
|
|
1052
1064
|
// Keep arrivals behind the early prime() cursor; never re-prime at startup.
|
|
1053
1065
|
// Recoverable cancellation preserves the supervisor, not monitor readiness.
|
|
1054
1066
|
monitorMayStart = true;
|
|
@@ -1256,6 +1268,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
1256
1268
|
throw new SupervisorRecycleRequiredError();
|
|
1257
1269
|
return {
|
|
1258
1270
|
elapsedSecs: elapsed, exit: exitRecord, rotated, mode, modelRecovery,
|
|
1271
|
+
healthySecs: (deps.now() - healthyAt) / 1000,
|
|
1259
1272
|
...(layoutRetiring ? { retirementReason: 'operator-stop' } : retirementReason ? { retirementReason } : {}),
|
|
1260
1273
|
};
|
|
1261
1274
|
}
|
|
@@ -1339,6 +1352,51 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
|
|
|
1339
1352
|
}
|
|
1340
1353
|
if (taskMember && (shouldStop() || !taskSupervisorMayRun(name)))
|
|
1341
1354
|
break;
|
|
1355
|
+
if (attempt === runOnce && taskMember) {
|
|
1356
|
+
// Cowork can come up after the daemon on boot. Wait read-only so a
|
|
1357
|
+
// gateway still starting its upstream does not spend the failure budget.
|
|
1358
|
+
const { waitForTaskRoomManagement } = await import('./startup-readiness.js');
|
|
1359
|
+
await waitForTaskRoomManagement(name, { shouldStop });
|
|
1360
|
+
}
|
|
1361
|
+
if (taskMember && (shouldStop() || !taskSupervisorMayRun(name)))
|
|
1362
|
+
break;
|
|
1363
|
+
const fastFailSecs = taskMember ? tempFastFailSecs(name) : fastFailSecsFor(name, opts.configPath);
|
|
1364
|
+
// The fast-fail boundary starts a recovery episode; it must not also be
|
|
1365
|
+
// the boundary that declares recovery successful. Otherwise alternating
|
|
1366
|
+
// 19s and 20s deaths erase one another forever. Require the configured
|
|
1367
|
+
// number of fast-fail windows to survive before closing an active streak.
|
|
1368
|
+
// This hysteresis stays adapter-relative (100s for the current 20s/5-attempt
|
|
1369
|
+
// policy) and still lets a genuinely sustained session reset the breaker.
|
|
1370
|
+
const stableRecoverySecs = fastFailSecs * RESTART_FAIL_THRESHOLD;
|
|
1371
|
+
// A session that stays up that long has recovered, even if it never
|
|
1372
|
+
// returns: a host reset or OOM-kill ends it without reaching the
|
|
1373
|
+
// post-attempt reset below, and a stale streak would then hold the role
|
|
1374
|
+
// down after a few transient boot failures spread over days.
|
|
1375
|
+
const streakAtStart = ledger;
|
|
1376
|
+
const clearSustainedStreak = () => {
|
|
1377
|
+
if (streakAtStart.consecutiveImmediateFailures === 0)
|
|
1378
|
+
return;
|
|
1379
|
+
const current = readRestartLedger(dir);
|
|
1380
|
+
// Never overwrite an operator reset, an open circuit or a successor's state.
|
|
1381
|
+
if (current.circuit === 'open' || current.updatedAt !== streakAtStart.updatedAt
|
|
1382
|
+
|| current.consecutiveImmediateFailures !== streakAtStart.consecutiveImmediateFailures
|
|
1383
|
+
|| readRunMarker(dir)?.startedAt !== startedAt)
|
|
1384
|
+
return;
|
|
1385
|
+
writeRestartLedger(dir, {
|
|
1386
|
+
...emptyLedger(),
|
|
1387
|
+
...carriedForward(current),
|
|
1388
|
+
lastReason: `sustained healthy session (${stableRecoverySecs}s)`,
|
|
1389
|
+
updatedAt: stamp(),
|
|
1390
|
+
});
|
|
1391
|
+
deps.log(`[${name}] healthy for ${stableRecoverySecs}s; cleared `
|
|
1392
|
+
+ `${current.consecutiveImmediateFailures} earlier immediate failure(s)`);
|
|
1393
|
+
};
|
|
1394
|
+
let cancelStabilityTimer;
|
|
1395
|
+
const onHealthy = () => {
|
|
1396
|
+
if (cancelStabilityTimer || streakAtStart.consecutiveImmediateFailures === 0)
|
|
1397
|
+
return;
|
|
1398
|
+
cancelStabilityTimer = (deps.setTimer ?? unrefTimer)(clearSustainedStreak, stableRecoverySecs * 1000);
|
|
1399
|
+
};
|
|
1342
1400
|
let result;
|
|
1343
1401
|
try {
|
|
1344
1402
|
// Service-manager boot and automatic retry bypass the operator-facing
|
|
@@ -1351,9 +1409,10 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
|
|
|
1351
1409
|
await reconcilePermanentRoleIdentities(role, undefined, deps.log);
|
|
1352
1410
|
}
|
|
1353
1411
|
result = await attempt(name, { configPath: opts.configPath, ...(taskMember ? { temp: true } : {}),
|
|
1354
|
-
allowResumeRotation: taskMember ? false : !ledger.resumeDiscarded }, deps);
|
|
1412
|
+
allowResumeRotation: taskMember ? false : !ledger.resumeDiscarded, onHealthy }, deps);
|
|
1355
1413
|
}
|
|
1356
1414
|
catch (e) {
|
|
1415
|
+
cancelStabilityTimer?.();
|
|
1357
1416
|
if (e instanceof SupervisorRecycleRequiredError) {
|
|
1358
1417
|
writeRestartLedger(dir, {
|
|
1359
1418
|
...ledger,
|
|
@@ -1373,8 +1432,14 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
|
|
|
1373
1432
|
mode: 'fresh',
|
|
1374
1433
|
};
|
|
1375
1434
|
}
|
|
1376
|
-
|
|
1435
|
+
cancelStabilityTimer?.();
|
|
1436
|
+
if (stopping || (taskMember && (shouldStop() || result.retirementReason))) {
|
|
1437
|
+
// An orderly stop of a session that was healthy long enough is a
|
|
1438
|
+
// recovery too; only the timer's race with the stop is decided here.
|
|
1439
|
+
if ((result.healthySecs ?? 0) >= stableRecoverySecs)
|
|
1440
|
+
clearSustainedStreak();
|
|
1377
1441
|
break;
|
|
1442
|
+
}
|
|
1378
1443
|
// Re-read: the attempt itself may have taken minutes, and an operator may
|
|
1379
1444
|
// have reset the ledger meanwhile.
|
|
1380
1445
|
ledger = readRestartLedger(dir);
|
|
@@ -1391,15 +1456,6 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
|
|
|
1391
1456
|
await deps.sleep(HELD_DOWN_POLL_MS);
|
|
1392
1457
|
continue;
|
|
1393
1458
|
}
|
|
1394
|
-
const fastFailSecs = taskMember ? getAdapter(loadTempRole(name).harness).exitPolicy.fastFailSecs
|
|
1395
|
-
: fastFailSecsFor(name, opts.configPath);
|
|
1396
|
-
// The fast-fail boundary starts a recovery episode; it must not also be
|
|
1397
|
-
// the boundary that declares recovery successful. Otherwise alternating
|
|
1398
|
-
// 19s and 20s deaths erase one another forever. Require the configured
|
|
1399
|
-
// number of fast-fail windows to survive before closing an active streak.
|
|
1400
|
-
// This hysteresis stays adapter-relative (100s for the current 20s/5-attempt
|
|
1401
|
-
// policy) and still lets a genuinely sustained session reset the breaker.
|
|
1402
|
-
const stableRecoverySecs = fastFailSecs * RESTART_FAIL_THRESHOLD;
|
|
1403
1459
|
const recoveryFailed = result.elapsedSecs < fastFailSecs
|
|
1404
1460
|
|| (ledger.consecutiveImmediateFailures > 0 && result.elapsedSecs < stableRecoverySecs);
|
|
1405
1461
|
if (!recoveryFailed) {
|
|
@@ -1463,6 +1519,14 @@ function fastFailSecsFor(name, configPath) {
|
|
|
1463
1519
|
return 20;
|
|
1464
1520
|
}
|
|
1465
1521
|
}
|
|
1522
|
+
function tempFastFailSecs(name) {
|
|
1523
|
+
try {
|
|
1524
|
+
return getAdapter(loadTempRole(name).harness).exitPolicy.fastFailSecs;
|
|
1525
|
+
}
|
|
1526
|
+
catch {
|
|
1527
|
+
return 20;
|
|
1528
|
+
}
|
|
1529
|
+
}
|
|
1466
1530
|
/** Temp-agent entrypoint: run once, journal why it ended, then archive its evidence. */
|
|
1467
1531
|
/** The parent uses the same adapter-relative sustained-uptime reset boundary. */
|
|
1468
1532
|
export function stableSupervisorWindow(name, temp, configPath) {
|
package/dist/session/acp.d.ts
CHANGED
|
@@ -64,6 +64,11 @@ export interface AcpSessionOptions {
|
|
|
64
64
|
stateDir: string;
|
|
65
65
|
mode: 'fresh' | 'resume';
|
|
66
66
|
requireResume?: boolean;
|
|
67
|
+
/**
|
|
68
|
+
* Hold never-started prompts recovered from a prior generation until the
|
|
69
|
+
* caller's control plane is listening; see activateRecoveredPrompts().
|
|
70
|
+
*/
|
|
71
|
+
deferRecoveredPrompts?: boolean;
|
|
67
72
|
permissions: CommonPermissions;
|
|
68
73
|
/** Native permission-mode id to request via session/set_mode; undefined keeps the agent default. */
|
|
69
74
|
modeId?: string;
|
|
@@ -159,6 +164,7 @@ export declare class AcpSession implements AgentSession {
|
|
|
159
164
|
private advertisedCommands;
|
|
160
165
|
private presentedCommands;
|
|
161
166
|
private queueDepth;
|
|
167
|
+
private readonly deferredRecovered;
|
|
162
168
|
private readonly observation;
|
|
163
169
|
private exit;
|
|
164
170
|
private steeringSupported;
|
|
@@ -209,6 +215,14 @@ export declare class AcpSession implements AgentSession {
|
|
|
209
215
|
* never silently replayed.
|
|
210
216
|
*/
|
|
211
217
|
private recoverOpenPrompts;
|
|
218
|
+
/**
|
|
219
|
+
* Dispatch recovered prompts held by deferRecoveredPrompts. A recovered turn
|
|
220
|
+
* can run tools at once, including the managed Fleet CLI, so it must not
|
|
221
|
+
* start before the supervisor's control socket accepts connections. Prompts
|
|
222
|
+
* never activated stay admitted-unstarted and recover again on next start.
|
|
223
|
+
*/
|
|
224
|
+
activateRecoveredPrompts(): void;
|
|
225
|
+
private dispatchRecovered;
|
|
212
226
|
isAlive(): boolean;
|
|
213
227
|
/**
|
|
214
228
|
* Take the occupancy lease for a turn the adapter started on its own behalf.
|
package/dist/session/acp.js
CHANGED
|
@@ -284,6 +284,7 @@ export class AcpSession {
|
|
|
284
284
|
advertisedCommands = new Set();
|
|
285
285
|
presentedCommands = [];
|
|
286
286
|
queueDepth = 0;
|
|
287
|
+
deferredRecovered = [];
|
|
287
288
|
observation = new ConversationObservation(() => this.captureObservation());
|
|
288
289
|
exit = null;
|
|
289
290
|
steeringSupported = false;
|
|
@@ -457,14 +458,29 @@ export class AcpSession {
|
|
|
457
458
|
});
|
|
458
459
|
continue;
|
|
459
460
|
}
|
|
460
|
-
const
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
void run.finally(() => { this.queueDepth = Math.max(0, this.queueDepth - 1); this.observation.publish(); });
|
|
461
|
+
const recovered = { promptId: open.promptId, text: open.text };
|
|
462
|
+
if (this.options.deferRecoveredPrompts)
|
|
463
|
+
this.deferredRecovered.push(recovered);
|
|
464
|
+
else
|
|
465
|
+
this.dispatchRecovered(recovered);
|
|
466
466
|
}
|
|
467
467
|
}
|
|
468
|
+
/**
|
|
469
|
+
* Dispatch recovered prompts held by deferRecoveredPrompts. A recovered turn
|
|
470
|
+
* can run tools at once, including the managed Fleet CLI, so it must not
|
|
471
|
+
* start before the supervisor's control socket accepts connections. Prompts
|
|
472
|
+
* never activated stay admitted-unstarted and recover again on next start.
|
|
473
|
+
*/
|
|
474
|
+
activateRecoveredPrompts() {
|
|
475
|
+
for (const recovered of this.deferredRecovered.splice(0))
|
|
476
|
+
this.dispatchRecovered(recovered);
|
|
477
|
+
}
|
|
478
|
+
dispatchRecovered({ promptId, text }) {
|
|
479
|
+
this.queueDepth++;
|
|
480
|
+
const run = this.enqueue(promptId, text, undefined);
|
|
481
|
+
this.observation.publish();
|
|
482
|
+
void run.finally(() => { this.queueDepth = Math.max(0, this.queueDepth - 1); this.observation.publish(); });
|
|
483
|
+
}
|
|
468
484
|
isAlive() {
|
|
469
485
|
// child.killed only means kill() successfully SENT a signal. A process may
|
|
470
486
|
// ignore SIGTERM and remain alive. exitCode or signalCode is the actual
|
|
@@ -13,6 +13,8 @@ export interface CodexAppServerSessionOptions {
|
|
|
13
13
|
stateDir: string;
|
|
14
14
|
mode: 'fresh' | 'resume';
|
|
15
15
|
requireResume?: boolean;
|
|
16
|
+
/** Hold recovered never-started prompts until activateRecoveredPrompts(). */
|
|
17
|
+
deferRecoveredPrompts?: boolean;
|
|
16
18
|
permissions: CommonPermissions;
|
|
17
19
|
permissionMode: NonNullable<SessionSnapshot['permissionMode']>;
|
|
18
20
|
model?: string | null;
|
|
@@ -66,6 +68,7 @@ export declare class CodexAppServerSession implements AgentSession {
|
|
|
66
68
|
private lastError?;
|
|
67
69
|
private exit;
|
|
68
70
|
private queueDepth;
|
|
71
|
+
private readonly deferredRecovered;
|
|
69
72
|
private readonly observation;
|
|
70
73
|
private promptTail;
|
|
71
74
|
/** Admitted prompts not yet handed to the app-server, in admission order. */
|
|
@@ -123,6 +126,9 @@ export declare class CodexAppServerSession implements AgentSession {
|
|
|
123
126
|
submitPromptBrowser(command: SubmitPromptCommand): Promise<PromptReceipt>;
|
|
124
127
|
private initialize;
|
|
125
128
|
private recoverOpenPrompts;
|
|
129
|
+
/** Dispatch recovered prompts once the caller's control plane is listening. */
|
|
130
|
+
activateRecoveredPrompts(): void;
|
|
131
|
+
private dispatchRecovered;
|
|
126
132
|
private admit;
|
|
127
133
|
private runPrompt;
|
|
128
134
|
private steer;
|
|
@@ -150,6 +150,7 @@ export class CodexAppServerSession {
|
|
|
150
150
|
lastError;
|
|
151
151
|
exit = null;
|
|
152
152
|
queueDepth = 0;
|
|
153
|
+
deferredRecovered = [];
|
|
153
154
|
observation = new ConversationObservation(() => this.captureObservation());
|
|
154
155
|
promptTail = Promise.resolve();
|
|
155
156
|
/** Admitted prompts not yet handed to the app-server, in admission order. */
|
|
@@ -586,15 +587,27 @@ export class CodexAppServerSession {
|
|
|
586
587
|
});
|
|
587
588
|
continue;
|
|
588
589
|
}
|
|
589
|
-
this.queueDepth++;
|
|
590
590
|
const origin = open.commandId
|
|
591
591
|
? { kind: 'owner-admin-console', commandId: open.commandId }
|
|
592
592
|
: undefined;
|
|
593
|
-
const
|
|
594
|
-
this.
|
|
595
|
-
|
|
593
|
+
const recovered = { promptId: open.promptId, text: open.text, ...(origin ? { origin } : {}) };
|
|
594
|
+
if (this.options.deferRecoveredPrompts)
|
|
595
|
+
this.deferredRecovered.push(recovered);
|
|
596
|
+
else
|
|
597
|
+
this.dispatchRecovered(recovered);
|
|
596
598
|
}
|
|
597
599
|
}
|
|
600
|
+
/** Dispatch recovered prompts once the caller's control plane is listening. */
|
|
601
|
+
activateRecoveredPrompts() {
|
|
602
|
+
for (const recovered of this.deferredRecovered.splice(0))
|
|
603
|
+
this.dispatchRecovered(recovered);
|
|
604
|
+
}
|
|
605
|
+
dispatchRecovered({ promptId, text, origin }) {
|
|
606
|
+
this.queueDepth++;
|
|
607
|
+
const run = this.enqueue(promptId, text, origin);
|
|
608
|
+
this.observation.publish();
|
|
609
|
+
void run.finally(() => { this.queueDepth = Math.max(0, this.queueDepth - 1); this.observation.publish(); });
|
|
610
|
+
}
|
|
598
611
|
admit(promptId, text, queuedBehind, options) {
|
|
599
612
|
const { source, persistBody } = sourceFor(options.origin);
|
|
600
613
|
const bytes = Buffer.byteLength(text);
|
package/dist/session/types.d.ts
CHANGED
|
@@ -304,6 +304,12 @@ export interface AgentSession {
|
|
|
304
304
|
readonly pid: number;
|
|
305
305
|
/** Live capabilities; optional only for source compatibility with external adapters. */
|
|
306
306
|
readonly capabilities?: AgentSessionCapabilities;
|
|
307
|
+
/**
|
|
308
|
+
* Start never-started prompts recovered from a prior generation. Sessions
|
|
309
|
+
* started with deferRecoveredPrompts hold them until the supervisor's
|
|
310
|
+
* control plane is listening, so recovered work cannot reach a stale socket.
|
|
311
|
+
*/
|
|
312
|
+
activateRecoveredPrompts?(): void;
|
|
307
313
|
isAlive(): boolean;
|
|
308
314
|
snapshot(): SessionSnapshot;
|
|
309
315
|
taskNoticeBinding?(): TaskNoticeBinding | undefined;
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { type DaemonGenerationProbe } from './daemon-recovery.js';
|
|
2
2
|
export declare const STARTUP_WAIT_MS = 240000;
|
|
3
3
|
export declare const STARTUP_RETRY_MS = 5000;
|
|
4
|
+
/** Upper bound for one read-only Cowork room probe during the startup wait. */
|
|
5
|
+
export declare const ROOM_PROBE_TIMEOUT_MS = 10000;
|
|
4
6
|
interface ReadinessDeps {
|
|
5
7
|
probe(env: NodeJS.ProcessEnv): Promise<DaemonGenerationProbe>;
|
|
6
8
|
now(): number;
|
|
@@ -10,4 +12,21 @@ interface ReadinessDeps {
|
|
|
10
12
|
}
|
|
11
13
|
/** Read-only preflight: never binds identities, starts a harness or edits its ledger. */
|
|
12
14
|
export declare function waitForRoleDaemon(name: string, configPath?: string, partial?: Partial<ReadinessDeps>, temp?: boolean): Promise<void>;
|
|
15
|
+
interface RoomManagementDeps {
|
|
16
|
+
/** Read-only room observation; production uses Cowork `room.show`. */
|
|
17
|
+
showRoom(roomId: string, env: NodeJS.ProcessEnv, timeoutMs: number): Promise<unknown>;
|
|
18
|
+
now(): number;
|
|
19
|
+
sleep(ms: number): Promise<void>;
|
|
20
|
+
log(message: string): void;
|
|
21
|
+
env: NodeJS.ProcessEnv;
|
|
22
|
+
shouldStop(): boolean;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Read-only preflight for a room member: wait while Cowork management is still
|
|
26
|
+
* starting behind the gateway. Only a retryable `CoworkUnavailableError` waits;
|
|
27
|
+
* success, any other error and the deadline all return so the real attempt
|
|
28
|
+
* decides, fails closed and counts. Never redeems invites, binds identities or
|
|
29
|
+
* judges room/seat identity — that remains the attempt's admission check.
|
|
30
|
+
*/
|
|
31
|
+
export declare function waitForTaskRoomManagement(name: string, partial?: Partial<RoomManagementDeps>): Promise<'ready' | 'not-required' | 'unavailable' | 'deferred'>;
|
|
13
32
|
export {};
|
|
@@ -4,6 +4,8 @@ import { resolveConfigPath, loadTempRole } from './runner.js';
|
|
|
4
4
|
import { probeDaemonGeneration } from './daemon-recovery.js';
|
|
5
5
|
export const STARTUP_WAIT_MS = 240_000;
|
|
6
6
|
export const STARTUP_RETRY_MS = 5_000;
|
|
7
|
+
/** Upper bound for one read-only Cowork room probe during the startup wait. */
|
|
8
|
+
export const ROOM_PROBE_TIMEOUT_MS = 10_000;
|
|
7
9
|
/** Read-only preflight: never binds identities, starts a harness or edits its ledger. */
|
|
8
10
|
export async function waitForRoleDaemon(name, configPath, partial = {}, temp = false) {
|
|
9
11
|
const deps = {
|
|
@@ -36,3 +38,64 @@ export async function waitForRoleDaemon(name, configPath, partial = {}, temp = f
|
|
|
36
38
|
}
|
|
37
39
|
throw new Error(`Daemon not ready (${lastReason}); service manager will retry startup`);
|
|
38
40
|
}
|
|
41
|
+
/**
|
|
42
|
+
* Read-only preflight for a room member: wait while Cowork management is still
|
|
43
|
+
* starting behind the gateway. Only a retryable `CoworkUnavailableError` waits;
|
|
44
|
+
* success, any other error and the deadline all return so the real attempt
|
|
45
|
+
* decides, fails closed and counts. Never redeems invites, binds identities or
|
|
46
|
+
* judges room/seat identity — that remains the attempt's admission check.
|
|
47
|
+
*/
|
|
48
|
+
export async function waitForTaskRoomManagement(name, partial = {}) {
|
|
49
|
+
let role;
|
|
50
|
+
// An unreadable snapshot is the attempt's failure to report, not this wait's.
|
|
51
|
+
try {
|
|
52
|
+
role = loadTempRole(name);
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
return 'deferred';
|
|
56
|
+
}
|
|
57
|
+
const startup = role.roomMemberStartup;
|
|
58
|
+
if (!startup)
|
|
59
|
+
return 'not-required';
|
|
60
|
+
const { createCoworkAdapter, CoworkUnavailableError } = await import('./rooms-tasks/cowork-adapter.js');
|
|
61
|
+
const deps = {
|
|
62
|
+
showRoom: (roomId, env, timeoutMs) => createCoworkAdapter({ env, timeoutMs }).getRoom(roomId),
|
|
63
|
+
now: Date.now,
|
|
64
|
+
sleep: ms => new Promise(resolve => setTimeout(resolve, ms)),
|
|
65
|
+
log: message => process.stderr.write(message + '\n'),
|
|
66
|
+
env: process.env,
|
|
67
|
+
shouldStop: () => false,
|
|
68
|
+
...partial,
|
|
69
|
+
};
|
|
70
|
+
const env = { ...deps.env, ...role.env };
|
|
71
|
+
const started = deps.now();
|
|
72
|
+
const deadline = started + STARTUP_WAIT_MS;
|
|
73
|
+
let waited = false;
|
|
74
|
+
for (;;) {
|
|
75
|
+
// Each probe fits in what is left of the wait, so the wait itself is bounded.
|
|
76
|
+
const timeoutMs = Math.min(ROOM_PROBE_TIMEOUT_MS, Math.max(1, deadline - deps.now()));
|
|
77
|
+
try {
|
|
78
|
+
await deps.showRoom(startup.room_id, env, timeoutMs);
|
|
79
|
+
if (waited)
|
|
80
|
+
deps.log(`[${name}] cowork room management ready after ${deps.now() - started}ms`);
|
|
81
|
+
return 'ready';
|
|
82
|
+
}
|
|
83
|
+
catch (error) {
|
|
84
|
+
if (!(error instanceof CoworkUnavailableError) || !error.retryable)
|
|
85
|
+
return 'deferred';
|
|
86
|
+
if (!waited)
|
|
87
|
+
deps.log(`[${name}] waiting for cowork room management: ${error.message}`);
|
|
88
|
+
waited = true;
|
|
89
|
+
if (deps.shouldStop())
|
|
90
|
+
return 'unavailable';
|
|
91
|
+
if (deps.now() + STARTUP_RETRY_MS >= deadline) {
|
|
92
|
+
deps.log(`[${name}] cowork room management still unavailable after ${STARTUP_WAIT_MS}ms: `
|
|
93
|
+
+ `${error.message}; starting the attempt so the restart budget applies`);
|
|
94
|
+
return 'unavailable';
|
|
95
|
+
}
|
|
96
|
+
await deps.sleep(STARTUP_RETRY_MS);
|
|
97
|
+
if (deps.shouldStop())
|
|
98
|
+
return 'unavailable';
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { type Exec } from '../exec.js';
|
|
2
|
+
/** The release tree that owns a Fleet CLI script, if it is a packaged one. */
|
|
3
|
+
export declare function releaseRoot(script: string): string | undefined;
|
|
4
|
+
export type RegistrationState =
|
|
5
|
+
/** No definition installed; refresh never creates one. */
|
|
6
|
+
'absent'
|
|
7
|
+
/** Already runs the invoking release, or a stable launcher outside any release tree. */
|
|
8
|
+
| 'current'
|
|
9
|
+
/** Pins another release tree. */
|
|
10
|
+
| 'stale'
|
|
11
|
+
/** Rewritten to the invoking release by this refresh. */
|
|
12
|
+
| 'refreshed'
|
|
13
|
+
/** Rewritten, but the service manager did not reload it; it still starts the old release. */
|
|
14
|
+
| 'reload-failed'
|
|
15
|
+
/** Not a definition Fleet generated, or overridden; left untouched. */
|
|
16
|
+
| 'unrecognized';
|
|
17
|
+
export type RunningState = 'current' | 'stale' | 'stopped' | 'unknown';
|
|
18
|
+
export interface RegistrationReport {
|
|
19
|
+
service: 'supervisor' | 'web' | 'watchdogs';
|
|
20
|
+
path: string;
|
|
21
|
+
state: RegistrationState;
|
|
22
|
+
/** Release tree the definition pinned before this call. */
|
|
23
|
+
from?: string;
|
|
24
|
+
to?: string;
|
|
25
|
+
/** What the live process runs, which a definition rewrite does not change. */
|
|
26
|
+
running: RunningState;
|
|
27
|
+
detail?: string;
|
|
28
|
+
}
|
|
29
|
+
export interface RefreshOptions {
|
|
30
|
+
/** realpath of the invoking CLI script. */
|
|
31
|
+
binPath: string;
|
|
32
|
+
/** Rewrite stale definitions; false only reports. */
|
|
33
|
+
write: boolean;
|
|
34
|
+
platform?: NodeJS.Platform;
|
|
35
|
+
homeDir?: string;
|
|
36
|
+
exec?: Exec;
|
|
37
|
+
exists?(path: string): boolean;
|
|
38
|
+
/** argv of a live process, for comparing what it runs with its definition. */
|
|
39
|
+
processArgv?(pid: number): string[] | undefined;
|
|
40
|
+
}
|
|
41
|
+
/** Report, and with `write` refresh, each installed Fleet service definition. */
|
|
42
|
+
export declare function refreshServiceRegistrations(options: RefreshOptions): Promise<RegistrationReport[]>;
|
|
43
|
+
/** One operator line per definition, including the restart a refresh cannot do. */
|
|
44
|
+
export declare function describeRegistration(report: RegistrationReport): string;
|
|
45
|
+
export declare const registrationDrift: (reports: RegistrationReport[]) => RegistrationReport[];
|