agl 43.6.0 → 43.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.md +27 -0
- package/dist_serve/bundle.js +1801 -1783
- package/dist_serve/bundle.js.map +2 -2
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/aglapi/interfaces.d.ts +35 -0
- package/dist_ts/aglapi/interfaces.js +9 -1
- package/dist_ts/classes.accountsactions.d.ts +3 -1
- package/dist_ts/classes.accountsactions.js +17 -1
- package/dist_ts/classes.apnsclient.d.ts +97 -0
- package/dist_ts/classes.apnsclient.js +363 -0
- package/dist_ts/classes.authmodels.js +19 -6
- package/dist_ts/classes.authstore.js +15 -2
- package/dist_ts/classes.cli.js +8 -1
- package/dist_ts/classes.controller.d.ts +33 -6
- package/dist_ts/classes.controller.js +254 -12
- package/dist_ts/classes.controllerdevices.d.ts +35 -7
- package/dist_ts/classes.controllerdevices.js +86 -13
- package/dist_ts/classes.controllerpush.d.ts +111 -0
- package/dist_ts/classes.controllerpush.js +299 -0
- package/dist_ts/classes.fcmclient.d.ts +78 -0
- package/dist_ts/classes.fcmclient.js +278 -0
- package/dist_ts/classes.inbox.d.ts +10 -2
- package/dist_ts/classes.inbox.js +9 -2
- package/dist_ts/classes.inferencerouter.d.ts +21 -3
- package/dist_ts/classes.inferencerouter.js +40 -17
- package/dist_ts/classes.inferenceservice.js +2 -2
- package/dist_ts/classes.mcpclient.d.ts +5 -0
- package/dist_ts/classes.mcpclient.js +18 -6
- package/dist_ts/classes.mcpcontrollerhost.d.ts +3 -0
- package/dist_ts/classes.mcpcontrollerhost.js +19 -1
- package/dist_ts/classes.mcpvalidation.js +4 -1
- package/dist_ts/classes.mcpworkercaller.d.ts +6 -0
- package/dist_ts/classes.mcpworkercaller.js +26 -4
- package/dist_ts/classes.pushdelivery.d.ts +115 -0
- package/dist_ts/classes.pushdelivery.js +311 -0
- package/dist_ts/classes.pushhandover.d.ts +46 -0
- package/dist_ts/classes.pushhandover.js +131 -0
- package/dist_ts/classes.pushregistrations.d.ts +96 -0
- package/dist_ts/classes.pushregistrations.js +263 -0
- package/dist_ts/classes.pushsettings.d.ts +125 -0
- package/dist_ts/classes.pushsettings.js +377 -0
- package/dist_ts/classes.workercallers.d.ts +73 -2
- package/dist_ts/classes.workercallers.js +354 -43
- package/dist_ts/driver/classes.directreports.d.ts +5 -0
- package/dist_ts/driver/classes.directreports.js +3 -2
- package/dist_ts/fleet/classes.workerlaunches.d.ts +6 -4
- package/dist_ts/fleet/classes.workerlaunches.js +10 -9
- package/dist_ts/fleet/classes.workers.d.ts +12 -2
- package/dist_ts/fleet/classes.workers.js +82 -2
- package/dist_ts/functions.deviceapi.d.ts +23 -3
- package/dist_ts/functions.deviceapi.js +41 -3
- package/dist_ts/functions.inferencecli.js +2 -1
- package/dist_ts/functions.pushcli.d.ts +43 -0
- package/dist_ts/functions.pushcli.js +201 -0
- package/dist_ts/functions.pushcredentials.d.ts +53 -0
- package/dist_ts/functions.pushcredentials.js +111 -0
- package/dist_ts/functions.pushwire.d.ts +8 -0
- package/dist_ts/functions.pushwire.js +83 -0
- package/dist_ts/functions.remotecli.d.ts +2 -2
- package/dist_ts/functions.remotecli.js +4 -4
- package/dist_ts/functions.workercallervalidation.js +3 -2
- package/dist_ts/functions.workerprocessproof.d.ts +12 -0
- package/dist_ts/functions.workerprocessproof.js +32 -1
- package/dist_ts/interfaces.auth.d.ts +27 -3
- package/dist_ts/interfaces.auth.js +10 -1
- package/dist_ts/interfaces.lifecycle.d.ts +7 -1
- package/dist_ts/interfaces.push.d.ts +69 -0
- package/dist_ts/interfaces.push.js +25 -0
- package/dist_ts/plugins.d.ts +5 -2
- package/dist_ts/plugins.js +7 -3
- package/dist_ts_authswitch/authority-contract.d.ts +26 -0
- package/dist_ts_authswitch/authority-contract.js +1 -1
- package/dist_ts_authswitch/authority-overview.d.ts +8 -0
- package/dist_ts_authswitch/authority-overview.js +10 -4
- package/dist_ts_authswitch/authority-placement.d.ts +16 -9
- package/dist_ts_authswitch/authority-placement.js +10 -7
- package/dist_ts_authswitch/authority-priority.d.ts +20 -0
- package/dist_ts_authswitch/authority-priority.js +21 -0
- package/dist_ts_authswitch/classes.authoritybroker.d.ts +7 -0
- package/dist_ts_authswitch/classes.authoritybroker.js +26 -2
- package/dist_ts_authswitch/classes.authoritycli.d.ts +1 -0
- package/dist_ts_authswitch/classes.authoritycli.js +43 -3
- package/dist_ts_authswitch/classes.authorityclient.d.ts +6 -1
- package/dist_ts_authswitch/classes.authorityclient.js +12 -4
- package/dist_ts_authswitch/classes.authoritydaemon.js +7 -1
- package/dist_ts_authswitch/classes.authoritydatabase.d.ts +6 -0
- package/dist_ts_authswitch/classes.authoritydatabase.js +13 -2
- package/dist_ts_authswitch/classes.authoritymodels.d.ts +7 -0
- package/dist_ts_authswitch/classes.authoritymodels.js +17 -8
- package/dist_ts_interfaces/devices.d.ts +178 -2
- package/dist_ts_interfaces/devices.js +24 -2
- package/dist_ts_interfaces/index.d.ts +1 -0
- package/dist_ts_interfaces/index.js +2 -1
- package/dist_ts_interfaces/mcprequests.d.ts +3 -2
- package/dist_ts_interfaces/mcprequests.js +3 -1
- package/dist_ts_interfaces/push.d.ts +166 -0
- package/dist_ts_interfaces/push.js +24 -0
- package/dist_ts_interfaces/requests.accounts.d.ts +19 -0
- package/dist_ts_interfaces/requests.accounts.js +1 -1
- package/dist_ts_interfaces/requests.d.ts +2 -2
- package/dist_ts_interfaces/requests.js +1 -1
- package/dist_ts_interfaces/workercallers.d.ts +43 -3
- package/docs/authswitch.md +25 -0
- package/package.json +1 -1
- package/readme.md +93 -16
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/aglapi/interfaces.ts +8 -0
- package/ts/classes.accountsactions.ts +25 -0
- package/ts/classes.apnsclient.ts +389 -0
- package/ts/classes.authmodels.ts +21 -5
- package/ts/classes.authstore.ts +14 -1
- package/ts/classes.cli.ts +7 -0
- package/ts/classes.controller.ts +277 -10
- package/ts/classes.controllerdevices.ts +94 -12
- package/ts/classes.controllerpush.ts +351 -0
- package/ts/classes.fcmclient.ts +292 -0
- package/ts/classes.inbox.ts +13 -2
- package/ts/classes.inferencerouter.ts +61 -16
- package/ts/classes.inferenceservice.ts +1 -1
- package/ts/classes.mcpclient.ts +20 -5
- package/ts/classes.mcpcontrollerhost.ts +20 -0
- package/ts/classes.mcpvalidation.ts +2 -0
- package/ts/classes.mcpworkercaller.ts +24 -2
- package/ts/classes.pushdelivery.ts +378 -0
- package/ts/classes.pushhandover.ts +98 -0
- package/ts/classes.pushregistrations.ts +244 -0
- package/ts/classes.pushsettings.ts +428 -0
- package/ts/classes.workercallers.ts +329 -37
- package/ts/driver/classes.directreports.ts +7 -1
- package/ts/fleet/classes.workerlaunches.ts +9 -7
- package/ts/fleet/classes.workers.ts +65 -2
- package/ts/functions.deviceapi.ts +52 -2
- package/ts/functions.inferencecli.ts +1 -0
- package/ts/functions.pushcli.ts +212 -0
- package/ts/functions.pushcredentials.ts +129 -0
- package/ts/functions.pushwire.ts +106 -0
- package/ts/functions.remotecli.ts +3 -2
- package/ts/functions.workercallervalidation.ts +2 -1
- package/ts/functions.workerprocessproof.ts +34 -0
- package/ts/interfaces.auth.ts +27 -3
- package/ts/interfaces.lifecycle.ts +7 -1
- package/ts/interfaces.push.ts +78 -0
- package/ts/plugins.ts +6 -1
- package/ts_authswitch/authority-contract.ts +21 -0
- package/ts_authswitch/authority-overview.ts +19 -5
- package/ts_authswitch/authority-placement.ts +18 -10
- package/ts_authswitch/authority-priority.ts +25 -0
- package/ts_authswitch/classes.authoritybroker.ts +26 -1
- package/ts_authswitch/classes.authoritycli.ts +48 -2
- package/ts_authswitch/classes.authorityclient.ts +14 -4
- package/ts_authswitch/classes.authoritydaemon.ts +8 -1
- package/ts_authswitch/classes.authoritydatabase.ts +11 -1
- package/ts_authswitch/classes.authoritymodels.ts +17 -6
- package/ts_authswitch/readme.md +1 -1
- package/ts_interfaces/devices.schema.json +327 -0
- package/ts_interfaces/devices.ts +206 -1
- package/ts_interfaces/index.ts +1 -0
- package/ts_interfaces/mcprequests.ts +3 -0
- package/ts_interfaces/push.ts +162 -0
- package/ts_interfaces/requests.accounts.ts +17 -0
- package/ts_interfaces/requests.ts +2 -0
- package/ts_interfaces/workercallers.ts +34 -2
- package/ts_web/00_commitinfo_data.ts +1 -1
- package/ts_web/accounts.plugins.ts +4 -2
- package/ts_web/classes.accountsreadmodel.ts +36 -2
- package/ts_web/elements.accounts.ts +48 -0
|
@@ -6,11 +6,13 @@ import {
|
|
|
6
6
|
type IWorkerCallerReadiness, type TWorkerCallerEnrollment, type IWorkerCallerEnrollmentResponse,
|
|
7
7
|
type IConfigurationMaster, type IConfigurationMcpStdioServer, type IControllerMcpCaller, type IControllerMcpWireCaller,
|
|
8
8
|
type IWorkerClaudeJobRequest, type TWorkerClaudeJobObservation,
|
|
9
|
+
type IWorkerClaudeProcessRequest, type TWorkerClaudeProcessIdentification,
|
|
9
10
|
} from '../ts_interfaces/index.js';
|
|
10
11
|
import { anonymousControllerMcpCaller, ControllerMcpCallerRegistry } from './classes.mcpcallerregistry.js';
|
|
11
12
|
import {
|
|
12
|
-
resolveWorkerBridgeCommand, resolveClaudeWorkerExecutable, verifyClaudeWorkerProcess,
|
|
13
|
-
readWorkerBridge, verifyWorkerBridge, workerSnapshotsEqual, WorkerCallerProofError,
|
|
13
|
+
resolveWorkerBridgeCommand, resolveClaudeWorkerExecutable, verifyClaudeWorkerProcess, verifyClaudeReleaseProcess,
|
|
14
|
+
readWorkerBridge, verifyWorkerBridge, readWorkerProcessSnapshot, workerSnapshotsEqual, WorkerCallerProofError, WorkerProcessChangedError,
|
|
15
|
+
isWorkerCallerProofRefusal,
|
|
14
16
|
type IWorkerCallerConnection, type IWorkerProcessSnapshot, type IWorkerBridgeCommand,
|
|
15
17
|
} from './functions.workerprocessproof.js';
|
|
16
18
|
import {
|
|
@@ -65,8 +67,31 @@ interface IWorkerLaunch {
|
|
|
65
67
|
* inventory shows its job still belongs to the worker, so a launch of an archived or moved worker is never taken over.
|
|
66
68
|
*/
|
|
67
69
|
verified: boolean;
|
|
70
|
+
/**
|
|
71
|
+
* Whether the binding is kept in the store: a launch the fleet prepared is. A record of a bridge identified by its
|
|
72
|
+
* process is not: its bridge presents nothing to find it by, and its process proves itself again after a restart.
|
|
73
|
+
*/
|
|
74
|
+
kept: boolean;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Why a bridge that enrolled by its process was not identified: the check that refused it (`bridge`: the approved
|
|
79
|
+
* command over its own connection; `parent`: the process it runs under directly; `inventory`: the fleet's Claude
|
|
80
|
+
* inventory; `worker`: the fleet's registry; `release`: the Claude Code release its parent runs; `record`: the worker's
|
|
81
|
+
* caller record), with the job's session and the fleet worker once the inventory and registry named them. Never a
|
|
82
|
+
* credential, a launch id or a process id.
|
|
83
|
+
*/
|
|
84
|
+
export interface IWorkerProcessRefusal {
|
|
85
|
+
check: 'bridge' | 'parent' | 'inventory' | 'worker' | 'release' | 'record';
|
|
86
|
+
reason: string;
|
|
87
|
+
workerId?: string;
|
|
88
|
+
nativeSessionId?: string;
|
|
68
89
|
}
|
|
69
90
|
|
|
91
|
+
/** The outcome of a process enrollment: the bridge's enrollment answer, a refusal, or a refusal remembered from before. */
|
|
92
|
+
type TProcessEnrollment = { kind: 'enrolled'; response: IWorkerCallerEnrollmentResponse } | { kind: 'refused'; refusal: IWorkerProcessRefusal }
|
|
93
|
+
| { kind: 'remembered' };
|
|
94
|
+
|
|
70
95
|
/**
|
|
71
96
|
* The outcome of a job re-proof: the launch is bound to the bridge's process, the bridge is refused (or was refused
|
|
72
97
|
* before and is not proved again yet), or the job ended for the worker.
|
|
@@ -88,6 +113,12 @@ export interface IControllerWorkerCallerOptions {
|
|
|
88
113
|
* Without it a bridge of a process that replaced the bound one stays pending.
|
|
89
114
|
*/
|
|
90
115
|
observeClaudeJob?(request: IWorkerClaudeJobRequest): Promise<TWorkerClaudeJobObservation>;
|
|
116
|
+
/**
|
|
117
|
+
* Which fleet worker's Claude background job runs the process a bridge without a launch runs under, from a fresh
|
|
118
|
+
* inventory of the fleet's Claude homes and its registry (`FleetWorkers.identifyCallerProcess`). Without it such a
|
|
119
|
+
* bridge stays pending.
|
|
120
|
+
*/
|
|
121
|
+
identifyClaudeProcess?(request: IWorkerClaudeProcessRequest): Promise<TWorkerClaudeProcessIdentification>;
|
|
91
122
|
/** Where the reason a worker enrollment stays pending is written once; the controller log by default. */
|
|
92
123
|
log?(line: string): void;
|
|
93
124
|
}
|
|
@@ -96,14 +127,18 @@ export interface IWorkerCallerProcessProof {
|
|
|
96
127
|
resolveCommand: typeof resolveWorkerBridgeCommand;
|
|
97
128
|
resolveClaude: typeof resolveClaudeWorkerExecutable;
|
|
98
129
|
verifyClaude: typeof verifyClaudeWorkerProcess;
|
|
130
|
+
verifyClaudeRelease: typeof verifyClaudeReleaseProcess;
|
|
99
131
|
readBridge: typeof readWorkerBridge;
|
|
100
132
|
verifyBridge: typeof verifyWorkerBridge;
|
|
133
|
+
readProcess: typeof readWorkerProcessSnapshot;
|
|
101
134
|
}
|
|
102
135
|
|
|
103
136
|
const pending = (): IWorkerCallerEnrollmentResponse => ({ callerPending: true, bindingRevision: 0 });
|
|
104
137
|
/** How long a bridge refused for a reason that may pass (an unread inventory, a job not yet in its process) waits before its next proof. */
|
|
105
138
|
const transientRefusalMs = 5_000;
|
|
106
139
|
const maximumRefusals = 64;
|
|
140
|
+
/** Bridge images and parent processes a process enrollment refused, each a key; the oldest give way. */
|
|
141
|
+
const maximumProcessRefusals = 1024;
|
|
107
142
|
const maximumLoggedReasons = 4096;
|
|
108
143
|
/** A message as a clause of a pending reason, without its final period: the log line ends the sentence. */
|
|
109
144
|
const clause = (messageArg: string): string => messageArg.replace(/\.+$/u, '');
|
|
@@ -123,13 +158,32 @@ export class ControllerWorkerCallers implements IFleetWorkerCallerHooks {
|
|
|
123
158
|
private readonly workerTails = new Map<string, Promise<unknown>>();
|
|
124
159
|
/** Pending reasons logged for launches this controller does not hold, by launch digest and reason. */
|
|
125
160
|
private readonly loggedUnknown = new Set<string>();
|
|
161
|
+
/**
|
|
162
|
+
* Process enrollments refused once the bridge was proved, until when: by the digest of the bridge's whole snapshot
|
|
163
|
+
* and by its parent's PID and start, so neither that bridge nor another under the same process is proved, or reads
|
|
164
|
+
* the inventory, again while the refusal holds. Cleared when the fleet adopts or re-binds a worker.
|
|
165
|
+
*/
|
|
166
|
+
private readonly processRefusals = new Map<string, number>();
|
|
167
|
+
/**
|
|
168
|
+
* How often the fleet adopted, re-bound or revoked a worker: an identification that ran across such a change is
|
|
169
|
+
* discarded in both directions. Its refusal is remembered only while this is what it was when the evidence for it
|
|
170
|
+
* began to be read, and it creates a worker's record only while this is what it was when the fleet began to read.
|
|
171
|
+
* One counter for every worker: which worker a running identification names is known only once it returns.
|
|
172
|
+
*/
|
|
173
|
+
private refusalEpoch = 0;
|
|
174
|
+
/**
|
|
175
|
+
* The fleet identification running for a parent process, by its PID and start, and the epoch it began in: enrollments
|
|
176
|
+
* under it wait for that one read, until the fleet adopts or re-binds a worker.
|
|
177
|
+
*/
|
|
178
|
+
private readonly identifications = new Map<string, { epoch: number; identification: Promise<TWorkerClaudeProcessIdentification> }>();
|
|
126
179
|
private closed = false;
|
|
127
180
|
private readonly proof: IWorkerCallerProcessProof;
|
|
128
181
|
private readonly log: (lineArg: string) => void;
|
|
129
182
|
|
|
130
183
|
constructor(private readonly options: IControllerWorkerCallerOptions) {
|
|
131
184
|
this.proof = options.processProof ?? { resolveCommand: resolveWorkerBridgeCommand, resolveClaude: resolveClaudeWorkerExecutable,
|
|
132
|
-
verifyClaude: verifyClaudeWorkerProcess,
|
|
185
|
+
verifyClaude: verifyClaudeWorkerProcess, verifyClaudeRelease: verifyClaudeReleaseProcess, readBridge: readWorkerBridge,
|
|
186
|
+
verifyBridge: verifyWorkerBridge, readProcess: readWorkerProcessSnapshot };
|
|
133
187
|
this.log = options.log ?? ((lineArg) => console.warn(lineArg));
|
|
134
188
|
}
|
|
135
189
|
|
|
@@ -151,7 +205,8 @@ export class ControllerWorkerCallers implements IFleetWorkerCallerHooks {
|
|
|
151
205
|
}
|
|
152
206
|
if (refusable() || this.launches.size >= 65536) throw new Error('Worker launch cannot be prepared.');
|
|
153
207
|
const record: IWorkerLaunch = { launch: structuredClone(launchArg), generation: plugins.crypto.randomBytes(24).toString('base64url'),
|
|
154
|
-
state: 'prepared', revision: 0, credentials: new Map(), leases: 0, serial: Promise.resolve(), refusals: new Map(), logged: new Set(), verified: true
|
|
208
|
+
state: 'prepared', revision: 0, credentials: new Map(), leases: 0, serial: Promise.resolve(), refusals: new Map(), logged: new Set(), verified: true,
|
|
209
|
+
kept: true };
|
|
155
210
|
this.launches.set(launchArg.launchId, record);
|
|
156
211
|
this.workers.set(launchArg.workerId, record);
|
|
157
212
|
try {
|
|
@@ -205,8 +260,11 @@ export class ControllerWorkerCallers implements IFleetWorkerCallerHooks {
|
|
|
205
260
|
const forGood = requestArg.reason !== 'generation_change';
|
|
206
261
|
if ('workerId' in requestArg) {
|
|
207
262
|
const workerId = requestArg.workerId;
|
|
208
|
-
// Marked revoked and deleted in one step of the worker's chain, so no restoration or bind of it can follow.
|
|
263
|
+
// Marked revoked and deleted in one step of the worker's chain, so no restoration or bind of it can follow. The
|
|
264
|
+
// epoch moves in the same step: a record an identification created before it is revoked here, and one that began
|
|
265
|
+
// before it and tries afterwards is not created (`enrollIdentified`), whether or not the worker had a record.
|
|
209
266
|
const current = await this.withWorker(workerId, async () => {
|
|
267
|
+
this.refusalEpoch += 1;
|
|
210
268
|
const record = this.workers.get(workerId);
|
|
211
269
|
if (record) this.markRevoked(record);
|
|
212
270
|
if (forGood) await this.options.bindings?.delete(workerId);
|
|
@@ -220,7 +278,7 @@ export class ControllerWorkerCallers implements IFleetWorkerCallerHooks {
|
|
|
220
278
|
this.markRevoked(record);
|
|
221
279
|
await this.settleRevocation(record);
|
|
222
280
|
const bindings = this.options.bindings;
|
|
223
|
-
if (forGood && bindings && record.launch.harnessId === 'claude') {
|
|
281
|
+
if (forGood && bindings && record.launch.harnessId === 'claude' && record.kept) {
|
|
224
282
|
await this.withWorker(record.launch.workerId, () => bindings.delete(record.launch.workerId, workerCallerDigest(record.launch.launchId)));
|
|
225
283
|
}
|
|
226
284
|
}
|
|
@@ -228,12 +286,23 @@ export class ControllerWorkerCallers implements IFleetWorkerCallerHooks {
|
|
|
228
286
|
public async enroll(requestArg: TWorkerCallerEnrollment, connectionArg?: IWorkerCallerConnection): Promise<IWorkerCallerEnrollmentResponse> {
|
|
229
287
|
let record: IWorkerLaunch | undefined;
|
|
230
288
|
if ('launchId' in requestArg) {
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
289
|
+
// A launch is a hint: one this controller holds as presented proves the bridge against its job. One it does not
|
|
290
|
+
// hold (unknown, replaced, of another generation) or that is revoked, also by this proof, leaves the bridge to be
|
|
291
|
+
// identified by its process chain.
|
|
292
|
+
const found = this.launches.get(requestArg.launchId) ?? await this.restore(requestArg);
|
|
293
|
+
record = typeof found === 'object' ? found : undefined;
|
|
294
|
+
if (record && record.launch.harnessId === 'claude' && record.generation === requestArg.launchGeneration && record.state !== 'revoked') {
|
|
295
|
+
const response = await this.enrollClaude(record, requestArg.bridgePid, connectionArg);
|
|
296
|
+
if (!response.callerPending || !this.revoked(record)) return response;
|
|
297
|
+
}
|
|
298
|
+
const response = await this.enrollProcess(requestArg.bridgePid, connectionArg);
|
|
299
|
+
if (response.callerPending && typeof found === 'string') this.logUnknown(workerCallerDigest(requestArg.launchId), found);
|
|
300
|
+
return response;
|
|
234
301
|
} else if ('challenge' in requestArg) {
|
|
235
302
|
record = this.challenges.get(requestArg.challenge);
|
|
236
303
|
if (!record || record.binding?.nativeSessionId !== requestArg.nativeThreadId || record.launch.harnessId !== 'codex') return pending();
|
|
304
|
+
} else if (!('nativeThreadId' in requestArg)) {
|
|
305
|
+
return this.enrollProcess(requestArg.bridgePid, connectionArg);
|
|
237
306
|
} else {
|
|
238
307
|
record = this.sessions.get(`codex:${requestArg.nativeThreadId}`);
|
|
239
308
|
if (!record || !await this.prove(record, requestArg.bridgePid, connectionArg)) return pending();
|
|
@@ -280,6 +349,13 @@ export class ControllerWorkerCallers implements IFleetWorkerCallerHooks {
|
|
|
280
349
|
});
|
|
281
350
|
}
|
|
282
351
|
|
|
352
|
+
public clearCallerRefusals(): void {
|
|
353
|
+
this.refusalEpoch += 1;
|
|
354
|
+
this.processRefusals.clear();
|
|
355
|
+
// A read that began before the adoption may name the registry as it was: the next enrollment reads afresh.
|
|
356
|
+
this.identifications.clear();
|
|
357
|
+
}
|
|
358
|
+
|
|
283
359
|
public async close(): Promise<void> {
|
|
284
360
|
this.closed = true;
|
|
285
361
|
await Promise.all([...this.launches.keys()].map(launchId => this.revokeWorkerCaller({ launchId, reason: 'generation_change' })));
|
|
@@ -373,6 +449,219 @@ export class ControllerWorkerCallers implements IFleetWorkerCallerHooks {
|
|
|
373
449
|
}
|
|
374
450
|
}
|
|
375
451
|
|
|
452
|
+
/**
|
|
453
|
+
* Enrolls a bridge that holds no launch this controller can use by its process chain (AGL-1888). Every check is
|
|
454
|
+
* required: (1) the bridge holds its own connection and runs the bridge command this controller approves now; (2) its
|
|
455
|
+
* parent, the process it runs under directly and never one further up, is a process of the controller's user; (5) the
|
|
456
|
+
* parent runs an installed Claude Code release, proved locally before anything of the fleet is read; (3) a fresh
|
|
457
|
+
* inventory of the fleet's Claude homes lists that parent exactly once, as the process of a background job, and the
|
|
458
|
+
* parent is unchanged across the read; (4) exactly one live fleet Claude worker binds that job's session. The bridge
|
|
459
|
+
* then enrolls into that worker's caller record.
|
|
460
|
+
*/
|
|
461
|
+
private async enrollProcess(bridgePidArg: number, connectionArg?: IWorkerCallerConnection): Promise<IWorkerCallerEnrollmentResponse> {
|
|
462
|
+
if (this.closed) return pending();
|
|
463
|
+
const outcome = await this.identifyProcess(bridgePidArg, connectionArg);
|
|
464
|
+
if (outcome.kind === 'enrolled') return outcome.response;
|
|
465
|
+
if (outcome.kind === 'refused') this.noteProcessRefusal(outcome.refusal);
|
|
466
|
+
return pending();
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
private async identifyProcess(bridgePidArg: number, connectionArg?: IWorkerCallerConnection): Promise<TProcessEnrollment> {
|
|
470
|
+
const refused = (refusalArg: IWorkerProcessRefusal): TProcessEnrollment => ({ kind: 'refused', refusal: refusalArg });
|
|
471
|
+
// The epoch the local proof below begins in; the fleet's identification carries the epoch it began in.
|
|
472
|
+
const started = this.refusalEpoch;
|
|
473
|
+
// The bridge's image and the process it runs under directly, as the kernel shows them; nothing is proved by them yet.
|
|
474
|
+
let image: IWorkerProcessSnapshot, parent: IWorkerProcessSnapshot;
|
|
475
|
+
try { image = await this.proof.readProcess(bridgePidArg); } catch (errorArg) {
|
|
476
|
+
if (!isWorkerCallerProofRefusal(errorArg)) throw errorArg;
|
|
477
|
+
return refused({ check: 'bridge', reason: 'its bridge is not a process of the controller\'s user' });
|
|
478
|
+
}
|
|
479
|
+
try { parent = await this.proof.readProcess(image.parentPid); } catch (errorArg) {
|
|
480
|
+
if (!isWorkerCallerProofRefusal(errorArg)) throw errorArg;
|
|
481
|
+
return refused({ check: 'parent', reason: 'its bridge does not run under a process of the controller\'s user' });
|
|
482
|
+
}
|
|
483
|
+
// A worker record bound to that process proves the bridge against it (connection, command, parent), as for a launch.
|
|
484
|
+
const held = this.recordOfProcess(parent);
|
|
485
|
+
if (held) return { kind: 'enrolled', response: await this.enrollClaude(held, bridgePidArg, connectionArg) };
|
|
486
|
+
// A refusal is remembered only for a bridge image and a parent that the checks below proved, and it only refuses: it
|
|
487
|
+
// is answered before the bridge's connection is proved again.
|
|
488
|
+
const keys = [`bridge:${workerProcessSnapshotDigest(image)}`, `process:${parent.pid}:${parent.startTicks}`];
|
|
489
|
+
if (keys.some(key => (this.processRefusals.get(key) ?? 0) > Date.now())) return { kind: 'remembered' };
|
|
490
|
+
// 1. The bridge holds its own connection and runs the command this controller approves now; a launch it presented is no proof.
|
|
491
|
+
let command: IWorkerBridgeCommand;
|
|
492
|
+
try { command = await this.currentCommand(); } catch (errorArg) {
|
|
493
|
+
return refused({ check: 'bridge', reason: `the approved bridge command cannot be resolved (${errorArg instanceof Error ? clause(errorArg.message) : 'unknown failure'})` });
|
|
494
|
+
}
|
|
495
|
+
let bridge: IWorkerProcessSnapshot;
|
|
496
|
+
try { bridge = await this.proof.readBridge(connectionArg, bridgePidArg, command, this.options.controllerPort); } catch (errorArg) {
|
|
497
|
+
if (!isWorkerCallerProofRefusal(errorArg)) throw errorArg;
|
|
498
|
+
return refused({ check: 'bridge', reason: proofReason(errorArg) });
|
|
499
|
+
}
|
|
500
|
+
// 2. Its parent is the process read above, the one it runs under directly: no ancestor further up is ever looked at.
|
|
501
|
+
if (!workerSnapshotsEqual(bridge, image)) return refused({ check: 'bridge', reason: 'its bridge process changed while it was proved' });
|
|
502
|
+
// Remembered only while no adoption, re-bind or revoke happened since the evidence for it began to be read (`epochArg`): one
|
|
503
|
+
// read before may name the registry as it was, and would refuse the session just adopted for the process's life.
|
|
504
|
+
const remember = (forGoodArg: boolean, epochArg: number): void => {
|
|
505
|
+
if (epochArg !== this.refusalEpoch) return;
|
|
506
|
+
for (const key of keys) {
|
|
507
|
+
this.processRefusals.delete(key);
|
|
508
|
+
if (this.processRefusals.size >= maximumProcessRefusals) this.processRefusals.delete(this.processRefusals.keys().next().value!);
|
|
509
|
+
this.processRefusals.set(key, forGoodArg ? Number.POSITIVE_INFINITY : Date.now() + transientRefusalMs);
|
|
510
|
+
}
|
|
511
|
+
};
|
|
512
|
+
const refuse = (refusalArg: IWorkerProcessRefusal, forGoodArg: boolean, epochArg: number): TProcessEnrollment => {
|
|
513
|
+
remember(forGoodArg, epochArg);
|
|
514
|
+
return refused(refusalArg);
|
|
515
|
+
};
|
|
516
|
+
// 5. Its parent runs an installed Claude Code release, as `reprove` requires of a job's next process: a local read of
|
|
517
|
+
// the process, before anything of the fleet is read, so a shell or a terminal the bridge runs under is refused for
|
|
518
|
+
// the process's life without an inventory read.
|
|
519
|
+
let release: string;
|
|
520
|
+
try { release = await this.proof.resolveClaude(); } catch {
|
|
521
|
+
return refuse({ check: 'release', reason: 'the approved Claude Code release cannot be resolved' }, false, started);
|
|
522
|
+
}
|
|
523
|
+
let job: IWorkerProcessSnapshot;
|
|
524
|
+
try { job = await this.proof.verifyClaudeRelease({ pid: parent.pid, startTicks: parent.startTicks }, release); } catch (errorArg) {
|
|
525
|
+
if (!isWorkerCallerProofRefusal(errorArg)) throw errorArg;
|
|
526
|
+
// A process that changed while its release was read may pass at its next read; one that runs none never does.
|
|
527
|
+
if (errorArg instanceof WorkerProcessChangedError) {
|
|
528
|
+
return refuse({ check: 'release', reason: 'the process its bridge runs under changed while its release was proved' }, false, started);
|
|
529
|
+
}
|
|
530
|
+
return refuse({ check: 'release', reason: 'the process its bridge runs under is not an installed Claude Code release' }, true, started);
|
|
531
|
+
}
|
|
532
|
+
if (!workerSnapshotsEqual(job, parent)) {
|
|
533
|
+
return refuse({ check: 'release', reason: 'the process its bridge runs under changed while its release was proved' }, false, started);
|
|
534
|
+
}
|
|
535
|
+
// 3 and 4. One fresh read of the fleet's inventory and registry for every enrollment waiting under this process.
|
|
536
|
+
const identify = this.options.identifyClaudeProcess;
|
|
537
|
+
if (!identify) return refused({ check: 'inventory', reason: 'no fleet identifies the job its bridge runs under' });
|
|
538
|
+
const { epoch, identification } = this.identifyOnce(parent, identify);
|
|
539
|
+
const identified = await identification;
|
|
540
|
+
if (identified.state === 'refused') {
|
|
541
|
+
return refuse({ check: identified.check, reason: identified.reason, ...(identified.workerId ? { workerId: identified.workerId } : {}),
|
|
542
|
+
...(identified.nativeSessionId ? { nativeSessionId: identified.nativeSessionId } : {}) }, identified.forGood, epoch);
|
|
543
|
+
}
|
|
544
|
+
const named = { workerId: identified.workerId, nativeSessionId: identified.nativeSessionId };
|
|
545
|
+
let after: IWorkerProcessSnapshot | undefined;
|
|
546
|
+
try { after = await this.proof.readProcess(parent.pid); } catch (errorArg) {
|
|
547
|
+
if (!isWorkerCallerProofRefusal(errorArg)) throw errorArg;
|
|
548
|
+
}
|
|
549
|
+
if (!after || !workerSnapshotsEqual(parent, after)) {
|
|
550
|
+
return refuse({ check: 'inventory', reason: 'the process its bridge runs under changed while the inventory was read', ...named }, false, epoch);
|
|
551
|
+
}
|
|
552
|
+
const outcome = await this.enrollIdentified(identified, epoch, job, command, bridgePidArg, connectionArg);
|
|
553
|
+
// A record that proves nothing for this bridge yet is asked again only once the short wait passed.
|
|
554
|
+
if (outcome.kind === 'refused' || outcome.response.callerPending) remember(false, epoch);
|
|
555
|
+
return outcome;
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
/**
|
|
559
|
+
* Enrolls a bridge whose process the fleet identified into its worker's caller record: the worker's current bound
|
|
560
|
+
* record proves it as for any of its bridges (`enrollClaude`); only a worker without one, or with one revoked, also by
|
|
561
|
+
* that proof, gets a record created and bound to the job's process, and only while the epoch is still `epochArg`, the
|
|
562
|
+
* one the identification began in. A record a fleet wake prepared is never replaced.
|
|
563
|
+
*/
|
|
564
|
+
private async enrollIdentified(identifiedArg: Extract<TWorkerClaudeProcessIdentification, { state: 'identified' }>, epochArg: number,
|
|
565
|
+
parentArg: IWorkerProcessSnapshot, commandArg: IWorkerBridgeCommand, bridgePidArg: number,
|
|
566
|
+
connectionArg?: IWorkerCallerConnection): Promise<Exclude<TProcessEnrollment, { kind: 'remembered' }>> {
|
|
567
|
+
const workerId = identifiedArg.workerId;
|
|
568
|
+
const named = { workerId, nativeSessionId: identifiedArg.nativeSessionId };
|
|
569
|
+
const state = (recordArg: IWorkerLaunch): IWorkerLaunch['state'] => recordArg.state;
|
|
570
|
+
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
571
|
+
const current = this.workers.get(workerId);
|
|
572
|
+
if (current && state(current) !== 'revoked') {
|
|
573
|
+
// A record bound for another bridge of the same job right now: its bind runs on the record's chain.
|
|
574
|
+
if (!current.kept && state(current) !== 'bound') await this.serialize(current, async () => undefined);
|
|
575
|
+
if (state(current) === 'prepared' || state(current) === 'quiescing') {
|
|
576
|
+
return { kind: 'refused', refusal: { check: 'record', reason: 'a fleet wake of its worker is in flight', ...named } };
|
|
577
|
+
}
|
|
578
|
+
if (state(current) === 'bound') {
|
|
579
|
+
const response = await this.enrollClaude(current, bridgePidArg, connectionArg);
|
|
580
|
+
if (!response.callerPending || !this.revoked(current)) return { kind: 'enrolled', response };
|
|
581
|
+
}
|
|
582
|
+
continue;
|
|
583
|
+
}
|
|
584
|
+
const created = await this.withWorker(workerId, async () => {
|
|
585
|
+
// The fleet adopted, re-bound or revoked a worker since its registry was read: the identification may name the
|
|
586
|
+
// worker as it was (the old session of a re-bound worker, one archived since), so it creates nothing. A revoke
|
|
587
|
+
// of this worker runs on this chain, so a record created before it is revoked by it.
|
|
588
|
+
if (epochArg !== this.refusalEpoch) return 'stale' as const;
|
|
589
|
+
const holder = this.workers.get(workerId);
|
|
590
|
+
if (this.closed || holder && holder.state !== 'revoked') return undefined;
|
|
591
|
+
for (const [launchId, record] of this.launches) {
|
|
592
|
+
if (this.launches.size < 65536) break;
|
|
593
|
+
if (record.state === 'revoked') this.launches.delete(launchId);
|
|
594
|
+
}
|
|
595
|
+
if (this.launches.size >= 65536) return undefined;
|
|
596
|
+
const launchId = `process:${plugins.crypto.randomBytes(18).toString('base64url')}`;
|
|
597
|
+
const record: IWorkerLaunch = { launch: { launchId, projectId: identifiedArg.projectId, workerId, harnessId: 'claude' },
|
|
598
|
+
generation: plugins.crypto.randomBytes(24).toString('base64url'), state: 'prepared', revision: 0, command: structuredClone(commandArg),
|
|
599
|
+
claudeExecutable: parentArg.executable, credentials: new Map(), leases: 0, serial: Promise.resolve(), refusals: new Map(),
|
|
600
|
+
logged: new Set(), verified: true, kept: false };
|
|
601
|
+
this.launches.set(launchId, record);
|
|
602
|
+
this.workers.set(workerId, record);
|
|
603
|
+
// Started on the record's chain before anyone else sees the record: another bridge of the job waits for it.
|
|
604
|
+
const binding = this.serialize(record, () => this.bind(record, { launchId, projectId: identifiedArg.projectId, harnessId: 'claude',
|
|
605
|
+
nativeSessionId: identifiedArg.nativeSessionId, expectedBindingRevision: 0, claudeJobId: identifiedArg.jobId,
|
|
606
|
+
claudeProcess: { jobId: identifiedArg.jobId, pid: parentArg.pid, startTicks: parentArg.startTicks } }));
|
|
607
|
+
return { record, binding };
|
|
608
|
+
});
|
|
609
|
+
if (created === 'stale') {
|
|
610
|
+
return { kind: 'refused', refusal: { check: 'record', reason: 'its worker changed in the fleet while its process was identified', ...named } };
|
|
611
|
+
}
|
|
612
|
+
if (!created) {
|
|
613
|
+
if (this.closed) break;
|
|
614
|
+
continue;
|
|
615
|
+
}
|
|
616
|
+
try { await created.binding; } catch (errorArg) {
|
|
617
|
+
await this.revokeWorkerCaller({ launchId: created.record.launch.launchId, reason: 'failure' });
|
|
618
|
+
return { kind: 'refused', refusal: { check: 'record', reason: `its caller record could not be bound (${errorArg instanceof Error ? clause(errorArg.message) : 'unknown failure'})`, ...named } };
|
|
619
|
+
}
|
|
620
|
+
return { kind: 'enrolled', response: await this.enrollClaude(created.record, bridgePidArg, connectionArg) };
|
|
621
|
+
}
|
|
622
|
+
return { kind: 'refused', refusal: { check: 'record', reason: 'its worker\'s caller record changed while it was enrolled', ...named } };
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
/** The current, verified record of a worker that is bound to `processArg`: the same PID and start. */
|
|
626
|
+
private recordOfProcess(processArg: IWorkerProcessSnapshot): IWorkerLaunch | undefined {
|
|
627
|
+
for (const record of this.workers.values()) {
|
|
628
|
+
if (record.state === 'bound' && record.verified && record.launch.harnessId === 'claude' && record.parent?.pid === processArg.pid
|
|
629
|
+
&& record.parent.startTicks === processArg.startTicks) return record;
|
|
630
|
+
}
|
|
631
|
+
return undefined;
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
/**
|
|
635
|
+
* The fleet's identification of the process `parentArg`, shared by every enrollment that waits for it, with the epoch
|
|
636
|
+
* it began in. One that began before the fleet's last adoption or re-bind is not shared any more.
|
|
637
|
+
*/
|
|
638
|
+
private identifyOnce(parentArg: IWorkerProcessSnapshot, identifyArg: NonNullable<IControllerWorkerCallerOptions['identifyClaudeProcess']>): { epoch: number; identification: Promise<TWorkerClaudeProcessIdentification> } {
|
|
639
|
+
const key = `${parentArg.pid}:${parentArg.startTicks}`;
|
|
640
|
+
const running = this.identifications.get(key);
|
|
641
|
+
if (running) return running;
|
|
642
|
+
const shared = { epoch: this.refusalEpoch, identification: identifyArg({ pid: parentArg.pid, parentPid: parentArg.parentPid }) };
|
|
643
|
+
this.identifications.set(key, shared);
|
|
644
|
+
const settle = () => { if (this.identifications.get(key) === shared) this.identifications.delete(key); };
|
|
645
|
+
void shared.identification.then(settle, settle);
|
|
646
|
+
return shared;
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
/**
|
|
650
|
+
* Where a process enrollment's refusal goes: once per job session or worker and reason to the controller log, never
|
|
651
|
+
* for a bridge whose parent runs no Claude job of the inventory (an AGL terminal, a harness's own server).
|
|
652
|
+
*/
|
|
653
|
+
private noteProcessRefusal(refusalArg: IWorkerProcessRefusal): void {
|
|
654
|
+
const subject = refusalArg.workerId
|
|
655
|
+
? `fleet worker ${refusalArg.workerId}${refusalArg.nativeSessionId ? ` (claude:${refusalArg.nativeSessionId})` : ''}`
|
|
656
|
+
: refusalArg.nativeSessionId ? `Claude Code job claude:${refusalArg.nativeSessionId}` : undefined;
|
|
657
|
+
if (!subject) return;
|
|
658
|
+
const key = `process:${subject}:${refusalArg.reason}`;
|
|
659
|
+
if (this.loggedUnknown.has(key)) return;
|
|
660
|
+
if (this.loggedUnknown.size >= maximumLoggedReasons) this.loggedUnknown.clear();
|
|
661
|
+
this.loggedUnknown.add(key);
|
|
662
|
+
this.log(`The MCP caller of ${subject} stays unverified: ${refusalArg.reason}.`);
|
|
663
|
+
}
|
|
664
|
+
|
|
376
665
|
/**
|
|
377
666
|
* Proves the bridge's process to be the current process of the job the launch is bound to, from a fresh inventory the
|
|
378
667
|
* fleet reads, and binds the launch to it at the current revision. A bridge refused for good is remembered for its
|
|
@@ -424,19 +713,32 @@ export class ControllerWorkerCallers implements IFleetWorkerCallerHooks {
|
|
|
424
713
|
if (observed.process.jobId !== binding.claudeJobId) return { kind: 'ended', reason: 'its session continues as another Claude job' };
|
|
425
714
|
// The inventory can still list the process Claude Code just replaced, so this refusal may pass.
|
|
426
715
|
if (observed.process.pid !== bridge.parentPid) return refuse('its bridge does not run under the process of its Claude job', false);
|
|
427
|
-
//
|
|
428
|
-
|
|
429
|
-
|
|
716
|
+
// The process the launch was proved against is still the job's: the inventory confirmed it, and it runs unchanged
|
|
717
|
+
// the Claude Code executable it was proved on. A Claude Code update since then moves what `claude` resolves to,
|
|
718
|
+
// never the image of a running process, so that process keeps its proof under either approved bridge command (AGL-1888).
|
|
719
|
+
const keptParent = record.parent, keptExecutable = record.claudeExecutable;
|
|
720
|
+
const kept = keptParent !== undefined && keptExecutable !== undefined && keptParent.pid === observed.process.pid
|
|
721
|
+
&& keptParent.startTicks === observed.process.startTicks && await this.processStands(record, keptParent);
|
|
722
|
+
// A restored launch under the bridge command it kept: nothing is bound again.
|
|
723
|
+
if (kept && sameCommand) {
|
|
430
724
|
record.verified = true;
|
|
431
725
|
return { kind: 'rebound' };
|
|
432
726
|
}
|
|
433
727
|
let executable: string, parent: IWorkerProcessSnapshot;
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
}
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
728
|
+
if (kept) {
|
|
729
|
+
executable = keptExecutable; parent = keptParent;
|
|
730
|
+
} else {
|
|
731
|
+
// Another process of the job: it must run an installed Claude Code release, in the directory of the one `claude`
|
|
732
|
+
// resolves to now; a job Claude Code continues on the release it started on is the same job (AGL-1888).
|
|
733
|
+
let release: string;
|
|
734
|
+
try { release = await this.proof.resolveClaude(); } catch {
|
|
735
|
+
return refuse('the approved Claude Code release cannot be resolved', false);
|
|
736
|
+
}
|
|
737
|
+
try { parent = await this.proof.verifyClaudeRelease(observed.process, release); } catch (errorArg) {
|
|
738
|
+
if (!isWorkerCallerProofRefusal(errorArg)) throw errorArg;
|
|
739
|
+
return refuse('the process of its Claude job is not an installed Claude Code release', false);
|
|
740
|
+
}
|
|
741
|
+
executable = parent.executable;
|
|
440
742
|
}
|
|
441
743
|
try { await this.proof.verifyBridge(connectionArg, bridge.pid, command, parent, port); } catch (errorArg) {
|
|
442
744
|
if (!isWorkerCallerProofRefusal(errorArg)) throw errorArg;
|
|
@@ -473,8 +775,9 @@ export class ControllerWorkerCallers implements IFleetWorkerCallerHooks {
|
|
|
473
775
|
* checked against its generation's, and only while it is still its worker's current launch. It is restored
|
|
474
776
|
* unverified: its bridge enrolls only after a fresh inventory shows the job still the worker's, in the kept process
|
|
475
777
|
* when that still runs unchanged, or in the bridge's process, which then proves itself as the job's next process.
|
|
778
|
+
* Otherwise the reason it cannot be restored, which is logged only when the bridge's process does not identify it.
|
|
476
779
|
*/
|
|
477
|
-
private async restore(requestArg: { launchId: string; launchGeneration: string }): Promise<IWorkerLaunch | undefined> {
|
|
780
|
+
private async restore(requestArg: { launchId: string; launchGeneration: string }): Promise<IWorkerLaunch | string | undefined> {
|
|
478
781
|
const store = this.options.bindings;
|
|
479
782
|
if (!store || this.closed) return undefined;
|
|
480
783
|
const launchDigest = workerCallerDigest(requestArg.launchId);
|
|
@@ -482,36 +785,25 @@ export class ControllerWorkerCallers implements IFleetWorkerCallerHooks {
|
|
|
482
785
|
const refusal = !found ? 'its launch is not the current launch of a fleet worker'
|
|
483
786
|
: found.generationDigest !== workerCallerDigest(requestArg.launchGeneration) ? 'its launch generation does not match'
|
|
484
787
|
: !found.binding ? 'its launch was never bound to a Claude job' : undefined;
|
|
485
|
-
if (!found || refusal)
|
|
486
|
-
this.logUnknown(launchDigest, refusal!);
|
|
487
|
-
return undefined;
|
|
488
|
-
}
|
|
788
|
+
if (!found || refusal) return refusal!;
|
|
489
789
|
return this.withWorker(found.id, async () => {
|
|
490
790
|
const kept = await store.find(launchDigest);
|
|
491
|
-
if (!kept?.binding || kept.id !== found.id || kept.generationDigest !== found.generationDigest)
|
|
492
|
-
this.logUnknown(launchDigest, 'its worker holds another launch now');
|
|
493
|
-
return undefined;
|
|
494
|
-
}
|
|
791
|
+
if (!kept?.binding || kept.id !== found.id || kept.generationDigest !== found.generationDigest) return 'its worker holds another launch now';
|
|
495
792
|
const ticketSessionId = `claude:${kept.binding.nativeSessionId}`;
|
|
496
793
|
const record: IWorkerLaunch = { launch: { launchId: requestArg.launchId, projectId: kept.projectId, workerId: kept.id, harnessId: 'claude' },
|
|
497
794
|
generation: requestArg.launchGeneration, state: 'bound', revision: kept.revision, command: structuredClone(kept.command),
|
|
498
795
|
claudeExecutable: kept.claudeExecutable, binding: { launchId: requestArg.launchId, projectId: kept.projectId, harnessId: 'claude',
|
|
499
796
|
nativeSessionId: kept.binding.nativeSessionId, expectedBindingRevision: kept.revision - 1, claudeJobId: kept.binding.jobId },
|
|
500
|
-
ticketSessionId, credentials: new Map(), leases: 0, serial: Promise.resolve(), refusals: new Map(), logged: new Set(), verified: false
|
|
797
|
+
ticketSessionId, credentials: new Map(), leases: 0, serial: Promise.resolve(), refusals: new Map(), logged: new Set(), verified: false,
|
|
798
|
+
kept: true };
|
|
501
799
|
const parent = kept.binding.process ? await this.keptProcess(record, kept.binding.process) : undefined;
|
|
502
800
|
if (parent) record.parent = parent;
|
|
503
801
|
// Decided in one step after the last await: a launch prepared or restored meanwhile wins.
|
|
504
802
|
const held = this.launches.get(requestArg.launchId);
|
|
505
803
|
if (held) return held;
|
|
506
804
|
const current = this.workers.get(kept.id), other = this.sessions.get(ticketSessionId);
|
|
507
|
-
if (this.closed || current && current.state !== 'revoked' || this.launches.size >= 65536)
|
|
508
|
-
|
|
509
|
-
return undefined;
|
|
510
|
-
}
|
|
511
|
-
if (other && other.state !== 'revoked') {
|
|
512
|
-
this.logUnknown(launchDigest, 'its session belongs to another launch');
|
|
513
|
-
return undefined;
|
|
514
|
-
}
|
|
805
|
+
if (this.closed || current && current.state !== 'revoked' || this.launches.size >= 65536) return 'its worker holds another launch now';
|
|
806
|
+
if (other && other.state !== 'revoked') return 'its session belongs to another launch';
|
|
515
807
|
this.launches.set(requestArg.launchId, record);
|
|
516
808
|
this.workers.set(kept.id, record);
|
|
517
809
|
this.sessions.set(ticketSessionId, record);
|
|
@@ -536,7 +828,7 @@ export class ControllerWorkerCallers implements IFleetWorkerCallerHooks {
|
|
|
536
828
|
private async keep(recordArg: IWorkerLaunch, revisionArg: number, bindingArg?: TWorkerCallerBindingRecord['binding'],
|
|
537
829
|
commandArg: IWorkerBridgeCommand = recordArg.command!): Promise<void> {
|
|
538
830
|
const store = this.options.bindings;
|
|
539
|
-
if (!store || recordArg.launch.harnessId !== 'claude') return;
|
|
831
|
+
if (!store || recordArg.launch.harnessId !== 'claude' || !recordArg.kept) return;
|
|
540
832
|
await this.withWorker(recordArg.launch.workerId, async () => {
|
|
541
833
|
if (this.closed || recordArg.state === 'revoked') throw new Error('The worker launch was revoked.');
|
|
542
834
|
await store.save({ id: recordArg.launch.workerId, projectId: recordArg.launch.projectId, harnessId: 'claude',
|
|
@@ -35,6 +35,11 @@ export interface IDirectReportsHost {
|
|
|
35
35
|
reportFailure?(message: string, error: unknown): void;
|
|
36
36
|
/** One of the owner's reports was written: appointed, adopted, retired, moved or given a newer result. */
|
|
37
37
|
changed?(owner: string): void;
|
|
38
|
+
/**
|
|
39
|
+
* One of the owner's reports was given a newer result (`latestResult`), the report as written. `observe` is the one
|
|
40
|
+
* writer of a result: a fleet worker's `result:` line; a report that is no fleet worker gets none.
|
|
41
|
+
*/
|
|
42
|
+
resultRecorded?(owner: string, report: IDirectReport): void;
|
|
38
43
|
}
|
|
39
44
|
|
|
40
45
|
/**
|
|
@@ -333,9 +338,10 @@ export class DirectReports implements IFleetSessionBindings {
|
|
|
333
338
|
await this.serialized(owner, report.name, async () => {
|
|
334
339
|
const current = await this.store.get(owner, report.name);
|
|
335
340
|
if (current?.state !== 'active' || event.reportId && current.latestResult?.reportId === event.reportId) return;
|
|
336
|
-
await this.store.update(owner, report.name, (next) => {
|
|
341
|
+
const written = await this.store.update(owner, report.name, (next) => {
|
|
337
342
|
next.latestResult = { text, at: event.occurredAt, ...(event.reportId ? { reportId: event.reportId } : {}) };
|
|
338
343
|
});
|
|
344
|
+
this.host.resultRecorded?.(owner, written);
|
|
339
345
|
});
|
|
340
346
|
})().catch((error: unknown) => this.host.reportFailure?.(`Direct report result of fleet worker ${workerName} was not recorded.`, error));
|
|
341
347
|
}
|
|
@@ -30,8 +30,6 @@ type TRevokeReason = Parameters<IFleetWorkerCallerHooks['revokeWorkerCaller']>[0
|
|
|
30
30
|
*/
|
|
31
31
|
export class FleetWorkerLaunches {
|
|
32
32
|
private readonly launches = new Map<string, ILaunch>();
|
|
33
|
-
/** Workers whose launch this fleet revoked and did not prepare again: the controller holds none for them to revoke. */
|
|
34
|
-
private readonly revoked = new Set<string>();
|
|
35
33
|
private readonly codexRuntimes = new Map<string, IFleetCodexCallerRuntime>();
|
|
36
34
|
constructor(private readonly options: IFleetWorkerLaunchesOptions) {}
|
|
37
35
|
/** The owner part of a worker id, `${owner}:${name}`: the launch's project. */
|
|
@@ -46,7 +44,6 @@ export class FleetWorkerLaunches {
|
|
|
46
44
|
private async prepare(worker: IFleetWorker, launch: TFleetWorkerCallerLaunch): Promise<Awaited<ReturnType<IFleetWorkerCallerHooks['prepareWorkerCaller']>>> {
|
|
47
45
|
// The controller replaces any other launch of the worker as it prepares this one, also one it kept from before a restart.
|
|
48
46
|
if (this.launches.has(worker.id)) await this.revoke(worker, 'replacement');
|
|
49
|
-
this.revoked.delete(worker.id);
|
|
50
47
|
const ready = await this.options.hooks().prepareWorkerCaller(launch);
|
|
51
48
|
this.launches.set(worker.id, { launchId: launch.launchId, projectId: launch.projectId, harnessId: launch.harnessId, bound: false,
|
|
52
49
|
...(launch.harnessId === 'codex' ? { supervisorGeneration: launch.codexOrigin.supervisorGeneration } : {}) });
|
|
@@ -129,13 +126,18 @@ export class FleetWorkerLaunches {
|
|
|
129
126
|
return runtime?.isCurrent() ? runtime : undefined;
|
|
130
127
|
}
|
|
131
128
|
/**
|
|
132
|
-
* Revokes the worker's current
|
|
133
|
-
* replaces it.
|
|
129
|
+
* Revokes the worker's current caller record for good: the worker stopped or is archived, its launch failed, or
|
|
130
|
+
* another replaces it. Every revocation reaches the controller, also when this fleet holds no launch for the worker:
|
|
131
|
+
* the controller may hold one this fleet never prepared, kept from before its restart or recorded for a bridge it
|
|
132
|
+
* identified by its process (AGL-1888), at any time a worker is live. Revoking a worker without a record is a no-op.
|
|
134
133
|
*/
|
|
135
134
|
public async revoke(worker: IFleetWorker, reason: TRevokeReason): Promise<void> {
|
|
136
|
-
|
|
135
|
+
this.launches.delete(worker.id);
|
|
137
136
|
await this.options.hooks().revokeWorkerCaller({ workerId: worker.id, reason });
|
|
138
|
-
|
|
137
|
+
}
|
|
138
|
+
/** The fleet adopted or re-bound a worker: every process refused before is identified afresh at its bridge's next call. */
|
|
139
|
+
public adopted(): void {
|
|
140
|
+
this.options.hooks().clearCallerRefusals();
|
|
139
141
|
}
|
|
140
142
|
/** Revokes the worker's launch and removes its overlay, if it has one, once the worker is archived. */
|
|
141
143
|
public async retire(worker: IFleetWorker): Promise<void> {
|