agl 43.9.0 → 43.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.md +20 -0
- package/dist_serve/bundle.js +997 -975
- package/dist_serve/bundle.js.map +2 -2
- package/dist_ts/00_commitinfo_data.js +2 -2
- package/dist_ts/aglapi/aglapisurface.document.js +3 -3
- package/dist_ts/classes.accountsactions.d.ts +3 -1
- package/dist_ts/classes.accountsactions.js +17 -1
- package/dist_ts/classes.authstore.d.ts +6 -0
- package/dist_ts/classes.authstore.js +8 -1
- package/dist_ts/classes.controller.d.ts +7 -1
- package/dist_ts/classes.controller.js +39 -7
- package/dist_ts/classes.inferencegateway.js +7 -5
- package/dist_ts/classes.inferenceplacement.d.ts +1 -1
- package/dist_ts/classes.inferencerouter.d.ts +20 -3
- package/dist_ts/classes.inferencerouter.js +37 -10
- package/dist_ts/classes.inferenceservice.js +2 -2
- package/dist_ts/classes.mcpcallerregistry.d.ts +19 -1
- package/dist_ts/classes.mcpcallerregistry.js +33 -7
- package/dist_ts/classes.terminalcallers.d.ts +40 -0
- package/dist_ts/classes.terminalcallers.js +90 -0
- package/dist_ts/classes.terminalmanager.d.ts +20 -1
- package/dist_ts/classes.terminalmanager.js +36 -2
- package/dist_ts/classes.ticketnotifier.d.ts +2 -2
- package/dist_ts/classes.ticketnotifier.js +6 -4
- package/dist_ts/classes.ticketorderspread.d.ts +25 -0
- package/dist_ts/classes.ticketorderspread.js +52 -0
- package/dist_ts/classes.ticketstore.d.ts +60 -5
- package/dist_ts/classes.ticketstore.js +176 -40
- package/dist_ts/classes.workercallers.d.ts +40 -2
- package/dist_ts/classes.workercallers.js +156 -30
- package/dist_ts/fleet/classes.boardpump.d.ts +1 -1
- package/dist_ts/fleet/classes.boardpump.js +4 -2
- package/dist_ts/fleet/classes.claudeworkers.d.ts +18 -5
- package/dist_ts/fleet/classes.claudeworkers.js +105 -31
- package/dist_ts/fleet/classes.ticketeffects.d.ts +11 -0
- package/dist_ts/fleet/classes.ticketeffects.js +28 -1
- package/dist_ts/fleet/classes.workers.d.ts +23 -4
- package/dist_ts/fleet/classes.workers.js +65 -18
- package/dist_ts/fleet/functions.workervalidation.js +7 -4
- package/dist_ts/fleet/interfaces.workers.d.ts +16 -2
- package/dist_ts/fleet/interfaces.workers.js +8 -1
- package/dist_ts/functions.fleetworkercli.d.ts +3 -1
- package/dist_ts/functions.fleetworkercli.js +6 -2
- package/dist_ts/functions.inferencecli.js +4 -1
- package/dist_ts/functions.mcpfleetworkertools.js +3 -3
- package/dist_ts/functions.proxycapacity.d.ts +2 -1
- package/dist_ts/functions.proxycapacity.js +6 -3
- package/dist_ts/functions.proxycapacitycli.js +2 -2
- package/dist_ts/functions.workerprocessproof.d.ts +6 -0
- package/dist_ts/functions.workerprocessproof.js +8 -3
- package/dist_ts/interfaces.auth.d.ts +1 -1
- package/dist_ts/interfaces.auth.js +2 -1
- package/dist_ts/interfaces.flexipc.d.ts +3 -1
- package/dist_ts/interfaces.flexipc.js +1 -1
- package/dist_ts_authswitch/authority-contract.d.ts +25 -0
- package/dist_ts_authswitch/authority-contract.js +1 -1
- package/dist_ts_authswitch/authority-overview.d.ts +7 -0
- package/dist_ts_authswitch/authority-overview.js +10 -6
- package/dist_ts_authswitch/authority-placement.d.ts +5 -2
- package/dist_ts_authswitch/authority-placement.js +6 -3
- package/dist_ts_authswitch/classes.authoritybroker.d.ts +6 -0
- package/dist_ts_authswitch/classes.authoritybroker.js +26 -1
- package/dist_ts_authswitch/classes.authoritycli.d.ts +1 -0
- package/dist_ts_authswitch/classes.authoritycli.js +50 -4
- package/dist_ts_authswitch/classes.authorityclient.d.ts +3 -1
- package/dist_ts_authswitch/classes.authorityclient.js +9 -4
- package/dist_ts_authswitch/classes.authoritydaemon.js +7 -1
- package/dist_ts_authswitch/classes.authoritydatabase.js +5 -1
- package/dist_ts_authswitch/classes.authoritymodels.d.ts +7 -0
- package/dist_ts_authswitch/classes.authoritymodels.js +13 -6
- package/dist_ts_interfaces/fleetworkers.d.ts +26 -1
- package/dist_ts_interfaces/fleetworkers.js +1 -1
- package/dist_ts_interfaces/interfaces.d.ts +4 -2
- package/dist_ts_interfaces/interfaces.js +1 -1
- package/dist_ts_interfaces/mcprequests.d.ts +2 -1
- package/dist_ts_interfaces/mcprequests.js +3 -2
- package/dist_ts_interfaces/proxycapacity.d.ts +1 -1
- package/dist_ts_interfaces/requests.accounts.d.ts +19 -0
- package/dist_ts_interfaces/requests.accounts.js +1 -1
- package/dist_ts_interfaces/requests.d.ts +2 -2
- package/dist_ts_interfaces/requests.fleetworkers.d.ts +3 -1
- package/dist_ts_interfaces/requests.js +1 -1
- package/dist_ts_migration/classes.documentmigrationrunner.d.ts +8 -1
- package/dist_ts_migration/classes.documentmigrationrunner.js +11 -2
- package/dist_ts_migration/index.d.ts +1 -0
- package/dist_ts_migration/index.js +2 -1
- package/dist_ts_migration/v41_ticketclosedcolumn.js +7 -3
- package/dist_ts_migration/v43_ticketorderstride.d.ts +54 -0
- package/dist_ts_migration/v43_ticketorderstride.js +161 -0
- package/docs/authswitch.md +22 -0
- package/package.json +1 -1
- package/readme.md +71 -4
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/aglapi/aglapisurface.document.ts +2 -2
- package/ts/classes.accountsactions.ts +25 -0
- package/ts/classes.authstore.ts +9 -0
- package/ts/classes.controller.ts +37 -3
- package/ts/classes.inferencegateway.ts +6 -4
- package/ts/classes.inferenceplacement.ts +1 -1
- package/ts/classes.inferencerouter.ts +49 -10
- package/ts/classes.inferenceservice.ts +1 -1
- package/ts/classes.mcpcallerregistry.ts +49 -7
- package/ts/classes.terminalcallers.ts +91 -0
- package/ts/classes.terminalmanager.ts +41 -1
- package/ts/classes.ticketnotifier.ts +5 -3
- package/ts/classes.ticketorderspread.ts +50 -0
- package/ts/classes.ticketstore.ts +164 -35
- package/ts/classes.workercallers.ts +183 -30
- package/ts/fleet/classes.boardpump.ts +3 -2
- package/ts/fleet/classes.claudeworkers.ts +101 -29
- package/ts/fleet/classes.ticketeffects.ts +25 -0
- package/ts/fleet/classes.workers.ts +62 -17
- package/ts/fleet/functions.workervalidation.ts +6 -3
- package/ts/fleet/interfaces.workers.ts +16 -2
- package/ts/functions.fleetworkercli.ts +5 -1
- package/ts/functions.inferencecli.ts +2 -0
- package/ts/functions.mcpfleetworkertools.ts +2 -2
- package/ts/functions.proxycapacity.ts +5 -2
- package/ts/functions.proxycapacitycli.ts +1 -1
- package/ts/functions.workerprocessproof.ts +10 -2
- package/ts/interfaces.auth.ts +1 -0
- package/ts/interfaces.flexipc.ts +3 -1
- package/ts_authswitch/authority-contract.ts +20 -0
- package/ts_authswitch/authority-overview.ts +17 -6
- package/ts_authswitch/authority-placement.ts +7 -3
- package/ts_authswitch/classes.authoritybroker.ts +24 -0
- package/ts_authswitch/classes.authoritycli.ts +58 -4
- package/ts_authswitch/classes.authorityclient.ts +11 -4
- package/ts_authswitch/classes.authoritydaemon.ts +8 -1
- package/ts_authswitch/classes.authoritydatabase.ts +3 -0
- package/ts_authswitch/classes.authoritymodels.ts +12 -4
- package/ts_interfaces/fleetworkers.ts +26 -1
- package/ts_interfaces/interfaces.ts +4 -2
- package/ts_interfaces/mcprequests.ts +2 -1
- package/ts_interfaces/proxycapacity.ts +1 -1
- package/ts_interfaces/requests.accounts.ts +17 -0
- package/ts_interfaces/requests.fleetworkers.ts +3 -1
- package/ts_interfaces/requests.ts +2 -0
- package/ts_migration/classes.documentmigrationrunner.ts +12 -1
- package/ts_migration/index.ts +1 -0
- package/ts_migration/v41_ticketclosedcolumn.ts +6 -2
- package/ts_migration/v43_ticketorderstride.ts +156 -0
- package/ts_web/00_commitinfo_data.ts +1 -1
- package/ts_web/accounts.templates.ts +12 -2
- package/ts_web/classes.accountsreadmodel.ts +42 -6
- package/ts_web/elements.accounts.ts +45 -3
- package/ts_web/elements.harnesscontrollerapp.ts +6 -1
|
@@ -5,7 +5,7 @@ import { dropClaudeHookSwitches } from '../functions.proxyharnessconfig.js';
|
|
|
5
5
|
import type { IFleetWorker, IFleetWorkerProfile, IFleetWorkerReport, IFleetWorkerTurnOutcome, IFleetRelaunchBlocker, IFleetWorkerAbsenceProof, IFleetWorkerSessionProof, IFleetWorkerRebindProof } from '../../ts_interfaces/fleetworkers.js';
|
|
6
6
|
import { controllerSessionFeedExcerpt, type IControllerMessageBundle, type IControllerNativeWait } from '../../ts_interfaces/index.js';
|
|
7
7
|
import { claudePage, claudeTranscript } from '../classes.claudeprojection.js';
|
|
8
|
-
import { FleetDispatchError, FleetNativeTurnError, boundedFleetReport, fleetClaudeNativeWait, fleetMessageDigest, fleetReportMarker, type IFleetWorkerBackend, type IFleetWorkerObservedTurn, type IFleetNativeEvent, type IFleetNativeAgent, type IFleetNativeSession, type IFleetNativePage, type IFleetNativeSendTurn, type IFleetNativeSendReceipt } from './interfaces.workers.js';
|
|
8
|
+
import { FleetDispatchError, FleetInventoryReadError, FleetNativeTurnError, boundedFleetReport, fleetClaudeNativeWait, fleetMessageDigest, fleetReportMarker, type IFleetWorkerBackend, type IFleetWorkerObservedTurn, type IFleetNativeEvent, type IFleetNativeAgent, type IFleetNativeSession, type IFleetNativePage, type IFleetNativeSendTurn, type IFleetNativeSendReceipt, type TFleetWorkerPollPass } from './interfaces.workers.js';
|
|
9
9
|
import { runFleetWorkerCommand, type IFleetWorkerCommand, type TFleetWorkerOutputWatch, type TFleetWorkerRunner } from './classes.workercommand.js';
|
|
10
10
|
import { sanitizeFailureText } from '../functions.failuredescription.js';
|
|
11
11
|
import type { FleetWorkerLaunches } from './classes.workerlaunches.js';
|
|
@@ -70,20 +70,38 @@ const status = (agent?: TClaudeAgent): IFleetWorkerReport['status'] => !agent ?
|
|
|
70
70
|
* Claude Code does not mark as its own (`isMeta`) and that holds text, not only tool results; named by its uuid, as
|
|
71
71
|
* turn endings are.
|
|
72
72
|
*/
|
|
73
|
+
const claudePromptOf = (entry: Record<string, unknown>): { id: string; startedAt?: string } | undefined => {
|
|
74
|
+
const id = entry.uuid ?? entry.timestamp;
|
|
75
|
+
if (entry.type !== 'user' || entry.isSidechain === true || entry.isMeta === true || typeof id !== 'string' || !id) return undefined;
|
|
76
|
+
const content = record.safeParse(entry.message).data?.content;
|
|
77
|
+
if (typeof content !== 'string' && !(Array.isArray(content) && content.some((part) => {
|
|
78
|
+
const type = record.safeParse(part).data?.type;
|
|
79
|
+
return typeof type === 'string' && type !== 'tool_result';
|
|
80
|
+
}))) return undefined;
|
|
81
|
+
const at = typeof entry.timestamp === 'string' ? Date.parse(entry.timestamp) : Number.NaN;
|
|
82
|
+
return { id, ...(Number.isFinite(at) ? { startedAt: new Date(at).toISOString() } : {}) };
|
|
83
|
+
};
|
|
73
84
|
const claudePrompt = (records: Record<string, unknown>[]): { id: string; startedAt?: string } | undefined => {
|
|
74
85
|
for (let index = records.length - 1; index >= 0; index -= 1) {
|
|
75
|
-
const
|
|
76
|
-
if (
|
|
77
|
-
const content = record.safeParse(entry.message).data?.content;
|
|
78
|
-
if (typeof content !== 'string' && !(Array.isArray(content) && content.some((part) => {
|
|
79
|
-
const type = record.safeParse(part).data?.type;
|
|
80
|
-
return typeof type === 'string' && type !== 'tool_result';
|
|
81
|
-
}))) continue;
|
|
82
|
-
const at = typeof entry.timestamp === 'string' ? Date.parse(entry.timestamp) : Number.NaN;
|
|
83
|
-
return { id, ...(Number.isFinite(at) ? { startedAt: new Date(at).toISOString() } : {}) };
|
|
86
|
+
const prompt = claudePromptOf(records[index]!);
|
|
87
|
+
if (prompt) return prompt;
|
|
84
88
|
}
|
|
85
89
|
return undefined;
|
|
86
90
|
};
|
|
91
|
+
/**
|
|
92
|
+
* The turn endings Claude Code recorded among transcript records, `system`/`turn_duration`, in order: each ends the turn
|
|
93
|
+
* the newest prompt before it opened, or the turn `previous` while no prompt precedes it among `records`. `record` names
|
|
94
|
+
* the ending record by its uuid.
|
|
95
|
+
*/
|
|
96
|
+
const claudeTurnEndings = (records: Record<string, unknown>[], previous: string): Array<{ turn: string; record: string }> => {
|
|
97
|
+
const endings: Array<{ turn: string; record: string }> = [];
|
|
98
|
+
let turn = previous;
|
|
99
|
+
for (const entry of records) {
|
|
100
|
+
turn = claudePromptOf(entry)?.id ?? turn;
|
|
101
|
+
if (entry.type === 'system' && entry.subtype === 'turn_duration') endings.push({ turn, record: String(entry.uuid ?? entry.timestamp) });
|
|
102
|
+
}
|
|
103
|
+
return endings;
|
|
104
|
+
};
|
|
87
105
|
/** How a Claude Code job's turn ended, as its inventory lists the job: failed, or done. */
|
|
88
106
|
const claudeOutcome = (agent: TClaudeAgent | undefined, turnId: string): IFleetWorkerTurnOutcome | undefined => (
|
|
89
107
|
agent?.state === 'failed' ? { turnId, state: 'failed' } : agent?.state === 'done' ? { turnId, state: 'completed' } : undefined);
|
|
@@ -278,17 +296,29 @@ export const fleetClaudeRelayFailure = (result: IFleetWorkerCommand): FleetDispa
|
|
|
278
296
|
return new FleetDispatchError(`Claude -p failed (exit ${result.code})${unsent ? ' before it sent any message' : ''}${output ? `: ${output}` : '.'}`, !unsent);
|
|
279
297
|
};
|
|
280
298
|
|
|
281
|
-
/**
|
|
299
|
+
/**
|
|
300
|
+
* `offset` is absent once the poll baseline was forgotten: the next poll reads afresh but keeps the observed state, turn
|
|
301
|
+
* and wait. `ended` is the newest turn end a poll published: the turn (`turn`'s id) and the transcript's ending record,
|
|
302
|
+
* absent while only the inventory showed it, so a turn's end is published once whichever shows it first. `since`, set by
|
|
303
|
+
* a start or wake, is the newest inventory read begun by then: that read and every older one cannot show the job it
|
|
304
|
+
* started or woke, so a poll never observes it from one (AGL-1970).
|
|
305
|
+
*/
|
|
282
306
|
interface IClaudeObservation {
|
|
283
307
|
offset?: number; state?: IFleetWorkerReport['status']; turnState?: string; path?: string; excerpt?: string; turn?: IFleetWorkerObservedTurn;
|
|
284
|
-
nativeWait?: IControllerNativeWait;
|
|
308
|
+
nativeWait?: IControllerNativeWait; ended?: { turn: string; record?: string }; since?: number;
|
|
285
309
|
}
|
|
310
|
+
/** One read of a profile home's inventory: when and as which read it began (`serial`, counting this backend's reads). */
|
|
311
|
+
interface IClaudeInventoryRead { at: number; serial: number; agents: Promise<TClaudeAgent[]> }
|
|
286
312
|
/** The SDK's transcript chain reader, which builds a conversation from transcript records. */
|
|
287
313
|
export type TFleetClaudeTranscriptReader = Pick<typeof plugins.claudeAgentSdk, 'getSessionMessages'>;
|
|
288
314
|
export class FleetClaudeWorkers implements IFleetWorkerBackend {
|
|
289
315
|
private readonly observations = new Map<string, IClaudeObservation>();
|
|
290
316
|
private readonly versions = new Map<string, string>();
|
|
291
|
-
private readonly inventories = new Map<string,
|
|
317
|
+
private readonly inventories = new Map<string, IClaudeInventoryRead>();
|
|
318
|
+
/** Per poll pass, the inventory read its workers share, per profile home (AGL-1970). */
|
|
319
|
+
private readonly passes = new WeakMap<TFleetWorkerPollPass, Map<string, IClaudeInventoryRead>>();
|
|
320
|
+
/** The inventory reads begun so far, which number them (`IClaudeInventoryRead.serial`). */
|
|
321
|
+
private reads = 0;
|
|
292
322
|
constructor(
|
|
293
323
|
private readonly signal: AbortSignal,
|
|
294
324
|
private readonly runner: TFleetWorkerRunner = runFleetWorkerCommand,
|
|
@@ -382,17 +412,41 @@ export class FleetClaudeWorkers implements IFleetWorkerBackend {
|
|
|
382
412
|
if (starting) await this.trustedWorkspace(profile, worker.cwd);
|
|
383
413
|
}
|
|
384
414
|
/**
|
|
385
|
-
* The profile home's inventory. The fleet reads it fresh
|
|
386
|
-
* old, the fleet's own included, and concurrent readers share one read.
|
|
415
|
+
* The profile home's inventory. The fleet reads it fresh, a poll pass once for all of its workers (`passAgents`); an
|
|
416
|
+
* outside observer accepts a read at most `maxAgeMs` old, the fleet's own included, and concurrent readers share one read.
|
|
387
417
|
*/
|
|
388
418
|
private async agents(worker: Pick<IFleetWorker, 'cwd'>, profile: Pick<IFleetWorkerProfile, 'home'>, maxAgeMs = 0): Promise<TClaudeAgent[]> {
|
|
389
|
-
const
|
|
419
|
+
const shared = this.inventories.get(profile.home ?? '');
|
|
390
420
|
if (shared && maxAgeMs > 0 && Date.now() - shared.at <= maxAgeMs) return await shared.agents;
|
|
391
|
-
|
|
421
|
+
return await this.read(worker, profile).agents;
|
|
422
|
+
}
|
|
423
|
+
/** A fresh read of the profile home's inventory, which outside observers then share for their age. */
|
|
424
|
+
private read(worker: Pick<IFleetWorker, 'cwd'>, profile: Pick<IFleetWorkerProfile, 'home'>): IClaudeInventoryRead {
|
|
425
|
+
const key = profile.home ?? '';
|
|
426
|
+
const read = { at: Date.now(), serial: ++this.reads, agents: this.command(profile, worker.cwd, ['agents', '--json', '--all'])
|
|
392
427
|
.then((output) => plugins.z.array(agentSchema).max(4096).parse(JSON.parse(output))) };
|
|
393
428
|
this.inventories.set(key, read);
|
|
394
429
|
read.agents.catch(() => { if (this.inventories.get(key) === read) this.inventories.delete(key); });
|
|
395
|
-
return
|
|
430
|
+
return read;
|
|
431
|
+
}
|
|
432
|
+
/**
|
|
433
|
+
* The inventory a poll observes from (AGL-1970): with `pass`, the read the pass's workers of the profile home share,
|
|
434
|
+
* begun once per home and pass, since it lists every job of the home; without, a fresh read. A failed read fails the
|
|
435
|
+
* pass's workers of its home alike, as a `FleetInventoryReadError`, which keeps their baselines, and is not read again
|
|
436
|
+
* in the pass; the next pass reads again.
|
|
437
|
+
*/
|
|
438
|
+
private async passAgents(pass: TFleetWorkerPollPass | undefined, worker: Pick<IFleetWorker, 'cwd'>, profile: Pick<IFleetWorkerProfile, 'home'>): Promise<{ serial: number; agents: TClaudeAgent[] }> {
|
|
439
|
+
let read: IClaudeInventoryRead;
|
|
440
|
+
if (!pass) read = this.read(worker, profile);
|
|
441
|
+
else {
|
|
442
|
+
const key = profile.home ?? '';
|
|
443
|
+
let byHome = this.passes.get(pass);
|
|
444
|
+
if (!byHome) { byHome = new Map(); this.passes.set(pass, byHome); }
|
|
445
|
+
read = byHome.get(key) ?? this.read(worker, profile);
|
|
446
|
+
byHome.set(key, read);
|
|
447
|
+
}
|
|
448
|
+
try { return { serial: read.serial, agents: await read.agents }; }
|
|
449
|
+
catch (error) { throw new FleetInventoryReadError(error instanceof Error ? error.message : 'Claude inventory read failed.', { cause: error }); }
|
|
396
450
|
}
|
|
397
451
|
public async inventory(home: string | undefined, cwd: string, maxAgeMs: number): Promise<IFleetNativeAgent[]> {
|
|
398
452
|
return await this.agents({ cwd }, { home }, maxAgeMs);
|
|
@@ -619,7 +673,7 @@ export class FleetClaudeWorkers implements IFleetWorkerBackend {
|
|
|
619
673
|
if (!observed) {
|
|
620
674
|
const observation = await options.observation(agent);
|
|
621
675
|
if (!observation) { await plugins.timersPromises.setTimeout(250, undefined, { signal: this.signal }); continue; }
|
|
622
|
-
this.observations.set(agent.sessionId, observation); observed = true;
|
|
676
|
+
this.observations.set(agent.sessionId, { ...observation, since: this.reads }); observed = true;
|
|
623
677
|
}
|
|
624
678
|
// The caller launch binds to the job's process; a job still spawning it shows no pid yet.
|
|
625
679
|
if (this.callers && !live(agent) && agent.state === 'working') { await plugins.timersPromises.setTimeout(250, undefined, { signal: this.signal }); continue; }
|
|
@@ -714,16 +768,20 @@ export class FleetClaudeWorkers implements IFleetWorkerBackend {
|
|
|
714
768
|
return undefined;
|
|
715
769
|
} finally { await file.close(); }
|
|
716
770
|
}
|
|
717
|
-
public async observe(worker: IFleetWorker, profile: IFleetWorkerProfile): Promise<IFleetWorkerReport> {
|
|
718
|
-
const agent = await this.findAgent(worker, profile), transcript = await this.transcript(worker, profile);
|
|
771
|
+
public async observe(worker: IFleetWorker, profile: IFleetWorkerProfile, pass?: TFleetWorkerPollPass): Promise<IFleetWorkerReport> {
|
|
772
|
+
const agent = await this.findAgent(worker, profile, pass ? (await this.passAgents(pass, worker, profile)).agents : undefined), transcript = await this.transcript(worker, profile);
|
|
719
773
|
// A turn may continue past its report; the newest report outranks later chatter.
|
|
720
774
|
const texts = fleetClaudeAssistantTexts(transcript.records), latest = texts.findLast((item) => fleetReportMarker.test(item.text)) ?? texts.at(-1);
|
|
721
775
|
const outcome = claudeObservedTurn(worker.nativeId!, agent, transcript.records, this.observations.get(worker.nativeId!)?.turn).outcome;
|
|
722
776
|
return { worker, status: status(agent), ...boundedFleetReport(latest?.text ?? ''), ...(latest ? { reportId: latest.id } : {}),
|
|
723
777
|
...(agent ? { turnStatus: agent.state ?? agent.status } : {}), ...(outcome ? { outcome } : {}) };
|
|
724
778
|
}
|
|
725
|
-
public async poll(worker: IFleetWorker, profile: IFleetWorkerProfile): Promise<IFleetNativeEvent[]> {
|
|
726
|
-
const
|
|
779
|
+
public async poll(worker: IFleetWorker, profile: IFleetWorkerProfile, pass?: TFleetWorkerPollPass): Promise<IFleetNativeEvent[]> {
|
|
780
|
+
const read = await this.passAgents(pass, worker, profile), observation = this.observations.get(worker.nativeId!);
|
|
781
|
+
// A start or wake set the observation after the read began, which then cannot show the job it started or woke and
|
|
782
|
+
// would end its new turn at once: the worker waits for the next pass's read, its observation untouched (AGL-1970).
|
|
783
|
+
if (observation?.since !== undefined && observation.since >= read.serial) return [];
|
|
784
|
+
const agent = await this.findAgent(worker, profile, read.agents);
|
|
727
785
|
if (!agent) throw new Error('Claude background worker is absent from its inventory; its reaped state is not proven.');
|
|
728
786
|
const current = status(agent), previous = observation?.offset === undefined ? undefined : observation;
|
|
729
787
|
const tail = await this.transcript(worker, profile, previous?.offset);
|
|
@@ -731,16 +789,28 @@ export class FleetClaudeWorkers implements IFleetWorkerBackend {
|
|
|
731
789
|
const texts = fleetClaudeAssistantTexts(tail.records);
|
|
732
790
|
const excerpt = controllerSessionFeedExcerpt(texts.at(-1)?.text) ?? observation?.excerpt;
|
|
733
791
|
const turn = claudeObservedTurn(worker.nativeId!, agent, tail.records, observation?.turn), nativeWait = fleetClaudeNativeWait(agent);
|
|
792
|
+
// A turn ends once (`ended`): by its transcript's ending record, each of which ends a turn of its own, or by the
|
|
793
|
+
// inventory as the job stops working while the transcript shows no ending, whichever a poll sees first. The record
|
|
794
|
+
// of an end the inventory showed first, and the inventory's sight of an end a record showed, publish nothing.
|
|
795
|
+
const endings = previous ? claudeTurnEndings(tail.records, observation?.turn?.id ?? worker.nativeId!) : [], ends: IFleetNativeEvent[] = [];
|
|
796
|
+
let ended = observation?.ended;
|
|
797
|
+
for (const ending of endings) {
|
|
798
|
+
const seen = ended !== undefined && ended.record === undefined && ended.turn === ending.turn;
|
|
799
|
+
ended = ending;
|
|
800
|
+
if (!seen) ends.push({ nativeId: worker.nativeId!, kind: 'worker.turn.ended', turnId: ending.record, status: current });
|
|
801
|
+
}
|
|
802
|
+
if (previous && !endings.length && previous.turnState === 'working' && agent.state !== 'working' && ended?.turn !== turn.id) {
|
|
803
|
+
ended = { turn: turn.id };
|
|
804
|
+
ends.push({ nativeId: worker.nativeId!, kind: 'worker.turn.ended', status: current });
|
|
805
|
+
}
|
|
734
806
|
this.observations.set(worker.nativeId!, { offset: tail.end, state: current, turnState: agent.state, path: tail.path, ...(excerpt ? { excerpt } : {}), turn,
|
|
735
|
-
...(nativeWait ? { nativeWait } : {}) });
|
|
807
|
+
...(nativeWait ? { nativeWait } : {}), ...(ended ? { ended } : {}) });
|
|
736
808
|
const events: IFleetNativeEvent[] = current === 'reaped' && observation?.state !== 'reaped'
|
|
737
809
|
? [{ nativeId: worker.nativeId!, kind: 'worker.reaped', status: 'reaped' }] : [];
|
|
738
810
|
if (!previous) return events;
|
|
739
811
|
events.push(...texts.filter((item) => fleetReportMarker.test(item.text)).map((item) => ({ nativeId: worker.nativeId!, kind: 'worker.report.mid_turn' as const, reportId: item.id, ...boundedFleetReport(item.text), status: current })));
|
|
740
812
|
events.push(...fleetClaudeCredentialFailures(tail.records).map((item) => ({ nativeId: worker.nativeId!, kind: 'worker.credential.failed' as const, reportId: item.id, error: item.error, ...boundedFleetReport(item.text), status: current })));
|
|
741
|
-
|
|
742
|
-
for (const entry of endings) events.push({ nativeId: worker.nativeId!, kind: 'worker.turn.ended', turnId: String(entry.uuid ?? entry.timestamp), status: current });
|
|
743
|
-
if (!endings.length && previous.turnState === 'working' && agent.state !== 'working') events.push({ nativeId: worker.nativeId!, kind: 'worker.turn.ended', status: current });
|
|
813
|
+
events.push(...ends);
|
|
744
814
|
return events;
|
|
745
815
|
}
|
|
746
816
|
/**
|
|
@@ -755,7 +825,8 @@ export class FleetClaudeWorkers implements IFleetWorkerBackend {
|
|
|
755
825
|
await this.callers?.bindClaude(worker, worker.nativeId!, agent);
|
|
756
826
|
const previous = this.observations.get(worker.nativeId!);
|
|
757
827
|
this.observations.set(worker.nativeId!, { offset: previous?.offset ?? baseline.end, path: previous?.path ?? baseline.path, state: 'busy', turnState: 'working',
|
|
758
|
-
...(previous?.excerpt ? { excerpt: previous.excerpt } : {}), ...(previous?.turn ? { turn: previous.turn } : {}) })
|
|
828
|
+
...(previous?.excerpt ? { excerpt: previous.excerpt } : {}), ...(previous?.turn ? { turn: previous.turn } : {}), ...(previous?.ended ? { ended: previous.ended } : {}),
|
|
829
|
+
since: this.reads });
|
|
759
830
|
} catch (error) {
|
|
760
831
|
await this.callers?.revoke(worker, 'failure');
|
|
761
832
|
throw new FleetDispatchError(`Claude revival was acknowledged but reconciliation failed: ${error instanceof Error ? error.message : 'unknown failure.'} Inspect before retrying.`, true, { cause: error });
|
|
@@ -975,7 +1046,8 @@ export class FleetClaudeWorkers implements IFleetWorkerBackend {
|
|
|
975
1046
|
public forget(worker: IFleetWorker): void {
|
|
976
1047
|
const observation = worker.nativeId ? this.observations.get(worker.nativeId) : undefined;
|
|
977
1048
|
if (observation) this.observations.set(worker.nativeId!, { state: observation.state, turnState: observation.turnState, ...(observation.excerpt ? { excerpt: observation.excerpt } : {}),
|
|
978
|
-
...(observation.turn ? { turn: observation.turn } : {}), ...(observation.nativeWait ? { nativeWait: observation.nativeWait } : {})
|
|
1049
|
+
...(observation.turn ? { turn: observation.turn } : {}), ...(observation.nativeWait ? { nativeWait: observation.nativeWait } : {}),
|
|
1050
|
+
...(observation.ended ? { ended: observation.ended } : {}), ...(observation.since !== undefined ? { since: observation.since } : {}) });
|
|
979
1051
|
}
|
|
980
1052
|
public observed(worker: IFleetWorker): IFleetWorkerReport['status'] | undefined {
|
|
981
1053
|
return worker.nativeId ? this.observations.get(worker.nativeId)?.state : undefined;
|
|
@@ -8,6 +8,11 @@ import { releasedEvents } from './classes.releasereconciler.js';
|
|
|
8
8
|
|
|
9
9
|
/** How often an effect whose ticket transaction met a concurrent change is tried again before it is marked failed. */
|
|
10
10
|
export const fleetTicketEffectRetries = 4;
|
|
11
|
+
/**
|
|
12
|
+
* Who writes the notice that a fleet worker's MCP calls run without its identity (AGL-1888): the controller, in the
|
|
13
|
+
* driver role. `TicketNotifier` skips it, so the notice never steers the worker it is about.
|
|
14
|
+
*/
|
|
15
|
+
export const workerIdentityActor: ITicketActor = { id: 'agl:worker-identity', name: 'AGL worker identity', role: 'driver' };
|
|
11
16
|
|
|
12
17
|
export interface IFleetTicketEffectDocument {
|
|
13
18
|
id: string; owner: string; request: TTicketFleetEvent; state: IFleetTicketEffect['state']; updateId: string;
|
|
@@ -189,6 +194,26 @@ export class FleetTicketEffects {
|
|
|
189
194
|
const { effect, commentId } = await this.dispatch(await this.prepare(owner, event, `${worker.harness}:${worker.nativeId}`), signal);
|
|
190
195
|
return { sync: this.summary([effect]), ...(commentId ? { commentId } : {}) };
|
|
191
196
|
}
|
|
197
|
+
/**
|
|
198
|
+
* Notices on the worker's ticket that its MCP calls run without its identity for `reason` (AGL-1888), once per
|
|
199
|
+
* worker, session, ticket and reason: the key and its board receipt are durable, so a notice applied before a
|
|
200
|
+
* restart is never written again. Resolves with the sync of the notice it dispatched, undefined for one applied before.
|
|
201
|
+
*/
|
|
202
|
+
public async unverifiedCaller(owner: string, worker: IFleetWorker, reason: string, signal?: AbortSignal): Promise<IFleetTicketSync | undefined> {
|
|
203
|
+
if (!worker.ticketKey || !worker.nativeId || !reason.trim()) throw new Error('A caller identity notice requires the worker\'s ticket, native session and reason.');
|
|
204
|
+
const body = `Fleet worker ${worker.name} (${worker.harness}:${worker.nativeId}) calls AGL without its identity: its MCP bridge called the `
|
|
205
|
+
+ `controller and was traced to this worker, but the controller refused to verify it: ${reason}.\n`
|
|
206
|
+
+ 'Until one of its calls is verified, its MCP calls are anonymous, so its ticket writes are refused and nothing it does is attributed to it. '
|
|
207
|
+
+ 'Nothing was sent to the worker.\n'
|
|
208
|
+
+ `What the owner can do: stop ${worker.name} and steer it again, so Claude Code continues its job in a new process whose MCP bridge proves `
|
|
209
|
+
+ 'itself at its first call; or, if its lane continues in another session, re-bind the worker onto that session (agl fleet worker adopt … --rebind). '
|
|
210
|
+
+ '`agl fleet worker list` shows the worker\'s identity as unverified until its next verified call.';
|
|
211
|
+
const identity = plugins.crypto.createHash('sha256').update(JSON.stringify([owner, worker.id, worker.ticketKey, worker.harness, worker.nativeId, reason, body])).digest('hex');
|
|
212
|
+
const prepared = await this.prepare(owner, { ticketKey: worker.ticketKey, actor: this.actor(workerIdentityActor), kind: 'comment', body,
|
|
213
|
+
idempotencyKey: `fleet:caller-identity:${identity}` });
|
|
214
|
+
if (prepared.state === 'applied') return undefined;
|
|
215
|
+
return this.summary([(await this.dispatch(prepared, signal)).effect]);
|
|
216
|
+
}
|
|
192
217
|
public async lostDispatch(owner: string, worker: IFleetWorker, signal?: AbortSignal): Promise<IFleetTicketSync> {
|
|
193
218
|
const lost = worker.lostDispatch;
|
|
194
219
|
if (!lost || !worker.ticketKey) throw new Error('Lost dispatch comment requires a durable attributed resolution and ticket.');
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import * as plugins from '../plugins.js';
|
|
2
2
|
import { controllerRuntimeIdKey, type IWorkerClaudeJobRequest, type TWorkerClaudeJobObservation, type IWorkerClaudeProcessRequest,
|
|
3
3
|
type TWorkerClaudeProcessIdentification } from '../../ts_interfaces/index.js';
|
|
4
|
-
import type { IFleetEvent, IFleetWorker, IFleetWorkerProfile, IFleetWorkerJob, IFleetNativeMessageReceipt, TFleetEventKind, TFleetWorkerAction, TFleetWorkerRequest, TFleetWorkerResponse, TFleetWorkerResponseTo, TFleetWorkerJobAction, IFleetWorkerReport, IFleetRelaunchBlocker, IFleetWorkerDispatchHolder, IFleetWorkerSummary, IFleetWorkerSummaryPage } from '../../ts_interfaces/fleetworkers.js';
|
|
4
|
+
import type { IFleetEvent, IFleetWorker, IFleetWorkerProfile, IFleetWorkerJob, IFleetNativeMessageReceipt, TFleetEventKind, TFleetWorkerAction, TFleetWorkerRequest, TFleetWorkerResponse, TFleetWorkerResponseTo, TFleetWorkerJobAction, IFleetWorkerReport, IFleetRelaunchBlocker, IFleetWorkerDispatchHolder, IFleetWorkerSummary, IFleetWorkerSummaryPage, IFleetWorkerIdentity } from '../../ts_interfaces/fleetworkers.js';
|
|
5
5
|
import type { IFleetConfiguration, IFleetOperation, IFleetTicketSync } from '../../ts_interfaces/fleet.js';
|
|
6
6
|
import type { ITicketActor } from '../../ts_interfaces/tickets.js';
|
|
7
7
|
import type { FleetTicketEffects, IFleetTicketReport } from './classes.ticketeffects.js';
|
|
@@ -11,7 +11,7 @@ import { FleetEvents } from './classes.events.js';
|
|
|
11
11
|
import { FleetClaudeWorkers } from './classes.claudeworkers.js';
|
|
12
12
|
import { FleetCodexWorkers } from './classes.codexworkers.js';
|
|
13
13
|
import type { FleetWorkerLaunches } from './classes.workerlaunches.js';
|
|
14
|
-
import { FleetDispatchError, FleetWorkerError, fleetMessageDigest, fleetReportMarker, type IFleetWorkerBackend, type IFleetWorkerObservedTurn, type IFleetNativeAgent, type IFleetNativeThread, type IFleetNativeSession, type IFleetNativePendingAttention, type IFleetNativePage, type IFleetNativeSendTurn, type IFleetNativeSendReceipt, type TFleetNativeReply } from './interfaces.workers.js';
|
|
14
|
+
import { FleetDispatchError, FleetInventoryReadError, FleetWorkerError, fleetMessageDigest, fleetReportMarker, type IFleetWorkerBackend, type IFleetWorkerObservedTurn, type IFleetNativeAgent, type IFleetNativeThread, type IFleetNativeSession, type IFleetNativePendingAttention, type IFleetNativePage, type IFleetNativeSendTurn, type IFleetNativeSendReceipt, type TFleetNativeReply, type TFleetWorkerPollPass } from './interfaces.workers.js';
|
|
15
15
|
import { assertFleetWorkerRequest, validateFleetWorkerProfiles } from './functions.workervalidation.js';
|
|
16
16
|
import { readControllerProcessIdentity, controllerProcessIdentityIsLive } from '../classes.processinspection.js';
|
|
17
17
|
import { ticketActorSchema } from '../functions.ticketvalidation.js';
|
|
@@ -52,6 +52,11 @@ export interface IFleetWorkersOptions {
|
|
|
52
52
|
claudeReleases?: (releaseArg: string) => void;
|
|
53
53
|
/** Told of each native Codex thread whose requests the default Codex backend holds for AGL came or went. */
|
|
54
54
|
onNativeCodexAttention?: (nativeIdArg: string) => void;
|
|
55
|
+
/**
|
|
56
|
+
* Whether the worker's MCP calls run without its identity, as the controller shows it (`workerCallerIdentity`); the
|
|
57
|
+
* views and lists show it. Absent, no worker shows it.
|
|
58
|
+
*/
|
|
59
|
+
callerIdentity?: (workerIdArg: string) => IFleetWorkerIdentity | undefined;
|
|
55
60
|
}
|
|
56
61
|
/** The fleet events that change what the Driver view shows of the fleet's workers. */
|
|
57
62
|
export const fleetWorkerViewEventKinds: ReadonlySet<TFleetEventKind> = new Set<TFleetEventKind>(['worker.changed', 'worker.turn.ended', 'worker.report.mid_turn', 'worker.reaped', 'worker.credential.failed']);
|
|
@@ -110,6 +115,8 @@ export class FleetWorkers {
|
|
|
110
115
|
private callerJobs: Promise<unknown> = Promise.resolve();
|
|
111
116
|
/** Set once the controller began to stop: no new worker job is admitted. */
|
|
112
117
|
private admissionClosed = false;
|
|
118
|
+
/** The notices of workers whose MCP calls run without their identity that are being written; `close` waits for them. */
|
|
119
|
+
private readonly identityNotices = new Set<Promise<void>>();
|
|
113
120
|
constructor(private readonly options: IFleetWorkersOptions = {}) {
|
|
114
121
|
this.store = options.store ?? new FleetWorkerStore(); this.events = options.events ?? new FleetEvents();
|
|
115
122
|
this.backends = options.backends ?? { claude: new FleetClaudeWorkers(this.abort.signal, undefined, undefined, options.callers, options.claudeReleases),
|
|
@@ -136,11 +143,13 @@ export class FleetWorkers {
|
|
|
136
143
|
const observable = documents.filter(({ worker }) => worker.nativeId && !worker.lostDispatch && worker.state !== 'archived' && (!worker.pending || worker.state === 'unknown'));
|
|
137
144
|
// A worker no longer observed, archived for example, keeps no remembered failure.
|
|
138
145
|
for (const id of this.observationFailures.keys()) if (!observable.some(({ worker }) => worker.id === id)) this.observationFailures.delete(id);
|
|
146
|
+
// A backend reads what all of the pass's workers share, Claude Code's inventory of a profile home, once (AGL-1970).
|
|
147
|
+
const pass: TFleetWorkerPollPass = {};
|
|
139
148
|
for (const { owner, worker } of observable) {
|
|
140
149
|
try {
|
|
141
150
|
const backend = this.backends[worker.harness], observed = backend.observed?.(worker), excerpt = backend.observedExcerpt?.(worker);
|
|
142
151
|
const turnAndWait = JSON.stringify([backend.observedTurn?.(worker) ?? null, backend.observedNativeWait?.(worker) ?? null]);
|
|
143
|
-
const events = await this.backends[worker.harness].poll(worker, worker.profile);
|
|
152
|
+
const events = await this.backends[worker.harness].poll(worker, worker.profile, pass);
|
|
144
153
|
const ticketed = this.options.ticketEffects && worker.ticketKey && worker.ticketActor;
|
|
145
154
|
for (const { nativeId: _nativeId, ...event } of events) {
|
|
146
155
|
const published = { ...event, workerId: worker.id, workerName: worker.name, ...(worker.ticketKey ? { ticketKey: worker.ticketKey } : {}) };
|
|
@@ -151,7 +160,7 @@ export class FleetWorkers {
|
|
|
151
160
|
if (ticketed && event.kind === 'worker.turn.ended') {
|
|
152
161
|
let report: IFleetWorkerReport;
|
|
153
162
|
// The turn's end is published whether or not its report can be observed.
|
|
154
|
-
try { report = await this.backends[worker.harness].observe(worker, worker.profile); }
|
|
163
|
+
try { report = await this.backends[worker.harness].observe(worker, worker.profile, pass); }
|
|
155
164
|
catch (error) { this.events.publish(published, owner); throw error; }
|
|
156
165
|
if (report.reportId && report.text.trim()) { await this.reportTicket(owner, worker, report, published); continue; }
|
|
157
166
|
}
|
|
@@ -161,7 +170,7 @@ export class FleetWorkers {
|
|
|
161
170
|
const baseline = `${worker.id}:${worker.nativeId}:${worker.ticketKey}`;
|
|
162
171
|
if (!this.reportBaselines.has(baseline)) {
|
|
163
172
|
if (!events.some(event => event.kind === 'worker.report.mid_turn' || event.kind === 'worker.turn.ended')) {
|
|
164
|
-
const report = await this.backends[worker.harness].observe(worker, worker.profile);
|
|
173
|
+
const report = await this.backends[worker.harness].observe(worker, worker.profile, pass);
|
|
165
174
|
if (report.reportId && report.text.trim() && (report.status !== 'busy' && report.status !== 'gone' || fleetReportMarker.test(report.text))) await this.reportTicket(owner, worker, report);
|
|
166
175
|
}
|
|
167
176
|
this.reportBaselines.add(baseline);
|
|
@@ -171,7 +180,7 @@ export class FleetWorkers {
|
|
|
171
180
|
if (backend.observed?.(worker) !== observed || backend.observedExcerpt?.(worker) !== excerpt
|
|
172
181
|
|| JSON.stringify([backend.observedTurn?.(worker) ?? null, backend.observedNativeWait?.(worker) ?? null]) !== turnAndWait) for (const listener of this.viewListeners) listener();
|
|
173
182
|
this.observationFailures.delete(worker.id);
|
|
174
|
-
} catch (error) { await this.observationFailed(owner, worker, error); }
|
|
183
|
+
} catch (error) { await this.observationFailed(owner, worker, error, pass); }
|
|
175
184
|
if (this.abort.signal.aborted) return;
|
|
176
185
|
}
|
|
177
186
|
if (this.options.sample && Date.now() - this.lastLoad >= 10_000) {
|
|
@@ -181,19 +190,22 @@ export class FleetWorkers {
|
|
|
181
190
|
}
|
|
182
191
|
}
|
|
183
192
|
/**
|
|
184
|
-
* A failed observation is published once, and the fleet then requests a
|
|
185
|
-
* baseline, so the next poll observes the worker afresh, and the newest report is published now. The
|
|
186
|
-
* later polls stays quiet until an observation succeeds or the failure changes.
|
|
193
|
+
* A failed observation is published once, and the fleet then requests a report itself, in the same poll pass: the
|
|
194
|
+
* backend drops its baseline, so the next poll observes the worker afresh, and the newest report is published now. The
|
|
195
|
+
* same failure on later polls stays quiet until an observation succeeds or the failure changes. A failed read of what
|
|
196
|
+
* the pass shares (`FleetInventoryReadError`) says nothing about the worker: its baseline stays, nothing is read again in
|
|
197
|
+
* the pass, and the next pass that reads observes everything since the baseline (AGL-1970).
|
|
187
198
|
*/
|
|
188
|
-
private async observationFailed(owner: string, worker: IFleetWorker, cause: unknown): Promise<void> {
|
|
199
|
+
private async observationFailed(owner: string, worker: IFleetWorker, cause: unknown, pass: TFleetWorkerPollPass): Promise<void> {
|
|
189
200
|
const error = this.error(cause), ticket = worker.ticketKey ? { ticketKey: worker.ticketKey } : {};
|
|
190
201
|
if (this.observationFailures.get(worker.id) === error) return;
|
|
191
202
|
this.observationFailures.set(worker.id, error);
|
|
192
203
|
this.events.publish({ kind: 'worker.observation.failed', workerId: worker.id, workerName: worker.name, ...ticket, error }, owner);
|
|
204
|
+
if (cause instanceof FleetInventoryReadError) return;
|
|
193
205
|
const backend = this.backends[worker.harness];
|
|
194
206
|
backend.forget?.(worker);
|
|
195
207
|
let report: IFleetWorkerReport;
|
|
196
|
-
try { report = await backend.observe(worker, worker.profile); } catch { return; }
|
|
208
|
+
try { report = await backend.observe(worker, worker.profile, pass); } catch { return; }
|
|
197
209
|
if (!report.reportId || !report.text.trim() || !fleetReportMarker.test(report.text)) return;
|
|
198
210
|
const event: Omit<IFleetEvent, 'id' | 'occurredAt'> = { kind: 'worker.report.mid_turn', workerId: worker.id, workerName: worker.name, ...ticket, reportId: report.reportId, text: report.text, truncated: report.truncated, status: report.status };
|
|
199
211
|
if (this.options.ticketEffects && worker.ticketKey && worker.ticketActor) await this.reportTicket(owner, worker, report, event);
|
|
@@ -354,7 +366,12 @@ export class FleetWorkers {
|
|
|
354
366
|
const workers = (await this.store.list(owner)).map((doc) => doc.worker);
|
|
355
367
|
const matching = workers.filter((worker) => (request.archived || worker.state !== 'archived') && (!request.ticketKey || worker.ticketKey === request.ticketKey)
|
|
356
368
|
&& (!request.state || worker.state === request.state) && (!request.harness || worker.harness === request.harness));
|
|
357
|
-
if (request.view !== 'summary')
|
|
369
|
+
if (request.view !== 'summary') {
|
|
370
|
+
return { workers: matching.map((worker) => {
|
|
371
|
+
const identity = this.options.callerIdentity?.(worker.id);
|
|
372
|
+
return { ...this.publicWorker(worker), ...(identity ? { identity } : {}) };
|
|
373
|
+
}) };
|
|
374
|
+
}
|
|
358
375
|
const filter = listFilterDigest(request);
|
|
359
376
|
matching.sort(compareListed);
|
|
360
377
|
let start = 0;
|
|
@@ -374,13 +391,13 @@ export class FleetWorkers {
|
|
|
374
391
|
return result;
|
|
375
392
|
}
|
|
376
393
|
private summary(worker: IFleetWorker): IFleetWorkerSummary {
|
|
377
|
-
const { status, turn, nativeWait } = this.observation(worker), outcome = turn?.outcome;
|
|
394
|
+
const { status, turn, nativeWait } = this.observation(worker), outcome = turn?.outcome, identity = this.options.callerIdentity?.(worker.id);
|
|
378
395
|
return { id: worker.id, name: worker.name, harness: worker.harness, profileId: worker.profileId, ...(worker.ticketKey ? { ticketKey: worker.ticketKey } : {}),
|
|
379
396
|
state: worker.state, ...(worker.nativeId ? { nativeId: worker.nativeId } : {}), cwd: worker.cwd, updatedAt: worker.updatedAt,
|
|
380
397
|
observed: { ...(status ? { status } : {}), ...(outcome ? { outcome: structuredClone(outcome) } : {}), ...(nativeWait ? { nativeWait: structuredClone(nativeWait) } : {}) },
|
|
381
398
|
fence: { ...(worker.pending ? { pending: { action: worker.pending.action, since: worker.pending.startedAt } } : {}),
|
|
382
399
|
...(worker.lostDispatch ? { lostDispatch: true } : {}), ...(worker.reconfirmedDispatch ? { reconfirmed: true } : {}),
|
|
383
|
-
...(worker.error !== undefined ? { error: true } : {}) } };
|
|
400
|
+
...(worker.error !== undefined ? { error: true } : {}) }, ...(identity ? { identity } : {}) };
|
|
384
401
|
}
|
|
385
402
|
/**
|
|
386
403
|
* The owner's job `id` once it settled, or as it is after `timeoutMs` (`timedOut`). Neither the timeout nor an
|
|
@@ -409,9 +426,10 @@ export class FleetWorkers {
|
|
|
409
426
|
return this.publicJob(job.snapshot);
|
|
410
427
|
}
|
|
411
428
|
private view(worker: IFleetWorker): IControllerFleetWorkerView {
|
|
412
|
-
const { status, turn, nativeWait, excerpt } = this.observation(worker);
|
|
429
|
+
const { status, turn, nativeWait, excerpt } = this.observation(worker), identity = this.options.callerIdentity?.(worker.id);
|
|
413
430
|
return { worker: this.publicWorker(worker), ...(status ? { status } : {}), ...(turn?.outcome ? { outcome: structuredClone(turn.outcome) } : {}),
|
|
414
|
-
...(turn?.startedAt ? { turnStartedAt: turn.startedAt } : {}), ...(nativeWait ? { nativeWait: structuredClone(nativeWait) } : {}), ...(excerpt ? { excerpt } : {})
|
|
431
|
+
...(turn?.startedAt ? { turnStartedAt: turn.startedAt } : {}), ...(nativeWait ? { nativeWait: structuredClone(nativeWait) } : {}), ...(excerpt ? { excerpt } : {}),
|
|
432
|
+
...(identity ? { identity } : {}) };
|
|
415
433
|
}
|
|
416
434
|
/**
|
|
417
435
|
* What the views show of a worker's native session, the one place a lost dispatch is applied: it is gone (archived once
|
|
@@ -514,6 +532,33 @@ export class FleetWorkers {
|
|
|
514
532
|
if ((worker.profile.home ?? '') !== home) return refused('worker', 'its fleet worker\'s profile home is not the one whose inventory lists it', true, named);
|
|
515
533
|
return { state: 'identified', workerId: worker.id, projectId: owner, nativeSessionId: agent.sessionId, jobId: agent.id };
|
|
516
534
|
}
|
|
535
|
+
/**
|
|
536
|
+
* The controller shows the worker `workerId`'s MCP calls as running without its identity, for the reason `identity`
|
|
537
|
+
* names, or as verified again (`undefined`) (AGL-1888). The views' listeners hear of it; a worker shown unverified
|
|
538
|
+
* gets one notice on its ticket for each reason the controller tells (`FleetTicketEffects.unverifiedCaller`), never
|
|
539
|
+
* again after a restart. A notice whose ticket write failed stays saved as failed or unknown, with its `ticket.failed`
|
|
540
|
+
* or `ticket.unknown` event, and is written by the next `fleet reconcile` of the ticket or when the reason is told
|
|
541
|
+
* again. Resolves once that notice is written or failed; it never rejects.
|
|
542
|
+
*/
|
|
543
|
+
public callerIdentityChanged(workerId: string, identity: IFleetWorkerIdentity | undefined): Promise<void> {
|
|
544
|
+
if (this.abort.signal.aborted) return Promise.resolve();
|
|
545
|
+
for (const listener of this.viewListeners) listener();
|
|
546
|
+
if (!identity) return Promise.resolve();
|
|
547
|
+
const notice = this.noticeCallerIdentity(workerId, identity.reason).catch((error: unknown) => {
|
|
548
|
+
if (!this.abort.signal.aborted) console.error(`The notice that fleet worker ${workerId} calls AGL without its identity was not written.`, error);
|
|
549
|
+
});
|
|
550
|
+
this.identityNotices.add(notice);
|
|
551
|
+
void notice.then(() => this.identityNotices.delete(notice));
|
|
552
|
+
return notice;
|
|
553
|
+
}
|
|
554
|
+
private async noticeCallerIdentity(workerId: string, reason: string): Promise<void> {
|
|
555
|
+
const effects = this.options.ticketEffects;
|
|
556
|
+
if (!effects) return;
|
|
557
|
+
const found = (await this.store.list()).find(({ worker }) => worker.id === workerId);
|
|
558
|
+
if (!found || found.worker.state === 'archived' || !found.worker.ticketKey || !found.worker.nativeId) return;
|
|
559
|
+
const sync = await effects.unverifiedCaller(found.owner, found.worker, reason, this.abort.signal);
|
|
560
|
+
if (sync) this.ticketEvent(found.owner, found.worker.ticketKey, sync, found.worker.id);
|
|
561
|
+
}
|
|
517
562
|
/** The newest root threads on the host's shared Codex app-server, read at most `maxAgeMs` ago. */
|
|
518
563
|
public async nativeThreads(maxAgeMs: number): Promise<IFleetNativeThread[]> {
|
|
519
564
|
return await this.backends.codex.threads?.(plugins.os.homedir(), maxAgeMs) ?? [];
|
|
@@ -1042,7 +1087,7 @@ export class FleetWorkers {
|
|
|
1042
1087
|
}
|
|
1043
1088
|
public async close(): Promise<void> {
|
|
1044
1089
|
if (this.pollTimer) clearInterval(this.pollTimer); this.abort.abort(); this.events.close(); this.viewListeners.clear();
|
|
1045
|
-
await Promise.allSettled([...this.jobs.values()].map((job) => job.completion)); await Promise.allSettled([this.polling]);
|
|
1090
|
+
await Promise.allSettled([...this.jobs.values()].map((job) => job.completion)); await Promise.allSettled([this.polling, ...this.identityNotices]);
|
|
1046
1091
|
await Promise.allSettled(Object.values(this.backends).map((backend) => backend.close()));
|
|
1047
1092
|
}
|
|
1048
1093
|
}
|
|
@@ -145,17 +145,20 @@ const event = plugins.z.object({ id: plugins.z.number().int().positive(),
|
|
|
145
145
|
ticketSync: ticketSync.optional(),
|
|
146
146
|
text: plugins.z.string().max(64 * 1024).optional(), truncated: plugins.z.boolean().optional(), error: plugins.z.string().max(4096).optional(),
|
|
147
147
|
repo: text.optional(), operationId: text.optional(), version: text.optional(), target: text.optional(), metrics: plugins.z.record(plugins.z.string(), plugins.z.unknown()).optional() }).strict();
|
|
148
|
+
/** `IFleetWorkerIdentity`: derived by the controller, never part of a stored worker. */
|
|
149
|
+
const identity = plugins.z.object({ unverified: plugins.z.literal(true), reason: plugins.z.string().min(1).max(4096), since: text }).strict();
|
|
148
150
|
const summary = plugins.z.object({ id: text, name, harness, profileId: name, ticketKey: ticketKey.optional(), state: workerState, nativeId: text.optional(), cwd, updatedAt: text,
|
|
149
151
|
observed: plugins.z.object({ status: reportStatus.optional(), outcome: outcome.optional(), nativeWait: nativeWait.optional() }).strict(),
|
|
150
152
|
fence: plugins.z.object({ pending: plugins.z.object({ action: text, since: text }).strict().optional(), lostDispatch: plugins.z.literal(true).optional(),
|
|
151
|
-
reconfirmed: plugins.z.literal(true).optional(), error: plugins.z.literal(true).optional() }).strict() }).strict();
|
|
153
|
+
reconfirmed: plugins.z.literal(true).optional(), error: plugins.z.literal(true).optional() }).strict(), identity: identity.optional() }).strict();
|
|
152
154
|
const count = plugins.z.number().int().min(0).max(1024);
|
|
153
155
|
const summaryPage = plugins.z.object({ view: plugins.z.literal('summary'), workers: plugins.z.array(summary).max(100), total: count, fenced: count, failed: count,
|
|
154
156
|
nextCursor: listCursor.optional() }).strict();
|
|
155
157
|
const view = plugins.z.object({ worker: fleetWorkerSchema, status: reportStatus.optional(), outcome: outcome.optional(), turnStartedAt: text.optional(),
|
|
156
|
-
nativeWait: nativeWait.optional(), excerpt: plugins.z.string().max(controllerSessionFeedLimits.excerptChars).optional() }).strict();
|
|
158
|
+
nativeWait: nativeWait.optional(), excerpt: plugins.z.string().max(controllerSessionFeedLimits.excerptChars).optional(), identity: identity.optional() }).strict();
|
|
159
|
+
const listedWorker = fleetWorkerSchema.safeExtend({ identity: identity.optional() });
|
|
157
160
|
export const assertFleetWorkerResponse = <T extends TFleetWorkerAction>(action: T, input: unknown): TFleetWorkerResponse<T> => {
|
|
158
|
-
const schema = action === 'list' ? plugins.z.union([plugins.z.object({ workers: plugins.z.array(
|
|
161
|
+
const schema = action === 'list' ? plugins.z.union([plugins.z.object({ workers: plugins.z.array(listedWorker).max(1024) }).strict(), summaryPage])
|
|
159
162
|
: action === 'get' ? view
|
|
160
163
|
: action === 'job-wait' ? plugins.z.object({ job, timedOut: plugins.z.boolean() }).strict()
|
|
161
164
|
: action === 'watch' ? plugins.z.object({ cursor: fleetWorkerRequestSchemas.watch.shape.cursor.unwrap(), reset: plugins.z.boolean(), events: plugins.z.array(event).max(64) }).strict()
|
|
@@ -24,6 +24,13 @@ export class FleetWorkerError extends Error {
|
|
|
24
24
|
export class FleetNativeTurnError extends Error {
|
|
25
25
|
constructor(public readonly code: 'concurrent_change' | 'turn_ended' | 'delivery_unsupported', message: string) { super(message); this.name = 'FleetNativeTurnError'; }
|
|
26
26
|
}
|
|
27
|
+
/**
|
|
28
|
+
* A poll failed because a read of native state its pass shares failed, a profile home's inventory (AGL-1970): it says
|
|
29
|
+
* nothing about the worker, so the fleet keeps the worker's baseline and the next pass observes everything since it.
|
|
30
|
+
*/
|
|
31
|
+
export class FleetInventoryReadError extends Error {
|
|
32
|
+
constructor(message: string, options?: ErrorOptions) { super(message, options); this.name = 'FleetInventoryReadError'; }
|
|
33
|
+
}
|
|
27
34
|
/**
|
|
28
35
|
* How a native send treats the turn the session runs. With `expectedTurnId` the text steers only that running turn. An
|
|
29
36
|
* `agentMessage` steers a turn another client runs only when `expectedTurnId` names it, as an agent message must.
|
|
@@ -80,6 +87,11 @@ export interface IFleetNativePendingAttention {
|
|
|
80
87
|
export interface IFleetNativeSession { nativeId: string; cwd: string; home?: string; jobDelivery?: TControllerNativeJobDelivery; peerHomes?: string[] }
|
|
81
88
|
/** A transcript page read-only: `nextCursor` pages to older bundles, and `earlier` says older bundles exist at all. */
|
|
82
89
|
export interface IFleetNativePage { bundles: IControllerMessageBundle[]; earlier: boolean; nextCursor?: string }
|
|
90
|
+
/**
|
|
91
|
+
* One pass of the fleet's poll over its observable workers, a new object for each pass. A backend may read native state
|
|
92
|
+
* that all of the pass's workers share once per pass, keyed by this object, and never keeps it beyond the pass (AGL-1970).
|
|
93
|
+
*/
|
|
94
|
+
export type TFleetWorkerPollPass = object;
|
|
83
95
|
export interface IFleetWorkerBackend {
|
|
84
96
|
/**
|
|
85
97
|
* Fresh native absence proof; failures and ambiguous state must throw. Sends nothing.
|
|
@@ -106,8 +118,10 @@ export interface IFleetWorkerBackend {
|
|
|
106
118
|
relaunch?(worker: IFleetWorker, profile: IFleetWorkerProfile, nativeId: string, prompt: string, bind: (nativeId: string) => Promise<void>): Promise<void>;
|
|
107
119
|
/** Why `relaunch` of `nativeId` as a worker in `cwd` would refuse now, by the same checks, read-only; `undefined` when none does. */
|
|
108
120
|
relaunchBlocker?(cwd: string, profile: IFleetWorkerProfile, nativeId: string): Promise<IFleetRelaunchBlocker | undefined>;
|
|
109
|
-
|
|
110
|
-
|
|
121
|
+
/** Observes the worker since its last poll; `pass` is the poll pass it is observed in, absent for a read of its own. */
|
|
122
|
+
poll(worker: IFleetWorker, profile: IFleetWorkerProfile, pass?: TFleetWorkerPollPass): Promise<IFleetNativeEvent[]>;
|
|
123
|
+
/** The worker's newest report; `pass` is the poll pass that asks for it, absent for a read of its own. */
|
|
124
|
+
observe(worker: IFleetWorker, profile: IFleetWorkerProfile, pass?: TFleetWorkerPollPass): Promise<IFleetWorkerReport>;
|
|
111
125
|
send(messages: Array<{ worker: IFleetWorker; profile: IFleetWorkerProfile; text: string }>): Promise<void>;
|
|
112
126
|
stop(worker: IFleetWorker, profile: IFleetWorkerProfile): Promise<void>;
|
|
113
127
|
archive(worker: IFleetWorker, profile: IFleetWorkerProfile): Promise<void>;
|
|
@@ -26,6 +26,8 @@ export const fleetWorkerCliHelp = `Usage: agl fleet worker start NAME --profile
|
|
|
26
26
|
[--gitea URL] [--gitea-token-file FILE] [--probe-interval SECONDS]
|
|
27
27
|
All accept --ticket KEY, --json and --port PORT. Actions and report require --ticket, except message; list/status/watch accept it as a filter.
|
|
28
28
|
Profiles live in fleet.workerProfiles in the Configuration master.
|
|
29
|
+
list prints the workers' records; a worker whose MCP calls the controller refused its identity carries
|
|
30
|
+
identity {unverified, reason, since} until its next verified call.
|
|
29
31
|
Mutation and report commands wait for the returned job; MCP tools return its handle immediately.
|
|
30
32
|
refresh --profile ID starts the successor on that configured profile, which must run the worker's own harness (only move
|
|
31
33
|
changes the harness); without --profile the successor keeps the worker's recorded profile.
|
|
@@ -185,6 +187,8 @@ export const parseFleetWorkerCliArguments = async (args: string[]): Promise<IFle
|
|
|
185
187
|
return { action, request, port, json, once, ...(lines ? { lines } : {}), ...(probing ? { probes } : {}),
|
|
186
188
|
...(message ? { target: (request as TFleetWorkerRequest<'steer'>).messages[0]!.name, ...(messageFile ? { messageFile } : {}) } : {}) };
|
|
187
189
|
};
|
|
190
|
+
/** What a worker command other than watch prints: its result as indented JSON, or as one line with --json. */
|
|
191
|
+
export const fleetWorkerCliOutput = (resultArg: unknown, jsonArg: boolean): string => `${JSON.stringify(resultArg, null, jsonArg ? undefined : 2)}\n`;
|
|
188
192
|
/** The line a continuing watch writes for each failed attempt to reach the controller. */
|
|
189
193
|
export interface IFleetWatchDisconnection { disconnected: { at: string; error: string; retryInMs: number } }
|
|
190
194
|
export type { IFleetWatchChange };
|
|
@@ -257,7 +261,7 @@ export const runFleetWorkerCli = async (args: string[]): Promise<void> => {
|
|
|
257
261
|
}
|
|
258
262
|
} catch (cause) { throw new Error(`Observation ended; inspect fleet worker status ${id} before retrying.`, { cause }); }
|
|
259
263
|
}
|
|
260
|
-
process.stdout.write(
|
|
264
|
+
process.stdout.write(fleetWorkerCliOutput(result, parsed.json));
|
|
261
265
|
if ('job' in result && result.job.state === 'failed') process.exitCode = 1;
|
|
262
266
|
if ('job' in result && result.job.ticketSync && result.job.ticketSync.state !== 'applied') process.exitCode = 1;
|
|
263
267
|
}, 60 * 60_000));
|
|
@@ -44,6 +44,8 @@ const scoreText = (scoreArg: number | null): string => scoreArg === null ? 'unsc
|
|
|
44
44
|
const placementText = (accountArg: NonNullable<TReportedProxyStatus['accounts']>[number]): string => {
|
|
45
45
|
if (accountArg.lifecycle === undefined) return '';
|
|
46
46
|
if (accountArg.lifecycle === 'lapsed') return `; lapsed ${accountArg.lapsesAt}, never routed to again`;
|
|
47
|
+
// Paused by its owner (AGL-1933): its state says so, and it has no placement to name.
|
|
48
|
+
if (accountArg.pausedAt) return '';
|
|
47
49
|
return `${accountArg.lifecycle === 'canceled' ? `; canceled, lapses ${accountArg.lapsesAt}` : ''}`
|
|
48
50
|
+ `${accountArg.priorityUntil ? `; priority until ${accountArg.priorityUntil} (set by the owner)` : ''}`
|
|
49
51
|
+ `${accountArg.renewal ? `; renews ${accountArg.renewal.at} (set by the owner`
|
|
@@ -13,8 +13,8 @@ const descriptions: Record<TFleetWorkerAction, string> = {
|
|
|
13
13
|
archive: 'Archive a settled worker. Codex archives natively; Claude archives the AGL registry only and retains its transcript/worktree. Returns a job handle.',
|
|
14
14
|
refresh: 'Start a fresh thread from the settled predecessor\'s self-contained Handoff report. Store the successor before retiring the predecessor. With profile, the successor launches on that configured profile, which must run the predecessor\'s harness (only move changes the harness); without it the successor keeps the predecessor\'s recorded profile. Returns a job handle.',
|
|
15
15
|
move: 'Move a settled Claude lane to a new shared-daemon Codex thread with its Handoff report. Identify the successor before retiring Claude. Returns a job handle.',
|
|
16
|
-
list: 'List this owner\'s fleet workers as summaries, newest first, in pages of limit (1-100, default 50): identity, ticket, state, native id, cwd, the native status, newest turn outcome and native wait last observed, and fence flags (an unresolved dispatch with its action and since, a lost or reconfirmed dispatch, a recorded error). total, fenced and failed (recorded errors and failed turn outcomes) count the whole filtered set, so a page never hides a fence. Continue with nextCursor and the same filter (archived, ticketKey, state, harness); a cursor of another filter or one after a worker no longer held as it was is refused. view full returns every full record unpaged; fleet_worker_get returns one worker whole.',
|
|
17
|
-
get: 'Read one of this owner\'s fleet workers, archived ones included, by name or id: its full record (dispatch fence, lost, reconfirmed and re-bind resolutions with their attribution and proof, error) and the native status, turn outcome and start, native wait and excerpt the controller last observed.',
|
|
16
|
+
list: 'List this owner\'s fleet workers as summaries, newest first, in pages of limit (1-100, default 50): identity, ticket, state, native id, cwd, the native status, newest turn outcome and native wait last observed, and fence flags (an unresolved dispatch with its action and since, a lost or reconfirmed dispatch, a recorded error), and identity {unverified, reason, since} while a worker\'s MCP calls are refused its identity. total, fenced and failed (recorded errors and failed turn outcomes) count the whole filtered set, so a page never hides a fence. Continue with nextCursor and the same filter (archived, ticketKey, state, harness); a cursor of another filter or one after a worker no longer held as it was is refused. view full returns every full record unpaged; fleet_worker_get returns one worker whole.',
|
|
17
|
+
get: 'Read one of this owner\'s fleet workers, archived ones included, by name or id: its full record (dispatch fence, lost, reconfirmed and re-bind resolutions with their attribution and proof, error) and the native status, turn outcome and start, native wait and excerpt the controller last observed, and identity {unverified, reason, since} while its MCP calls are refused its identity.',
|
|
18
18
|
status: 'Read a fleet operation job (the handle a fleet_worker_* action returns), not a host job. Jobs are transient; bindings and unresolved dispatches survive restart. Inspect unknown native outcomes before retrying.',
|
|
19
19
|
'job-wait': 'Wait until a fleet operation job (the handle a fleet_worker_* action returns, not a host job) succeeds or fails, or timeoutMs (at most 240000) passes; timedOut tells which. Waiting, a timeout or an abandoned wait never cancels the job. A job the controller no longer holds (jobs end with a controller restart or eviction) is JOB_UNAVAILABLE, not failed: read the worker with fleet_worker_get. A failed job\'s errorOutcome is refused when nothing reached a native session, unknown when it may have (a steer may have been delivered): never repeat an unknown one before reading the worker.',
|
|
20
20
|
'native-message': 'Reach a Claude Code background job that no fleet worker binds with an attributed message, through Claude Code itself, as the browser\'s native session send does: a settled (done, blocked or stopped) job is woken with it (`claude --bg --resume <id> <text>`), a running one takes it in its inbox through Claude Code\'s own SendMessage, accepted and queued there, not yet read. nativeId is the full session id or a unique prefix of at least 8 characters; an ambiguous prefix is refused. A running job without a session name of its own (none, or one another running session shares), a failed or held job, a fleet worker\'s session and a project conversation are refused with their reason. The message arrives behind the same attribution header as session_message, naming the sender label and the local owner capability, never as the job\'s user. It holds no dispatch fence, writes no ticket comment and does not observe the turn that reads it; the receipt (delivery native-wake or native-inbox) names the command that adopts the job as a fleet worker. Returns a job handle; an unknown outcome is never retried.',
|
|
@@ -145,7 +145,8 @@ const projectMember = (accountArg: IProxyCapacityAccount, planArg: string, nowAr
|
|
|
145
145
|
};
|
|
146
146
|
|
|
147
147
|
/**
|
|
148
|
-
* A pool's room for new work (AGL-1097). Its members are the pool's subscription sign-ins that have not lapsed
|
|
148
|
+
* A pool's room for new work (AGL-1097). Its members are the pool's subscription sign-ins that have not lapsed, are not
|
|
149
|
+
* paused by their owner and whose
|
|
149
150
|
* held reading states a plan the pool counts (`proxyCapacityPlans`), stale or not; API keys never count. The room is
|
|
150
151
|
* the headroom its members add now (see `projectMember`), summed.
|
|
151
152
|
*
|
|
@@ -161,7 +162,9 @@ export const proxyPoolCapacity = (providerArg: TProxyCapacityProvider, accountsA
|
|
|
161
162
|
const { now, minHeadroom, observedSince } = optionsArg;
|
|
162
163
|
const counted = accountsArg.flatMap((accountArg) => {
|
|
163
164
|
const { status } = accountArg;
|
|
164
|
-
|
|
165
|
+
// An account its owner holds paused (AGL-1933) takes no work, so it adds no room and is no member.
|
|
166
|
+
if (status.provider !== providerArg || status.credential !== 'subscription' || status.lifecycle === 'lapsed'
|
|
167
|
+
|| status.pausedAt) return [];
|
|
165
168
|
const plan = proxyCapacityHeldPlan(providerArg, accountArg.usage);
|
|
166
169
|
return plan !== null && proxyCapacityPlanCounts(providerArg, plan) ? [{ account: accountArg, plan }] : [];
|
|
167
170
|
}).sort((left, right) => left.account.status.id.localeCompare(right.account.status.id));
|