agl 43.0.4 → 43.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.md +25 -0
- package/dist_serve/bundle.js +5186 -4683
- package/dist_serve/bundle.js.map +2 -2
- package/dist_serve/bundle.js.third-party-notices.json +29 -29
- package/dist_serve/bundle.js.third-party-notices.txt +29 -29
- package/dist_serve/third-party-notices.md +1 -1
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/aglapi/aglapisurface.document.js +475 -4
- package/dist_ts/aglapi/interfaces.d.ts +10 -0
- package/dist_ts/aglapi/interfaces.js +3 -1
- package/dist_ts/classes.claudeharness.d.ts +14 -5
- package/dist_ts/classes.claudeharness.js +62 -2
- package/dist_ts/classes.codexclientadapter.d.ts +7 -5
- package/dist_ts/classes.codexclientadapter.js +38 -3
- package/dist_ts/classes.controller.d.ts +9 -2
- package/dist_ts/classes.controller.js +116 -24
- package/dist_ts/classes.flexservice.d.ts +6 -7
- package/dist_ts/classes.flexservice.js +12 -45
- package/dist_ts/classes.flexsupervisor.d.ts +37 -3
- package/dist_ts/classes.flexsupervisor.js +164 -37
- package/dist_ts/classes.inferencegateway.d.ts +8 -5
- package/dist_ts/classes.inferencegateway.js +62 -19
- package/dist_ts/classes.inferencespend.d.ts +9 -3
- package/dist_ts/classes.inferencespend.js +44 -16
- package/dist_ts/classes.mcpclient.d.ts +6 -4
- package/dist_ts/classes.mcpclient.js +6 -2
- package/dist_ts/classes.mcpcontrollerhost.js +5 -1
- package/dist_ts/classes.mcptools.d.ts +1 -1
- package/dist_ts/classes.mcptools.js +17 -4
- package/dist_ts/classes.opencodeclient.js +3 -1
- package/dist_ts/classes.proxycontrol.d.ts +22 -11
- package/dist_ts/classes.proxycontrol.js +6 -3
- package/dist_ts/classes.proxycontrolclient.d.ts +6 -5
- package/dist_ts/classes.proxycontrolclient.js +8 -6
- package/dist_ts/classes.proxydaemon.d.ts +8 -6
- package/dist_ts/classes.proxydaemon.js +35 -21
- package/dist_ts/classes.proxyleaseholder.d.ts +12 -4
- package/dist_ts/classes.proxyleaseholder.js +39 -9
- package/dist_ts/classes.proxystore.d.ts +4 -3
- package/dist_ts/classes.proxystore.js +17 -7
- package/dist_ts/classes.sessionfeed.d.ts +1 -1
- package/dist_ts/classes.validation.d.ts +7 -0
- package/dist_ts/classes.validation.js +49 -2
- package/dist_ts/driver/functions.drivertools.js +2 -2
- package/dist_ts/fleet/classes.codexworkers.d.ts +4 -11
- package/dist_ts/fleet/classes.codexworkers.js +6 -3
- package/dist_ts/fleet/classes.workers.d.ts +25 -13
- package/dist_ts/fleet/classes.workers.js +105 -9
- package/dist_ts/fleet/functions.workervalidation.d.ts +26 -1
- package/dist_ts/fleet/functions.workervalidation.js +40 -11
- package/dist_ts/fleet/interfaces.workers.d.ts +17 -11
- package/dist_ts/fleet/interfaces.workers.js +10 -1
- package/dist_ts/flexharness.child.js +9 -4
- package/dist_ts/functions.confinedrelay.d.ts +21 -0
- package/dist_ts/functions.confinedrelay.js +65 -0
- package/dist_ts/functions.deviceapi.d.ts +4 -2
- package/dist_ts/functions.deviceapi.js +20 -2
- package/dist_ts/functions.elicitation.d.ts +49 -0
- package/dist_ts/functions.elicitation.js +418 -0
- package/dist_ts/functions.flexchildshutdown.d.ts +7 -0
- package/dist_ts/functions.flexchildshutdown.js +18 -1
- package/dist_ts/functions.inferencecli.d.ts +3 -3
- package/dist_ts/functions.inferencecli.js +33 -10
- package/dist_ts/functions.inferencecliarguments.d.ts +2 -1
- package/dist_ts/functions.inferencecliarguments.js +29 -8
- package/dist_ts/functions.inferencelaunch.d.ts +35 -7
- package/dist_ts/functions.inferencelaunch.js +143 -16
- package/dist_ts/functions.mcpfleetworkertools.d.ts +63 -70
- package/dist_ts/functions.mcpfleetworkertools.js +32 -4
- package/dist_ts/functions.proxycontract.d.ts +21 -4
- package/dist_ts/functions.proxycontract.js +12 -5
- package/dist_ts/functions.proxyharnessconfig.d.ts +7 -0
- package/dist_ts/functions.proxyharnessconfig.js +11 -1
- package/dist_ts/interfaces.auth.d.ts +1 -1
- package/dist_ts/interfaces.auth.js +2 -1
- package/dist_ts_interfaces/devices.d.ts +133 -3
- package/dist_ts_interfaces/devices.js +3 -2
- package/dist_ts_interfaces/elicitations.d.ts +125 -0
- package/dist_ts_interfaces/elicitations.js +7 -0
- package/dist_ts_interfaces/fleetworkers.d.ts +119 -5
- package/dist_ts_interfaces/fleetworkers.js +14 -2
- package/dist_ts_interfaces/index.d.ts +1 -0
- package/dist_ts_interfaces/index.js +2 -1
- package/dist_ts_interfaces/interfaces.d.ts +3 -1
- package/dist_ts_interfaces/interfaces.js +2 -2
- package/dist_ts_interfaces/mcprequests.d.ts +5 -1
- package/dist_ts_interfaces/mcprequests.js +2 -1
- package/dist_ts_interfaces/nativesessions.d.ts +14 -0
- package/dist_ts_interfaces/requests.d.ts +3 -3
- package/dist_ts_interfaces/requests.fleetworkers.js +2 -2
- package/dist_ts_interfaces/requests.js +1 -1
- package/dist_ts_interfaces/requests.promptreply.d.ts +18 -0
- package/dist_ts_interfaces/sessionfeed.d.ts +3 -2
- package/dist_ts_interfaces/sessionfeed.js +1 -1
- package/dist_ts_migration/index.d.ts +0 -2
- package/dist_ts_migration/index.js +1 -3
- package/dist_ts_migration/v43_flexpostharnessmigrationmarkers.d.ts +6 -17
- package/dist_ts_migration/v43_flexpostharnessmigrationmarkers.js +6 -36
- package/package.json +3 -3
- package/readme.md +191 -24
- package/third-party-notices.md +1 -1
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/aglapi/aglapisurface.document.ts +474 -3
- package/ts/aglapi/interfaces.ts +2 -0
- package/ts/classes.claudeharness.ts +80 -9
- package/ts/classes.codexclientadapter.ts +41 -5
- package/ts/classes.controller.ts +142 -29
- package/ts/classes.flexservice.ts +10 -66
- package/ts/classes.flexsupervisor.ts +214 -37
- package/ts/classes.inferencegateway.ts +69 -23
- package/ts/classes.inferencespend.ts +46 -13
- package/ts/classes.mcpclient.ts +8 -4
- package/ts/classes.mcpcontrollerhost.ts +4 -0
- package/ts/classes.mcptools.ts +17 -3
- package/ts/classes.opencodeclient.ts +2 -0
- package/ts/classes.proxycontrol.ts +27 -13
- package/ts/classes.proxycontrolclient.ts +9 -6
- package/ts/classes.proxydaemon.ts +49 -27
- package/ts/classes.proxyleaseholder.ts +43 -11
- package/ts/classes.proxystore.ts +21 -8
- package/ts/classes.sessionfeed.ts +1 -1
- package/ts/classes.validation.ts +52 -0
- package/ts/driver/functions.drivertools.ts +1 -1
- package/ts/fleet/classes.codexworkers.ts +8 -6
- package/ts/fleet/classes.workers.ts +101 -17
- package/ts/fleet/functions.workervalidation.ts +35 -9
- package/ts/fleet/interfaces.workers.ts +11 -4
- package/ts/flexharness.child.ts +8 -3
- package/ts/functions.confinedrelay.ts +65 -0
- package/ts/functions.deviceapi.ts +24 -1
- package/ts/functions.elicitation.ts +443 -0
- package/ts/functions.flexchildshutdown.ts +22 -0
- package/ts/functions.inferencecli.ts +38 -13
- package/ts/functions.inferencecliarguments.ts +26 -5
- package/ts/functions.inferencelaunch.ts +143 -16
- package/ts/functions.mcpfleetworkertools.ts +34 -4
- package/ts/functions.proxycontract.ts +17 -4
- package/ts/functions.proxyharnessconfig.ts +11 -0
- package/ts/interfaces.auth.ts +1 -0
- package/ts_interfaces/devices.schema.json +476 -2
- package/ts_interfaces/devices.ts +146 -2
- package/ts_interfaces/elicitations.ts +148 -0
- package/ts_interfaces/fleetworkers.ts +96 -4
- package/ts_interfaces/index.ts +1 -0
- package/ts_interfaces/interfaces.ts +3 -1
- package/ts_interfaces/mcprequests.ts +1 -0
- package/ts_interfaces/nativesessions.ts +16 -0
- package/ts_interfaces/requests.fleetworkers.ts +1 -1
- package/ts_interfaces/requests.promptreply.ts +22 -0
- package/ts_interfaces/requests.ts +4 -0
- package/ts_interfaces/sessionfeed.ts +3 -2
- package/ts_migration/index.ts +0 -2
- package/ts_migration/v43_flexpostharnessmigrationmarkers.ts +6 -49
- package/ts_web/00_commitinfo_data.ts +1 -1
- package/ts_web/classes.elicitationcards.ts +106 -0
- package/ts_web/classes.nativesessionstate.ts +38 -1
- package/ts_web/elements.fleetworker.ts +27 -2
- package/ts_web/elements.harnesscontrollerapp.ts +68 -1
- package/ts_web/functions.sessionchat.ts +5 -0
- package/dist_ts_migration/classes.flexpostharnessmigrationrunner.d.ts +0 -19
- package/dist_ts_migration/classes.flexpostharnessmigrationrunner.js +0 -34
- package/dist_ts_migration/v19_flexprojectmanagement.d.ts +0 -14
- package/dist_ts_migration/v19_flexprojectmanagement.js +0 -23
- package/ts_migration/classes.flexpostharnessmigrationrunner.ts +0 -54
- package/ts_migration/v19_flexprojectmanagement.ts +0 -32
|
@@ -7,11 +7,6 @@ import {
|
|
|
7
7
|
} from './classes.flexmodels.private.js';
|
|
8
8
|
import { FlexStore, type IFlexProjectManagementHost } from './classes.flexstore.js';
|
|
9
9
|
import { FlexPersistenceMigrationRunner } from '../ts_migration/classes.flexmigrationrunner.js';
|
|
10
|
-
import { FlexPostHarnessMigrationRunner } from '../ts_migration/classes.flexpostharnessmigrationrunner.js';
|
|
11
|
-
import {
|
|
12
|
-
FlexPostHarnessMigrationMarkers,
|
|
13
|
-
type IFlexPostHarnessMigrationMarkers,
|
|
14
|
-
} from '../ts_migration/v43_flexpostharnessmigrationmarkers.js';
|
|
15
10
|
import {
|
|
16
11
|
assertCanonicalDirectory,
|
|
17
12
|
CanonicalDirectoryError,
|
|
@@ -606,11 +601,6 @@ interface IFlexChildBrowserChannel {
|
|
|
606
601
|
sessionGenerationSequence: number;
|
|
607
602
|
}
|
|
608
603
|
|
|
609
|
-
interface IFlexProjectRegistrationMigrationContext {
|
|
610
|
-
projectId: string;
|
|
611
|
-
registrationOperationId: string;
|
|
612
|
-
}
|
|
613
|
-
|
|
614
604
|
interface IFlexPendingProjectRemoval {
|
|
615
605
|
removalOperationId: string;
|
|
616
606
|
cleanupCohort: IFlexProjectCleanupCohortEntry[];
|
|
@@ -722,12 +712,8 @@ export class FlexService {
|
|
|
722
712
|
(errorArg, contextArg) => this.projectRunFailure(errorArg, contextArg),
|
|
723
713
|
);
|
|
724
714
|
private readonly pendingProjectRegistrations = new Set<string>();
|
|
725
|
-
/** Which projects' post-harness migrations completed, read by every child generation. */
|
|
726
|
-
private postHarnessMigrationMarkers?: IFlexPostHarnessMigrationMarkers;
|
|
727
715
|
private readonly pendingProjectRetirements = new Set<string>();
|
|
728
716
|
private readonly pendingProjectRemovals = new Map<string, IFlexPendingProjectRemoval>();
|
|
729
|
-
private readonly projectRegistrationMigrationContext =
|
|
730
|
-
new plugins.asyncHooks.AsyncLocalStorage<IFlexProjectRegistrationMigrationContext>();
|
|
731
717
|
private readonly catalogCache = new Map<string, IFlexCatalogCache>();
|
|
732
718
|
/** The models of the proxy's Claude pool, read in its connection's queue like an account's catalog. */
|
|
733
719
|
private claudePoolCatalog?: IFlexProxyClaudeModel[];
|
|
@@ -963,7 +949,6 @@ export class FlexService {
|
|
|
963
949
|
});
|
|
964
950
|
this.store = store;
|
|
965
951
|
await store.init();
|
|
966
|
-
this.postHarnessMigrationMarkers = new FlexPostHarnessMigrationMarkers(initArg.controllerId);
|
|
967
952
|
const activeProjectIds = new Set(this.projects.keys());
|
|
968
953
|
await new FlexPersistenceMigrationRunner({
|
|
969
954
|
store,
|
|
@@ -1211,12 +1196,7 @@ export class FlexService {
|
|
|
1211
1196
|
switch (methodArg) {
|
|
1212
1197
|
case 'project.register': {
|
|
1213
1198
|
const payload = payloadArg as IFlexRequestMap['project.register']['request'];
|
|
1214
|
-
const added = await this.registerProject(
|
|
1215
|
-
payload.project,
|
|
1216
|
-
payload.registrationOperationId,
|
|
1217
|
-
harness,
|
|
1218
|
-
signalArg,
|
|
1219
|
-
);
|
|
1199
|
+
const added = await this.registerProject(payload.project, signalArg);
|
|
1220
1200
|
return { result: { registered: true, added } };
|
|
1221
1201
|
}
|
|
1222
1202
|
case 'project.remove': {
|
|
@@ -3438,14 +3418,10 @@ export class FlexService {
|
|
|
3438
3418
|
);
|
|
3439
3419
|
return {
|
|
3440
3420
|
load: async (storageKey, sessionId, sessionContext) => {
|
|
3441
|
-
const registrationMigration = this.projectRegistrationMigrationContext.getStore();
|
|
3442
3421
|
const result = await this.requestHost('project-management.load', {
|
|
3443
3422
|
storageKey,
|
|
3444
3423
|
sessionId,
|
|
3445
3424
|
sessionContext: this.cloneProjectManagementSessionContext(sessionContext),
|
|
3446
|
-
...(registrationMigration?.projectId === storageKey
|
|
3447
|
-
? { registrationOperationId: registrationMigration.registrationOperationId }
|
|
3448
|
-
: {}),
|
|
3449
3425
|
});
|
|
3450
3426
|
return result.record ?? undefined;
|
|
3451
3427
|
},
|
|
@@ -5405,10 +5381,15 @@ export class FlexService {
|
|
|
5405
5381
|
}
|
|
5406
5382
|
}
|
|
5407
5383
|
|
|
5384
|
+
/**
|
|
5385
|
+
* Registration does no work per session: it runs the project's persistence migrations and makes
|
|
5386
|
+
* the project visible. FlexHarness loads the project's namespace (which repairs session
|
|
5387
|
+
* generations and reversion state) on the project's first request, and a session's
|
|
5388
|
+
* project-management state when something first reads it; that load is where the controller
|
|
5389
|
+
* copies a legacy scratchpad into the state (`loadFlexProjectManagement`).
|
|
5390
|
+
*/
|
|
5408
5391
|
private async registerProject(
|
|
5409
5392
|
projectArg: IFlexIdentityBoundProjectRecord,
|
|
5410
|
-
registrationOperationIdArg: string,
|
|
5411
|
-
harnessArg: TFlexHarness<IFlexProjectScope>,
|
|
5412
5393
|
signalArg: AbortSignal,
|
|
5413
5394
|
): Promise<boolean> {
|
|
5414
5395
|
const projectId = projectArg.projectId;
|
|
@@ -5420,7 +5401,6 @@ export class FlexService {
|
|
|
5420
5401
|
const current = this.projects.get(projectId);
|
|
5421
5402
|
if (current) {
|
|
5422
5403
|
if (!flexProjectDirectoriesEqual(current, projectArg)) throw new FlexServiceError('CONFLICT');
|
|
5423
|
-
await this.runPostHarnessMigrations(projectId, registrationOperationIdArg, harnessArg, signalArg);
|
|
5424
5404
|
return false;
|
|
5425
5405
|
}
|
|
5426
5406
|
// Registrations of other projects run concurrently, so a pending one holds its place in the
|
|
@@ -5430,57 +5410,22 @@ export class FlexService {
|
|
|
5430
5410
|
if (this.projects.size + pendingAdditions >= controllerProjectLimit) {
|
|
5431
5411
|
throw new FlexServiceError('LIMIT_EXCEEDED');
|
|
5432
5412
|
}
|
|
5433
|
-
if (!this.store
|
|
5413
|
+
if (!this.store) throw new FlexServiceError('NOT_READY');
|
|
5434
5414
|
|
|
5435
5415
|
this.pendingProjectRegistrations.add(projectId);
|
|
5436
|
-
const project = structuredClone(projectArg);
|
|
5437
|
-
let temporarilyVisible = false;
|
|
5438
5416
|
try {
|
|
5439
5417
|
await new FlexPersistenceMigrationRunner({
|
|
5440
5418
|
store: this.store,
|
|
5441
5419
|
storageKeys: [projectId],
|
|
5442
5420
|
}).run(signalArg);
|
|
5443
5421
|
signalArg.throwIfAborted();
|
|
5444
|
-
this.projects.set(projectId,
|
|
5445
|
-
temporarilyVisible = true;
|
|
5446
|
-
// FlexHarness recovers every session's reversion state when it loads the project's namespace,
|
|
5447
|
-
// on the project's first request; reading it adds only a check, which runs when a session is
|
|
5448
|
-
// opened (`session.reversion.info`, sent by `controller.slash.list`). Registration walks none.
|
|
5449
|
-
await this.runPostHarnessMigrations(projectId, registrationOperationIdArg, harnessArg, signalArg);
|
|
5450
|
-
signalArg.throwIfAborted();
|
|
5422
|
+
this.projects.set(projectId, structuredClone(projectArg));
|
|
5451
5423
|
return true;
|
|
5452
|
-
} catch (errorArg) {
|
|
5453
|
-
if (temporarilyVisible && this.projects.get(projectId) === project) {
|
|
5454
|
-
this.projects.delete(projectId);
|
|
5455
|
-
}
|
|
5456
|
-
throw errorArg;
|
|
5457
5424
|
} finally {
|
|
5458
5425
|
this.pendingProjectRegistrations.delete(projectId);
|
|
5459
5426
|
}
|
|
5460
5427
|
}
|
|
5461
5428
|
|
|
5462
|
-
/**
|
|
5463
|
-
* The project's post-harness migrations, each walked only until it completed once for the
|
|
5464
|
-
* project: its persisted marker spares every later child generation the per-session walk.
|
|
5465
|
-
*/
|
|
5466
|
-
private async runPostHarnessMigrations(
|
|
5467
|
-
projectIdArg: string,
|
|
5468
|
-
registrationOperationIdArg: string,
|
|
5469
|
-
harnessArg: TFlexHarness<IFlexProjectScope>,
|
|
5470
|
-
signalArg: AbortSignal,
|
|
5471
|
-
): Promise<void> {
|
|
5472
|
-
const markers = this.postHarnessMigrationMarkers;
|
|
5473
|
-
if (!markers) throw new FlexServiceError('NOT_READY');
|
|
5474
|
-
await this.projectRegistrationMigrationContext.run(
|
|
5475
|
-
{ projectId: projectIdArg, registrationOperationId: registrationOperationIdArg },
|
|
5476
|
-
() => new FlexPostHarnessMigrationRunner({
|
|
5477
|
-
harness: harnessArg,
|
|
5478
|
-
markers,
|
|
5479
|
-
scopeIds: [projectIdArg],
|
|
5480
|
-
}).run(signalArg),
|
|
5481
|
-
);
|
|
5482
|
-
}
|
|
5483
|
-
|
|
5484
5429
|
private async removeProject(
|
|
5485
5430
|
projectArg: IFlexIdentityBoundProjectRecord,
|
|
5486
5431
|
removalOperationIdArg: string,
|
|
@@ -5693,7 +5638,6 @@ export class FlexService {
|
|
|
5693
5638
|
this.pendingProjectRegistrations.clear();
|
|
5694
5639
|
this.pendingProjectRetirements.clear();
|
|
5695
5640
|
this.pendingProjectRemovals.clear();
|
|
5696
|
-
this.postHarnessMigrationMarkers = undefined;
|
|
5697
5641
|
this.catalogCache.clear();
|
|
5698
5642
|
this.claudePoolCatalog = undefined;
|
|
5699
5643
|
this.jobs.clear();
|
|
@@ -6,7 +6,7 @@ import {
|
|
|
6
6
|
GitReversionDirtyWorktreeError,
|
|
7
7
|
GitReversionFencedError,
|
|
8
8
|
} from './classes.gitreversion.js';
|
|
9
|
-
import type
|
|
9
|
+
import { sanitizeFailureText, type IFailureDescription } from './functions.failuredescription.js';
|
|
10
10
|
import { createSanitizedRuntimeEnvironment } from './functions.runtimeenvironment.js';
|
|
11
11
|
import {
|
|
12
12
|
FlexServiceError,
|
|
@@ -73,11 +73,19 @@ export interface IFlexSupervisorLogEntry {
|
|
|
73
73
|
/** An error of a child's log pipe, or of a started child process, the supervisor recorded and ignored. */
|
|
74
74
|
export interface IFlexSupervisorIgnoredError {
|
|
75
75
|
source: 'stdout' | 'stderr' | 'process';
|
|
76
|
+
/** The child generation, counted from 1 per supervisor, whose pipe or process the error came from. */
|
|
77
|
+
generation: number;
|
|
76
78
|
code?: string;
|
|
77
79
|
timestamp: number;
|
|
78
80
|
}
|
|
79
81
|
|
|
80
82
|
export interface IFlexSupervisorOptions {
|
|
83
|
+
/**
|
|
84
|
+
* Writes one record to the controller log; the controller's stderr (`console.error`) unless a test
|
|
85
|
+
* captures it. Each child stdout and stderr line arrives here as it is read, tagged with its child
|
|
86
|
+
* generation, credential-redacted and bounded, and so does every abnormal child exit.
|
|
87
|
+
*/
|
|
88
|
+
writeControllerLog?: (lineArg: string) => void;
|
|
81
89
|
onChildExitObserved?: (exitArg: IFlexSupervisorExit) => void;
|
|
82
90
|
onChildExit?: (exitArg: IFlexSupervisorExit) => void;
|
|
83
91
|
cancellationGraceTimeoutMs?: number;
|
|
@@ -156,6 +164,19 @@ interface ILogAccumulator {
|
|
|
156
164
|
decoder: TextDecoder;
|
|
157
165
|
partial: string;
|
|
158
166
|
dropping: boolean;
|
|
167
|
+
/** The stream's last suppressed lines of the current rate window, as buffered (at most 8 KiB each). */
|
|
168
|
+
suppressedTail: string[];
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** One child generation's log state: its pipes' partial lines and its controller-log rate window. */
|
|
172
|
+
interface IFlexChildLogGeneration {
|
|
173
|
+
generation: number;
|
|
174
|
+
pid?: number;
|
|
175
|
+
stdout: ILogAccumulator;
|
|
176
|
+
stderr: ILogAccumulator;
|
|
177
|
+
windowStartedAt: number;
|
|
178
|
+
windowLines: number;
|
|
179
|
+
suppressedLines: number;
|
|
159
180
|
}
|
|
160
181
|
|
|
161
182
|
const supervisorLogEntries = 200;
|
|
@@ -165,6 +186,17 @@ const supervisorTermTimeoutMs = 5_000;
|
|
|
165
186
|
const supervisorKillTimeoutMs = 2_000;
|
|
166
187
|
const logSuffix = '... [truncated]';
|
|
167
188
|
const supervisorExitMessageBytes = 1024;
|
|
189
|
+
/** A child line in the controller log keeps at most this many bytes, after its credentials are hidden. */
|
|
190
|
+
const supervisorControllerLogLineBytes = 2048;
|
|
191
|
+
/** A child generation writes at most this many lines to the controller log per window; the rest are counted. */
|
|
192
|
+
const supervisorControllerLogLinesPerWindow = 200;
|
|
193
|
+
const supervisorControllerLogWindowMs = 60_000;
|
|
194
|
+
/**
|
|
195
|
+
* The suppression record is followed by each stream's last this many suppressed lines, so a child that
|
|
196
|
+
* floods and then fails still leaves its fault: the crash line, the stack's header and its frames
|
|
197
|
+
* (`Error.stackTraceLimit`, 10 in the Flex child).
|
|
198
|
+
*/
|
|
199
|
+
const supervisorControllerLogSuppressedTailLines = 20;
|
|
168
200
|
|
|
169
201
|
export class FlexBrowserChannelNotificationUnavailableError extends Error {
|
|
170
202
|
constructor() {
|
|
@@ -326,12 +358,9 @@ export class FlexSupervisor {
|
|
|
326
358
|
private readonly eventListeners = new Set<(eventArg: TFlexChildEvent) => void>();
|
|
327
359
|
private readonly logs: IFlexSupervisorLogEntry[] = [];
|
|
328
360
|
private readonly ignoredChildErrors: IFlexSupervisorIgnoredError[] = [];
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
private readonly stderrAccumulator: ILogAccumulator = {
|
|
333
|
-
decoder: new TextDecoder(), partial: '', dropping: false,
|
|
334
|
-
};
|
|
361
|
+
/** Child generations observed so far; the controller log names each child by its number. */
|
|
362
|
+
private childGenerations = 0;
|
|
363
|
+
private readonly writeControllerLog: (lineArg: string) => void;
|
|
335
364
|
private readonly cancellationGraceTimeoutMs: number;
|
|
336
365
|
private readonly childTaskDrainTimeoutMs: number;
|
|
337
366
|
/** The writer identity of the current child generation, until its exit is confirmed. */
|
|
@@ -340,6 +369,7 @@ export class FlexSupervisor {
|
|
|
340
369
|
private readonly retiredWriterIds: string[] = [];
|
|
341
370
|
|
|
342
371
|
constructor(private readonly options: IFlexSupervisorOptions = {}) {
|
|
372
|
+
this.writeControllerLog = options.writeControllerLog ?? ((lineArg) => console.error(lineArg));
|
|
343
373
|
this.cancellationGraceTimeoutMs = this.normalizeDeadline(
|
|
344
374
|
options.cancellationGraceTimeoutMs,
|
|
345
375
|
flexIpcCancellationGraceTimeoutMs,
|
|
@@ -1379,19 +1409,40 @@ export class FlexSupervisor {
|
|
|
1379
1409
|
}
|
|
1380
1410
|
|
|
1381
1411
|
private observeChild(childArg: plugins.childProcess.ChildProcess): Promise<IFlexChildLifecycle> {
|
|
1412
|
+
this.childGenerations += 1;
|
|
1413
|
+
const logGeneration: IFlexChildLogGeneration = {
|
|
1414
|
+
generation: this.childGenerations,
|
|
1415
|
+
...(childArg.pid ? { pid: childArg.pid } : {}),
|
|
1416
|
+
stdout: { decoder: new TextDecoder(), partial: '', dropping: false, suppressedTail: [] },
|
|
1417
|
+
stderr: { decoder: new TextDecoder(), partial: '', dropping: false, suppressedTail: [] },
|
|
1418
|
+
windowStartedAt: Date.now(),
|
|
1419
|
+
windowLines: 0,
|
|
1420
|
+
suppressedLines: 0,
|
|
1421
|
+
};
|
|
1382
1422
|
// The log pipes and the process keep their error listeners for as long as they exist: an error
|
|
1383
|
-
// event without a listener throws, and the child's lifecycle never depends on its log pipes.
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1423
|
+
// event without a listener throws, and the child's lifecycle never depends on its log pipes. A
|
|
1424
|
+
// pipe's last partial line is written when the pipe closes, which can be after the child's exit,
|
|
1425
|
+
// and the generation's suppressed-line count and tails once both have closed.
|
|
1426
|
+
let openLogPipes = 0;
|
|
1427
|
+
const closeLogPipe = (streamArg: IFlexSupervisorLogEntry['stream']): void => {
|
|
1428
|
+
this.flushLog(logGeneration, streamArg);
|
|
1429
|
+
openLogPipes -= 1;
|
|
1430
|
+
if (openLogPipes === 0) this.reportSuppressedLines(logGeneration);
|
|
1431
|
+
};
|
|
1432
|
+
for (const stream of ['stdout', 'stderr'] as const) {
|
|
1433
|
+
const pipe = childArg[stream];
|
|
1434
|
+
if (!pipe) continue;
|
|
1435
|
+
openLogPipes += 1;
|
|
1436
|
+
pipe.on('data', (chunk) => this.consumeLog(logGeneration, stream, chunk));
|
|
1437
|
+
pipe.on('error', (errorArg) => this.recordIgnoredChildError(stream, logGeneration.generation, errorArg));
|
|
1438
|
+
pipe.once('close', () => closeLogPipe(stream));
|
|
1439
|
+
}
|
|
1388
1440
|
return new Promise((resolve) => {
|
|
1389
1441
|
let settled = false;
|
|
1390
1442
|
const finish = (outcomeArg: IFlexChildLifecycle): void => {
|
|
1391
1443
|
if (settled) return;
|
|
1392
1444
|
settled = true;
|
|
1393
|
-
this.
|
|
1394
|
-
this.flushLog('stderr', this.stderrAccumulator);
|
|
1445
|
+
this.logAbnormalExit(logGeneration, outcomeArg, this.status.state === 'stopping');
|
|
1395
1446
|
this.status = { state: this.status.state === 'stopping' ? 'stopped' : 'failed', ready: false };
|
|
1396
1447
|
this.browserPeerId = '';
|
|
1397
1448
|
this.browserChannelGenerations.clear();
|
|
@@ -1437,14 +1488,59 @@ export class FlexSupervisor {
|
|
|
1437
1488
|
// A child without a process ID never started, and its error is its outcome; a started
|
|
1438
1489
|
// child's error (a failed signal) leaves its lifecycle to its exit.
|
|
1439
1490
|
if (!childArg.pid) finish({ type: 'error', error });
|
|
1440
|
-
else this.recordIgnoredChildError('process', error);
|
|
1491
|
+
else this.recordIgnoredChildError('process', logGeneration.generation, error);
|
|
1441
1492
|
});
|
|
1442
1493
|
});
|
|
1443
1494
|
}
|
|
1444
1495
|
|
|
1445
|
-
|
|
1496
|
+
/**
|
|
1497
|
+
* Writes an exit the supervisor did not ask for, or one with a failure code or a signal, to the
|
|
1498
|
+
* controller log: the generation, its exit code and signal or why it never started, and the pipe
|
|
1499
|
+
* and process errors of that generation the supervisor ignored so far.
|
|
1500
|
+
*/
|
|
1501
|
+
private logAbnormalExit(
|
|
1502
|
+
logGenerationArg: IFlexChildLogGeneration,
|
|
1503
|
+
outcomeArg: IFlexChildLifecycle,
|
|
1504
|
+
requestedArg: boolean,
|
|
1505
|
+
): void {
|
|
1506
|
+
const child = this.describeChildGeneration(logGenerationArg);
|
|
1507
|
+
if (outcomeArg.type === 'error') {
|
|
1508
|
+
const reason = outcomeArg.error ? sanitizeFailureText(outcomeArg.error.message) : '';
|
|
1509
|
+
this.writeControllerLog(
|
|
1510
|
+
`${child} did not start: ${sanitizeExitMessage(reason) || 'the process reported no reason'}.`,
|
|
1511
|
+
);
|
|
1512
|
+
return;
|
|
1513
|
+
}
|
|
1514
|
+
const code = outcomeArg.code ?? null;
|
|
1515
|
+
const signal = outcomeArg.signal ?? null;
|
|
1516
|
+
if (requestedArg && code === 0 && signal === null) return;
|
|
1517
|
+
const ignored = this.ignoredChildErrors
|
|
1518
|
+
.filter((entryArg) => entryArg.generation === logGenerationArg.generation)
|
|
1519
|
+
.map((entryArg) => `${entryArg.source} ${entryArg.code ?? 'without code'}`);
|
|
1520
|
+
this.writeControllerLog(
|
|
1521
|
+
`${child} exited ${requestedArg ? 'while stopping' : 'unexpectedly'}: `
|
|
1522
|
+
+ `code ${code === null ? 'none' : String(code)}, signal ${signal ?? 'none'}`
|
|
1523
|
+
+ `${ignored.length > 0 ? `; ignored errors: ${ignored.join(', ')}` : ''}.`,
|
|
1524
|
+
);
|
|
1525
|
+
}
|
|
1526
|
+
|
|
1527
|
+
private describeChildGeneration(logGenerationArg: IFlexChildLogGeneration): string {
|
|
1528
|
+
return `Flex child generation ${logGenerationArg.generation}`
|
|
1529
|
+
+ `${logGenerationArg.pid === undefined ? '' : ` (PID ${logGenerationArg.pid})`}`;
|
|
1530
|
+
}
|
|
1531
|
+
|
|
1532
|
+
private recordIgnoredChildError(
|
|
1533
|
+
sourceArg: IFlexSupervisorIgnoredError['source'],
|
|
1534
|
+
generationArg: number,
|
|
1535
|
+
errorArg: Error,
|
|
1536
|
+
): void {
|
|
1446
1537
|
const code = 'code' in errorArg && typeof errorArg.code === 'string' ? errorArg.code : undefined;
|
|
1447
|
-
this.ignoredChildErrors.push({
|
|
1538
|
+
this.ignoredChildErrors.push({
|
|
1539
|
+
source: sourceArg,
|
|
1540
|
+
generation: generationArg,
|
|
1541
|
+
...(code ? { code } : {}),
|
|
1542
|
+
timestamp: Date.now(),
|
|
1543
|
+
});
|
|
1448
1544
|
if (this.ignoredChildErrors.length > supervisorIgnoredErrorEntries) this.ignoredChildErrors.shift();
|
|
1449
1545
|
}
|
|
1450
1546
|
|
|
@@ -1628,9 +1724,9 @@ export class FlexSupervisor {
|
|
|
1628
1724
|
}
|
|
1629
1725
|
|
|
1630
1726
|
private consumeLog(
|
|
1727
|
+
logGenerationArg: IFlexChildLogGeneration,
|
|
1631
1728
|
streamArg: IFlexSupervisorLogEntry['stream'],
|
|
1632
1729
|
chunkArg: unknown,
|
|
1633
|
-
accumulatorArg: ILogAccumulator,
|
|
1634
1730
|
): void {
|
|
1635
1731
|
const bytes = typeof chunkArg === 'string'
|
|
1636
1732
|
? Buffer.from(chunkArg, 'utf8')
|
|
@@ -1638,49 +1734,64 @@ export class FlexSupervisor {
|
|
|
1638
1734
|
? chunkArg
|
|
1639
1735
|
: undefined;
|
|
1640
1736
|
if (!bytes) return;
|
|
1641
|
-
|
|
1737
|
+
const accumulator = logGenerationArg[streamArg];
|
|
1738
|
+
this.consumeLogText(logGenerationArg, streamArg, accumulator.decoder.decode(bytes, { stream: true }));
|
|
1739
|
+
}
|
|
1740
|
+
|
|
1741
|
+
private consumeLogText(
|
|
1742
|
+
logGenerationArg: IFlexChildLogGeneration,
|
|
1743
|
+
streamArg: IFlexSupervisorLogEntry['stream'],
|
|
1744
|
+
textArg: string,
|
|
1745
|
+
): void {
|
|
1746
|
+
const accumulator = logGenerationArg[streamArg];
|
|
1747
|
+
let text = textArg;
|
|
1642
1748
|
while (text.length > 0) {
|
|
1643
|
-
if (
|
|
1749
|
+
if (accumulator.dropping) {
|
|
1644
1750
|
const newline = text.indexOf('\n');
|
|
1645
1751
|
if (newline < 0) return;
|
|
1646
|
-
|
|
1752
|
+
accumulator.dropping = false;
|
|
1647
1753
|
text = text.slice(newline + 1);
|
|
1648
1754
|
continue;
|
|
1649
1755
|
}
|
|
1650
1756
|
const newline = text.indexOf('\n');
|
|
1651
1757
|
if (newline >= 0) {
|
|
1652
|
-
const complete = `${
|
|
1653
|
-
|
|
1654
|
-
this.addLog(streamArg, complete);
|
|
1758
|
+
const complete = `${accumulator.partial}${text.slice(0, newline)}`.replace(/\r$/u, '');
|
|
1759
|
+
accumulator.partial = '';
|
|
1760
|
+
this.addLog(logGenerationArg, streamArg, complete);
|
|
1655
1761
|
text = text.slice(newline + 1);
|
|
1656
1762
|
continue;
|
|
1657
1763
|
}
|
|
1658
|
-
const complete = `${
|
|
1764
|
+
const complete = `${accumulator.partial}${text}`;
|
|
1659
1765
|
if (Buffer.byteLength(complete, 'utf8') > supervisorLogLineBytes) {
|
|
1660
|
-
this.addLog(streamArg, complete);
|
|
1661
|
-
|
|
1662
|
-
|
|
1766
|
+
this.addLog(logGenerationArg, streamArg, complete);
|
|
1767
|
+
accumulator.partial = '';
|
|
1768
|
+
accumulator.dropping = true;
|
|
1663
1769
|
} else {
|
|
1664
|
-
|
|
1770
|
+
accumulator.partial = complete;
|
|
1665
1771
|
}
|
|
1666
1772
|
return;
|
|
1667
1773
|
}
|
|
1668
1774
|
}
|
|
1669
1775
|
|
|
1670
1776
|
private flushLog(
|
|
1777
|
+
logGenerationArg: IFlexChildLogGeneration,
|
|
1671
1778
|
streamArg: IFlexSupervisorLogEntry['stream'],
|
|
1672
|
-
accumulatorArg: ILogAccumulator,
|
|
1673
1779
|
): void {
|
|
1674
|
-
const
|
|
1675
|
-
|
|
1676
|
-
if (
|
|
1677
|
-
|
|
1780
|
+
const accumulator = logGenerationArg[streamArg];
|
|
1781
|
+
const final = accumulator.decoder.decode();
|
|
1782
|
+
if (final) this.consumeLogText(logGenerationArg, streamArg, final);
|
|
1783
|
+
if (accumulator.partial && !accumulator.dropping) {
|
|
1784
|
+
this.addLog(logGenerationArg, streamArg, accumulator.partial);
|
|
1678
1785
|
}
|
|
1679
|
-
|
|
1680
|
-
|
|
1786
|
+
accumulator.partial = '';
|
|
1787
|
+
accumulator.dropping = false;
|
|
1681
1788
|
}
|
|
1682
1789
|
|
|
1683
|
-
private addLog(
|
|
1790
|
+
private addLog(
|
|
1791
|
+
logGenerationArg: IFlexChildLogGeneration,
|
|
1792
|
+
streamArg: IFlexSupervisorLogEntry['stream'],
|
|
1793
|
+
lineArg: string,
|
|
1794
|
+
): void {
|
|
1684
1795
|
const bounded = truncateLog(lineArg);
|
|
1685
1796
|
this.logs.push({
|
|
1686
1797
|
timestamp: Date.now(),
|
|
@@ -1691,5 +1802,71 @@ export class FlexSupervisor {
|
|
|
1691
1802
|
if (this.logs.length > supervisorLogEntries) {
|
|
1692
1803
|
this.logs.splice(0, this.logs.length - supervisorLogEntries);
|
|
1693
1804
|
}
|
|
1805
|
+
this.writeChildLine(logGenerationArg, streamArg, bounded.line);
|
|
1806
|
+
}
|
|
1807
|
+
|
|
1808
|
+
/**
|
|
1809
|
+
* Writes a child line to the controller log as it arrives, within the generation's rate window.
|
|
1810
|
+
* Lines over the window's budget are counted and the last `supervisorControllerLogSuppressedTailLines`
|
|
1811
|
+
* of each stream kept as buffered, not redacted; they are reported when the next window opens or
|
|
1812
|
+
* the generation's pipes closed.
|
|
1813
|
+
*/
|
|
1814
|
+
private writeChildLine(
|
|
1815
|
+
logGenerationArg: IFlexChildLogGeneration,
|
|
1816
|
+
streamArg: IFlexSupervisorLogEntry['stream'],
|
|
1817
|
+
lineArg: string,
|
|
1818
|
+
): void {
|
|
1819
|
+
const now = Date.now();
|
|
1820
|
+
if (now - logGenerationArg.windowStartedAt >= supervisorControllerLogWindowMs) {
|
|
1821
|
+
this.reportSuppressedLines(logGenerationArg);
|
|
1822
|
+
logGenerationArg.windowStartedAt = now;
|
|
1823
|
+
logGenerationArg.windowLines = 0;
|
|
1824
|
+
}
|
|
1825
|
+
if (logGenerationArg.windowLines >= supervisorControllerLogLinesPerWindow) {
|
|
1826
|
+
logGenerationArg.suppressedLines += 1;
|
|
1827
|
+
const tail = logGenerationArg[streamArg].suppressedTail;
|
|
1828
|
+
tail.push(lineArg);
|
|
1829
|
+
if (tail.length > supervisorControllerLogSuppressedTailLines) tail.shift();
|
|
1830
|
+
return;
|
|
1831
|
+
}
|
|
1832
|
+
logGenerationArg.windowLines += 1;
|
|
1833
|
+
this.emitChildLine(logGenerationArg, streamArg, lineArg);
|
|
1834
|
+
}
|
|
1835
|
+
|
|
1836
|
+
/**
|
|
1837
|
+
* Writes one child line to the controller log, live or from a suppression tail: at most
|
|
1838
|
+
* `supervisorControllerLogLineBytes`, control characters spaced out and every credential hidden
|
|
1839
|
+
* before the cut (`sanitizeFailureText`, then `redactAndBound`), so the cut never keeps part of a
|
|
1840
|
+
* credential. The input is the buffered line, at most 8 KiB, whose unread rest lies far beyond the
|
|
1841
|
+
* kept head.
|
|
1842
|
+
*/
|
|
1843
|
+
private emitChildLine(
|
|
1844
|
+
logGenerationArg: IFlexChildLogGeneration,
|
|
1845
|
+
streamArg: IFlexSupervisorLogEntry['stream'],
|
|
1846
|
+
lineArg: string,
|
|
1847
|
+
): void {
|
|
1848
|
+
const line = plugins.smartstring.redact.redactAndBound(sanitizeFailureText(lineArg), {
|
|
1849
|
+
maxBytes: supervisorControllerLogLineBytes,
|
|
1850
|
+
keep: 'head',
|
|
1851
|
+
marker: logSuffix,
|
|
1852
|
+
}).text;
|
|
1853
|
+
this.writeControllerLog(`${this.describeChildGeneration(logGenerationArg)} ${streamArg}: ${line}`);
|
|
1854
|
+
}
|
|
1855
|
+
|
|
1856
|
+
/** Writes the rate window's suppression record, then its stdout tail and its stderr tail, and empties them. */
|
|
1857
|
+
private reportSuppressedLines(logGenerationArg: IFlexChildLogGeneration): void {
|
|
1858
|
+
if (logGenerationArg.suppressedLines === 0) return;
|
|
1859
|
+
const suppressed = logGenerationArg.suppressedLines;
|
|
1860
|
+
logGenerationArg.suppressedLines = 0;
|
|
1861
|
+
this.writeControllerLog(
|
|
1862
|
+
`${this.describeChildGeneration(logGenerationArg)}: ${suppressed} lines suppressed `
|
|
1863
|
+
+ `(more than ${supervisorControllerLogLinesPerWindow} lines in ${supervisorControllerLogWindowMs / 1000} s); `
|
|
1864
|
+
+ `the last ${supervisorControllerLogSuppressedTailLines} of each stream follow.`,
|
|
1865
|
+
);
|
|
1866
|
+
for (const stream of ['stdout', 'stderr'] as const) {
|
|
1867
|
+
const tail = logGenerationArg[stream].suppressedTail;
|
|
1868
|
+
logGenerationArg[stream].suppressedTail = [];
|
|
1869
|
+
for (const line of tail) this.emitChildLine(logGenerationArg, stream, line);
|
|
1870
|
+
}
|
|
1694
1871
|
}
|
|
1695
1872
|
}
|