agl 43.9.0 → 43.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.md +20 -0
- package/dist_serve/bundle.js +997 -975
- package/dist_serve/bundle.js.map +2 -2
- package/dist_ts/00_commitinfo_data.js +2 -2
- package/dist_ts/aglapi/aglapisurface.document.js +3 -3
- package/dist_ts/classes.accountsactions.d.ts +3 -1
- package/dist_ts/classes.accountsactions.js +17 -1
- package/dist_ts/classes.authstore.d.ts +6 -0
- package/dist_ts/classes.authstore.js +8 -1
- package/dist_ts/classes.controller.d.ts +7 -1
- package/dist_ts/classes.controller.js +39 -7
- package/dist_ts/classes.inferencegateway.js +7 -5
- package/dist_ts/classes.inferenceplacement.d.ts +1 -1
- package/dist_ts/classes.inferencerouter.d.ts +20 -3
- package/dist_ts/classes.inferencerouter.js +37 -10
- package/dist_ts/classes.inferenceservice.js +2 -2
- package/dist_ts/classes.mcpcallerregistry.d.ts +19 -1
- package/dist_ts/classes.mcpcallerregistry.js +33 -7
- package/dist_ts/classes.terminalcallers.d.ts +40 -0
- package/dist_ts/classes.terminalcallers.js +90 -0
- package/dist_ts/classes.terminalmanager.d.ts +20 -1
- package/dist_ts/classes.terminalmanager.js +36 -2
- package/dist_ts/classes.ticketnotifier.d.ts +2 -2
- package/dist_ts/classes.ticketnotifier.js +6 -4
- package/dist_ts/classes.ticketorderspread.d.ts +25 -0
- package/dist_ts/classes.ticketorderspread.js +52 -0
- package/dist_ts/classes.ticketstore.d.ts +60 -5
- package/dist_ts/classes.ticketstore.js +176 -40
- package/dist_ts/classes.workercallers.d.ts +40 -2
- package/dist_ts/classes.workercallers.js +156 -30
- package/dist_ts/fleet/classes.boardpump.d.ts +1 -1
- package/dist_ts/fleet/classes.boardpump.js +4 -2
- package/dist_ts/fleet/classes.claudeworkers.d.ts +18 -5
- package/dist_ts/fleet/classes.claudeworkers.js +105 -31
- package/dist_ts/fleet/classes.ticketeffects.d.ts +11 -0
- package/dist_ts/fleet/classes.ticketeffects.js +28 -1
- package/dist_ts/fleet/classes.workers.d.ts +23 -4
- package/dist_ts/fleet/classes.workers.js +65 -18
- package/dist_ts/fleet/functions.workervalidation.js +7 -4
- package/dist_ts/fleet/interfaces.workers.d.ts +16 -2
- package/dist_ts/fleet/interfaces.workers.js +8 -1
- package/dist_ts/functions.fleetworkercli.d.ts +3 -1
- package/dist_ts/functions.fleetworkercli.js +6 -2
- package/dist_ts/functions.inferencecli.js +4 -1
- package/dist_ts/functions.mcpfleetworkertools.js +3 -3
- package/dist_ts/functions.proxycapacity.d.ts +2 -1
- package/dist_ts/functions.proxycapacity.js +6 -3
- package/dist_ts/functions.proxycapacitycli.js +2 -2
- package/dist_ts/functions.workerprocessproof.d.ts +6 -0
- package/dist_ts/functions.workerprocessproof.js +8 -3
- package/dist_ts/interfaces.auth.d.ts +1 -1
- package/dist_ts/interfaces.auth.js +2 -1
- package/dist_ts/interfaces.flexipc.d.ts +3 -1
- package/dist_ts/interfaces.flexipc.js +1 -1
- package/dist_ts_authswitch/authority-contract.d.ts +25 -0
- package/dist_ts_authswitch/authority-contract.js +1 -1
- package/dist_ts_authswitch/authority-overview.d.ts +7 -0
- package/dist_ts_authswitch/authority-overview.js +10 -6
- package/dist_ts_authswitch/authority-placement.d.ts +5 -2
- package/dist_ts_authswitch/authority-placement.js +6 -3
- package/dist_ts_authswitch/classes.authoritybroker.d.ts +6 -0
- package/dist_ts_authswitch/classes.authoritybroker.js +26 -1
- package/dist_ts_authswitch/classes.authoritycli.d.ts +1 -0
- package/dist_ts_authswitch/classes.authoritycli.js +50 -4
- package/dist_ts_authswitch/classes.authorityclient.d.ts +3 -1
- package/dist_ts_authswitch/classes.authorityclient.js +9 -4
- package/dist_ts_authswitch/classes.authoritydaemon.js +7 -1
- package/dist_ts_authswitch/classes.authoritydatabase.js +5 -1
- package/dist_ts_authswitch/classes.authoritymodels.d.ts +7 -0
- package/dist_ts_authswitch/classes.authoritymodels.js +13 -6
- package/dist_ts_interfaces/fleetworkers.d.ts +26 -1
- package/dist_ts_interfaces/fleetworkers.js +1 -1
- package/dist_ts_interfaces/interfaces.d.ts +4 -2
- package/dist_ts_interfaces/interfaces.js +1 -1
- package/dist_ts_interfaces/mcprequests.d.ts +2 -1
- package/dist_ts_interfaces/mcprequests.js +3 -2
- package/dist_ts_interfaces/proxycapacity.d.ts +1 -1
- package/dist_ts_interfaces/requests.accounts.d.ts +19 -0
- package/dist_ts_interfaces/requests.accounts.js +1 -1
- package/dist_ts_interfaces/requests.d.ts +2 -2
- package/dist_ts_interfaces/requests.fleetworkers.d.ts +3 -1
- package/dist_ts_interfaces/requests.js +1 -1
- package/dist_ts_migration/classes.documentmigrationrunner.d.ts +8 -1
- package/dist_ts_migration/classes.documentmigrationrunner.js +11 -2
- package/dist_ts_migration/index.d.ts +1 -0
- package/dist_ts_migration/index.js +2 -1
- package/dist_ts_migration/v41_ticketclosedcolumn.js +7 -3
- package/dist_ts_migration/v43_ticketorderstride.d.ts +54 -0
- package/dist_ts_migration/v43_ticketorderstride.js +161 -0
- package/docs/authswitch.md +22 -0
- package/package.json +1 -1
- package/readme.md +71 -4
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/aglapi/aglapisurface.document.ts +2 -2
- package/ts/classes.accountsactions.ts +25 -0
- package/ts/classes.authstore.ts +9 -0
- package/ts/classes.controller.ts +37 -3
- package/ts/classes.inferencegateway.ts +6 -4
- package/ts/classes.inferenceplacement.ts +1 -1
- package/ts/classes.inferencerouter.ts +49 -10
- package/ts/classes.inferenceservice.ts +1 -1
- package/ts/classes.mcpcallerregistry.ts +49 -7
- package/ts/classes.terminalcallers.ts +91 -0
- package/ts/classes.terminalmanager.ts +41 -1
- package/ts/classes.ticketnotifier.ts +5 -3
- package/ts/classes.ticketorderspread.ts +50 -0
- package/ts/classes.ticketstore.ts +164 -35
- package/ts/classes.workercallers.ts +183 -30
- package/ts/fleet/classes.boardpump.ts +3 -2
- package/ts/fleet/classes.claudeworkers.ts +101 -29
- package/ts/fleet/classes.ticketeffects.ts +25 -0
- package/ts/fleet/classes.workers.ts +62 -17
- package/ts/fleet/functions.workervalidation.ts +6 -3
- package/ts/fleet/interfaces.workers.ts +16 -2
- package/ts/functions.fleetworkercli.ts +5 -1
- package/ts/functions.inferencecli.ts +2 -0
- package/ts/functions.mcpfleetworkertools.ts +2 -2
- package/ts/functions.proxycapacity.ts +5 -2
- package/ts/functions.proxycapacitycli.ts +1 -1
- package/ts/functions.workerprocessproof.ts +10 -2
- package/ts/interfaces.auth.ts +1 -0
- package/ts/interfaces.flexipc.ts +3 -1
- package/ts_authswitch/authority-contract.ts +20 -0
- package/ts_authswitch/authority-overview.ts +17 -6
- package/ts_authswitch/authority-placement.ts +7 -3
- package/ts_authswitch/classes.authoritybroker.ts +24 -0
- package/ts_authswitch/classes.authoritycli.ts +58 -4
- package/ts_authswitch/classes.authorityclient.ts +11 -4
- package/ts_authswitch/classes.authoritydaemon.ts +8 -1
- package/ts_authswitch/classes.authoritydatabase.ts +3 -0
- package/ts_authswitch/classes.authoritymodels.ts +12 -4
- package/ts_interfaces/fleetworkers.ts +26 -1
- package/ts_interfaces/interfaces.ts +4 -2
- package/ts_interfaces/mcprequests.ts +2 -1
- package/ts_interfaces/proxycapacity.ts +1 -1
- package/ts_interfaces/requests.accounts.ts +17 -0
- package/ts_interfaces/requests.fleetworkers.ts +3 -1
- package/ts_interfaces/requests.ts +2 -0
- package/ts_migration/classes.documentmigrationrunner.ts +12 -1
- package/ts_migration/index.ts +1 -0
- package/ts_migration/v41_ticketclosedcolumn.ts +6 -2
- package/ts_migration/v43_ticketorderstride.ts +156 -0
- package/ts_web/00_commitinfo_data.ts +1 -1
- package/ts_web/accounts.templates.ts +12 -2
- package/ts_web/classes.accountsreadmodel.ts +42 -6
- package/ts_web/elements.accounts.ts +45 -3
- package/ts_web/elements.harnesscontrollerapp.ts +6 -1
|
@@ -89,6 +89,12 @@ export interface IInferenceAccount {
|
|
|
89
89
|
* account of the pool without one, as long as it takes the request (`inferencePlacementOrder`).
|
|
90
90
|
*/
|
|
91
91
|
priorityUntil: string | null;
|
|
92
|
+
/**
|
|
93
|
+
* Since when the owner holds the account paused (`IAuthSwitchAccount.pausedAt`, AGL-1933); `null` while it is not. A
|
|
94
|
+
* paused account is in no pool: no new session goes to it, a session on it moves (`account_paused`), and a request
|
|
95
|
+
* that could reach only paused accounts is refused as `accounts_paused`.
|
|
96
|
+
*/
|
|
97
|
+
pausedAt: string | null;
|
|
92
98
|
/**
|
|
93
99
|
* An API key's monthly spend limit in micro-dollars (`IAuthSwitchAccount.spendLimitMicroUsd`); `null` without one,
|
|
94
100
|
* and for a sign-in. It decides only whether a key whose spend the proxy holds no reading of may take work: one with
|
|
@@ -756,6 +762,12 @@ export interface IInferenceAccountStatus {
|
|
|
756
762
|
* account ranks before every account without one. A proxy of an earlier AGL reports none.
|
|
757
763
|
*/
|
|
758
764
|
priorityUntil?: string | null;
|
|
765
|
+
/**
|
|
766
|
+
* Since when the owner holds the account paused (`IInferenceAccount.pausedAt`), or `null`: it takes no new session
|
|
767
|
+
* and has no placement rank, whatever `state` says of its usage; `stateText` says it is paused. A proxy of an earlier
|
|
768
|
+
* AGL reports none.
|
|
769
|
+
*/
|
|
770
|
+
pausedAt?: string | null;
|
|
759
771
|
/** The account's placement score in percent per hour (`IAuthSwitchPlacement.score`); `null` without a counted window. */
|
|
760
772
|
placementScore: number | null;
|
|
761
773
|
/**
|
|
@@ -988,7 +1000,7 @@ const emptySelection = (accountsArg: number, pinnedArg: boolean): ISelectionStat
|
|
|
988
1000
|
const accountPlacement = (readingArg: plugins.authswitchAuthority.IAuthSwitchUsageSnapshot['reading'],
|
|
989
1001
|
accountArg: IInferenceAccount, nowArg: number): plugins.authswitchPlacement.IAuthSwitchPlacement =>
|
|
990
1002
|
plugins.authswitchPlacement.authSwitchPlacement(readingArg, accountArg.maxUsagePercent, accountArg.lapsesAt,
|
|
991
|
-
accountArg.renewsAt, nowArg, accountArg.reserveLiftUntil);
|
|
1003
|
+
accountArg.renewsAt, nowArg, accountArg.reserveLiftUntil, accountArg.pausedAt);
|
|
992
1004
|
|
|
993
1005
|
/** An account of a pool as a new session is placed on it: its credential tier and its rank (`IAuthSwitchPlacementRank`). */
|
|
994
1006
|
interface IInferencePlacementEntry {
|
|
@@ -1056,12 +1068,13 @@ export class InferenceRouter {
|
|
|
1056
1068
|
/**
|
|
1057
1069
|
* The policy's accounts of its provider and credential tiers (`tiers`), and every eligible account's id of that pool:
|
|
1058
1070
|
* an account absent from both left the pool. An account whose lapse date passed is in neither -- it is never routed to
|
|
1059
|
-
* again -- and is named in `lapsed`.
|
|
1071
|
+
* again -- and is named in `lapsed`. An account its owner holds paused (AGL-1933) is in neither as well, for as long
|
|
1072
|
+
* as it is paused, and is named in `paused`. A pool that names its credential serves that kind alone: Flex's API-key pool
|
|
1060
1073
|
* never takes a Claude account. One that names none spans both, and a session placed on a key returns to a Claude
|
|
1061
1074
|
* account once one can take its request.
|
|
1062
1075
|
*/
|
|
1063
1076
|
private async pool(policyArg: IInferenceRoutingPolicy, signalArg: AbortSignal): Promise<{ accounts: IInferenceAccount[];
|
|
1064
|
-
eligible: Set<string>; lapsed: Set<string>; tiers: readonly TInferenceCredential[] }> {
|
|
1077
|
+
eligible: Set<string>; lapsed: Set<string>; paused: Set<string>; tiers: readonly TInferenceCredential[] }> {
|
|
1065
1078
|
const everyAccount = await this.options.accounts(signalArg);
|
|
1066
1079
|
this.apiKeyAccounts = new Set(everyAccount.filter((account) => account.credential === 'api_key')
|
|
1067
1080
|
.map((account) => account.id));
|
|
@@ -1074,7 +1087,9 @@ export class InferenceRouter {
|
|
|
1074
1087
|
&& !(policyArg.claudeCode && accountArg.credential === 'api_key' && !accountArg.claudeCode);
|
|
1075
1088
|
const lapsed = new Set(everyAccount.filter((account) => inPool(account)
|
|
1076
1089
|
&& plugins.authswitchPlacement.authSwitchAccountLifecycle(account.lapsesAt, now) === 'lapsed').map((account) => account.id));
|
|
1077
|
-
const
|
|
1090
|
+
const paused = new Set(everyAccount.filter((account) => inPool(account) && !lapsed.has(account.id)
|
|
1091
|
+
&& account.pausedAt !== null).map((account) => account.id));
|
|
1092
|
+
const allAccounts = everyAccount.filter((account) => inPool(account) && !lapsed.has(account.id) && !paused.has(account.id));
|
|
1078
1093
|
const accounts = allAccounts.filter((account) => !policyArg.accountIds || policyArg.accountIds.includes(account.id));
|
|
1079
1094
|
if (accounts.length > 64) throw new InferenceRoutingError(503, 'pool_too_large',
|
|
1080
1095
|
'Select at most 64 accounts for this gateway.');
|
|
@@ -1085,21 +1100,38 @@ export class InferenceRouter {
|
|
|
1085
1100
|
for (const id of this.cooldowns.keys()) if (!known.has(id)) this.cooldowns.delete(id);
|
|
1086
1101
|
for (const id of this.modelCooldowns.keys()) if (!known.has(id)) this.modelCooldowns.delete(id);
|
|
1087
1102
|
for (const id of this.noted.keys()) if (!known.has(id)) this.noted.delete(id);
|
|
1088
|
-
return { accounts, eligible: new Set(allAccounts.map((account) => account.id)), lapsed, tiers };
|
|
1103
|
+
return { accounts, eligible: new Set(allAccounts.map((account) => account.id)), lapsed, paused, tiers };
|
|
1089
1104
|
}
|
|
1090
1105
|
|
|
1091
1106
|
/**
|
|
1092
1107
|
* The pool for one request of a session served by `accountIdArg`, which it refuses when that account cannot serve
|
|
1093
1108
|
* the session any more: one no longer eligible at all -- removed, signed out, or taken by a native tool -- left the
|
|
1094
1109
|
* pool, and one the policy's account restriction leaves out is outside it, as the operator chose. Either way the
|
|
1095
|
-
* session moves to an account of the pool. So it does from an account that lapsed
|
|
1096
|
-
* the session waiting.
|
|
1110
|
+
* session moves to an account of the pool. So it does from an account that lapsed, and from one its owner paused
|
|
1111
|
+
* (`account_paused`). A plan the policy leaves out keeps the session waiting.
|
|
1112
|
+
*
|
|
1113
|
+
* A request that reaches no account only because its accounts are paused -- every account of the pool, or every one
|
|
1114
|
+
* the client's account restriction names, as Flex's chosen account is -- is refused as `accounts_paused`, which says
|
|
1115
|
+
* so and how to end it. Only a request is refused so: the model lists read the pool without it.
|
|
1097
1116
|
*/
|
|
1098
1117
|
private async sessionPool(policyArg: IInferenceRoutingPolicy, signalArg: AbortSignal,
|
|
1099
1118
|
accountIdArg: string | undefined): Promise<{ accounts: IInferenceAccount[]; eligible: Set<string>;
|
|
1100
1119
|
tiers: readonly TInferenceCredential[] }> {
|
|
1101
1120
|
const pool = await this.pool(policyArg, signalArg);
|
|
1121
|
+
if (!pool.accounts.length) {
|
|
1122
|
+
const paused = [...pool.paused].filter((idArg) => !policyArg.accountIds || policyArg.accountIds.includes(idArg));
|
|
1123
|
+
if (paused.length) {
|
|
1124
|
+
throw new InferenceRoutingError(503, 'accounts_paused', policyArg.accountIds
|
|
1125
|
+
? `${paused.length === 1 ? 'The account this client uses is' : 'The accounts this client uses are'} paused in AGL `
|
|
1126
|
+
+ 'Accounts, so AGL places nothing on ' + (paused.length === 1 ? 'it' : 'them') + '. Resume '
|
|
1127
|
+
+ (paused.length === 1 ? 'it' : 'one') + ' there, or choose another account.'
|
|
1128
|
+
: 'Every account of this pool is paused in AGL Accounts, so AGL places nothing on them. Resume one there.');
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
1102
1131
|
if (!accountIdArg) return pool;
|
|
1132
|
+
if (pool.paused.has(accountIdArg)) {
|
|
1133
|
+
throw new InferenceRoutingError(503, 'account_paused', 'The account serving this session is paused by its owner.');
|
|
1134
|
+
}
|
|
1103
1135
|
if (pool.lapsed.has(accountIdArg)) {
|
|
1104
1136
|
throw new InferenceRoutingError(503, 'account_lapsed', 'The account serving this session lapsed; its subscription ended.');
|
|
1105
1137
|
}
|
|
@@ -2034,7 +2066,7 @@ export class InferenceRouter {
|
|
|
2034
2066
|
public accountStatus(accountsArg: readonly IInferenceAccount[],
|
|
2035
2067
|
heldArg: (accountIdArg: string) => plugins.authswitchAuthority.IAuthSwitchUsageSnapshot | undefined): IInferenceAccountStatus[] {
|
|
2036
2068
|
const now = this.now();
|
|
2037
|
-
const
|
|
2069
|
+
const judge = (account: IInferenceAccount): IInferenceAccountStatus => {
|
|
2038
2070
|
const apiKey = account.credential === 'api_key';
|
|
2039
2071
|
// A key's usage is its month's spend, with the payment problem the account service holds on it.
|
|
2040
2072
|
const held = heldArg(account.id);
|
|
@@ -2048,6 +2080,7 @@ export class InferenceRouter {
|
|
|
2048
2080
|
lifecycle: placement.lifecycle, lapsesAt: account.lapsesAt,
|
|
2049
2081
|
renewal: plugins.authswitchRenewal.authSwitchRenewal(account.renewsAt, now),
|
|
2050
2082
|
priorityUntil: plugins.authswitchPriority.authSwitchPriorityUntil(account.priorityUntil, now),
|
|
2083
|
+
pausedAt: account.pausedAt,
|
|
2051
2084
|
placementScore: placement.score, placementRank: null,
|
|
2052
2085
|
lastResort: placement.eligibility === 'last_resort', claudeCode: account.claudeCode,
|
|
2053
2086
|
readingSource: held?.observation ? 'headers' as const : held?.reading ? 'poll' as const : null,
|
|
@@ -2083,14 +2116,20 @@ export class InferenceRouter {
|
|
|
2083
2116
|
: 'available (no usage reported)'}${this.modelRestText(account.id, held, now)}`,
|
|
2084
2117
|
plan: limits.plan || null, headroom: limits.limit?.headroom ?? null, crossing: limits.limit?.crossing ?? [],
|
|
2085
2118
|
resetAt: state === 'cooling' ? new Date(cooldown).toISOString() : limits.limit?.resetAt ?? null };
|
|
2119
|
+
};
|
|
2120
|
+
const statuses = accountsArg.map((account): IInferenceAccountStatus => {
|
|
2121
|
+
const judged = judge(account);
|
|
2122
|
+
// Paused by its owner (AGL-1933): it says so instead of what its usage would allow, and names no reset to wait for.
|
|
2123
|
+
return account.pausedAt === null ? judged : { ...judged, resetAt: null,
|
|
2124
|
+
stateText: `paused by the owner since ${account.pausedAt}; takes no new session until it is resumed` };
|
|
2086
2125
|
});
|
|
2087
2126
|
// Each provider's accounts that take a new session now, in the order placement takes them: a priority account first,
|
|
2088
|
-
// then the best credential tier.
|
|
2127
|
+
// then the best credential tier. A paused account takes none.
|
|
2089
2128
|
const order = inferencePlacementOrder((accountIdArg) => this.active.get(accountIdArg) ?? 0);
|
|
2090
2129
|
const tiers = inferenceCredentialTiers(undefined);
|
|
2091
2130
|
for (const provider of new Set(statuses.map((statusArg) => statusArg.provider))) {
|
|
2092
2131
|
statuses.filter((statusArg) => statusArg.provider === provider && statusArg.state === 'available'
|
|
2093
|
-
&& statusArg.lifecycle !== 'lapsed')
|
|
2132
|
+
&& statusArg.lifecycle !== 'lapsed' && !statusArg.pausedAt)
|
|
2094
2133
|
.map((statusArg) => ({ status: statusArg, tier: tiers.indexOf(statusArg.credential), rank: { id: statusArg.id,
|
|
2095
2134
|
priority: Boolean(statusArg.priorityUntil), score: statusArg.placementScore, lastResort: statusArg.lastResort,
|
|
2096
2135
|
headroom: statusArg.headroom ?? unknownHeadroom } }))
|
|
@@ -159,7 +159,7 @@ export class InferenceService {
|
|
|
159
159
|
const login = findAuthorityGatewayLogin(projection, account.id);
|
|
160
160
|
return pool ? [{ id: account.id, label: account.label, revision: String(account.revision), providerId: pool.provider,
|
|
161
161
|
credential: pool.credential, maxUsagePercent: account.maxUsagePercent, reserveLiftUntil: account.reserveLiftUntil, lapsesAt: account.lapsesAt,
|
|
162
|
-
renewsAt: account.renewsAt, priorityUntil: account.priorityUntil, spendLimitMicroUsd: account.spendLimitMicroUsd, claudeCode: account.claudeCode, loginId: typeof login === 'string' ? null : login.id }] : [];
|
|
162
|
+
renewsAt: account.renewsAt, priorityUntil: account.priorityUntil, pausedAt: account.pausedAt, spendLimitMicroUsd: account.spendLimitMicroUsd, claudeCode: account.claudeCode, loginId: typeof login === 'string' ? null : login.id }] : [];
|
|
163
163
|
});
|
|
164
164
|
}
|
|
165
165
|
|
|
@@ -11,6 +11,10 @@ import {
|
|
|
11
11
|
* process is a controller child — terminals, the OpenCode server — so no credential can outlive
|
|
12
12
|
* the controller generation that minted it, and nothing here belongs in persistence.
|
|
13
13
|
*
|
|
14
|
+
* A terminal's credential also records the terminal's PTY child once it runs (`bindTerminalProcess`):
|
|
15
|
+
* every process started in the terminal inherits the credential, a daemon that leaves it as well, so
|
|
16
|
+
* admission honours it only for a process that still runs inside the terminal (AGL-1891).
|
|
17
|
+
*
|
|
14
18
|
* A credential is secret material. It never reaches audit records, error text, responses or logs:
|
|
15
19
|
* only the derived {@link IControllerMcpCaller.auditPeerId} does, and that is a one-way digest.
|
|
16
20
|
*/
|
|
@@ -50,9 +54,21 @@ const credentialKey = (credentialArg: string): string => plugins.crypto
|
|
|
50
54
|
.update(credentialArg, 'utf8')
|
|
51
55
|
.digest('base64url');
|
|
52
56
|
|
|
57
|
+
/** A process as the kernel identifies it for as long as the host runs: its PID and its start time in clock ticks. */
|
|
58
|
+
export interface IControllerMcpCallerProcess {
|
|
59
|
+
pid: number;
|
|
60
|
+
startTicks: string;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
interface ICallerEntry {
|
|
64
|
+
caller: IControllerMcpCaller;
|
|
65
|
+
/** A terminal's PTY child, once its start recorded it: the process the credential is honoured inside. */
|
|
66
|
+
process?: IControllerMcpCallerProcess;
|
|
67
|
+
}
|
|
68
|
+
|
|
53
69
|
/** Minted credentials, keyed by digest so the raw value is never retained. */
|
|
54
70
|
export class ControllerMcpCallerRegistry {
|
|
55
|
-
private readonly callersByCredentialKey = new Map<string,
|
|
71
|
+
private readonly callersByCredentialKey = new Map<string, ICallerEntry>();
|
|
56
72
|
|
|
57
73
|
public mint(subjectArg: Exclude<TControllerMcpSubject, { kind: 'anonymous' | 'worker_launch' }>): string {
|
|
58
74
|
return this.store(subjectArg);
|
|
@@ -65,19 +81,45 @@ export class ControllerMcpCallerRegistry {
|
|
|
65
81
|
private store(subjectArg: Exclude<TControllerMcpSubject, { kind: 'anonymous' }>): string {
|
|
66
82
|
const credential = plugins.crypto.randomBytes(credentialByteLength).toString('base64url');
|
|
67
83
|
this.callersByCredentialKey.set(credentialKey(credential), {
|
|
68
|
-
subject: subjectArg,
|
|
69
|
-
auditPeerId: auditPeerIdFor(subjectArg),
|
|
84
|
+
caller: { subject: subjectArg, auditPeerId: auditPeerIdFor(subjectArg) },
|
|
70
85
|
});
|
|
71
86
|
return credential;
|
|
72
87
|
}
|
|
73
88
|
|
|
74
|
-
/**
|
|
89
|
+
/**
|
|
90
|
+
* An unknown, malformed or revoked credential degrades to anonymous, never to an error. This names the subject a
|
|
91
|
+
* credential was minted for; whether its presenter may act as it is admission's question (`ControllerWorkerCallers.admit`).
|
|
92
|
+
*/
|
|
75
93
|
public resolve(credentialArg: string | undefined): IControllerMcpCaller {
|
|
76
94
|
if (credentialArg === undefined) return anonymousControllerMcpCaller;
|
|
77
|
-
return this.callersByCredentialKey.get(credentialKey(credentialArg))
|
|
95
|
+
return this.callersByCredentialKey.get(credentialKey(credentialArg))?.caller
|
|
78
96
|
?? anonymousControllerMcpCaller;
|
|
79
97
|
}
|
|
80
98
|
|
|
99
|
+
/**
|
|
100
|
+
* Records the PTY child a terminal start spawned with `credentialArg` in its environment as the process that
|
|
101
|
+
* credential is honoured inside (AGL-1891). Until it is made, the credential is honoured for no process; a revoked
|
|
102
|
+
* credential, or one minted for anything but a terminal, records nothing.
|
|
103
|
+
*/
|
|
104
|
+
public bindTerminalProcess(credentialArg: string, processArg: IControllerMcpCallerProcess): void {
|
|
105
|
+
const entry = this.callersByCredentialKey.get(credentialKey(credentialArg));
|
|
106
|
+
if (entry?.caller.subject.kind !== 'terminal') return;
|
|
107
|
+
entry.process = Object.freeze({ pid: processArg.pid, startTicks: processArg.startTicks });
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** The PTY child a terminal's credential is honoured inside, once its start recorded it; nothing for any other credential. */
|
|
111
|
+
public terminalProcess(credentialArg: string): IControllerMcpCallerProcess | undefined {
|
|
112
|
+
const entry = this.callersByCredentialKey.get(credentialKey(credentialArg));
|
|
113
|
+
return entry?.caller.subject.kind === 'terminal' ? entry.process : undefined;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** The PTY children of every terminal whose credential is live and recorded one. */
|
|
117
|
+
public terminalProcesses(): IControllerMcpCallerProcess[] {
|
|
118
|
+
return [...this.callersByCredentialKey.values()].flatMap((entryArg) => (
|
|
119
|
+
entryArg.caller.subject.kind === 'terminal' && entryArg.process ? [entryArg.process] : []
|
|
120
|
+
));
|
|
121
|
+
}
|
|
122
|
+
|
|
81
123
|
public revokeCredential(credentialArg: string): void {
|
|
82
124
|
this.callersByCredentialKey.delete(credentialKey(credentialArg));
|
|
83
125
|
}
|
|
@@ -114,8 +156,8 @@ export class ControllerMcpCallerRegistry {
|
|
|
114
156
|
}
|
|
115
157
|
|
|
116
158
|
private revokeWhere(predicateArg: (subjectArg: TControllerMcpSubject) => boolean): void {
|
|
117
|
-
for (const [key,
|
|
118
|
-
if (predicateArg(caller.subject)) this.callersByCredentialKey.delete(key);
|
|
159
|
+
for (const [key, entry] of [...this.callersByCredentialKey]) {
|
|
160
|
+
if (predicateArg(entry.caller.subject)) this.callersByCredentialKey.delete(key);
|
|
119
161
|
}
|
|
120
162
|
}
|
|
121
163
|
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import {
|
|
2
|
+
anonymousControllerMcpCaller, type ControllerMcpCallerRegistry, type IControllerMcpCallerProcess,
|
|
3
|
+
} from './classes.mcpcallerregistry.js';
|
|
4
|
+
import {
|
|
5
|
+
isWorkerCallerProofRefusal, type proveConnectionProcessAncestry, type IWorkerCallerConnection, type IWorkerProcessSnapshot,
|
|
6
|
+
} from './functions.workerprocessproof.js';
|
|
7
|
+
import type { IControllerMcpCaller } from '../ts_interfaces/index.js';
|
|
8
|
+
|
|
9
|
+
/** The bridges remembered per terminal, the newest first; the oldest give way. */
|
|
10
|
+
const maximumBridgesPerTerminal = 16;
|
|
11
|
+
/** How far up a bridge's parent chain the terminal's PTY child is looked for. */
|
|
12
|
+
const ancestryDepth = 32;
|
|
13
|
+
|
|
14
|
+
const processKey = (processArg: IControllerMcpCallerProcess): string => `${processArg.pid}:${processArg.startTicks}`;
|
|
15
|
+
const sameProcess = (leftArg: IControllerMcpCallerProcess, rightArg: IControllerMcpCallerProcess): boolean => (
|
|
16
|
+
leftArg.pid === rightArg.pid && leftArg.startTicks === rightArg.startTicks
|
|
17
|
+
);
|
|
18
|
+
|
|
19
|
+
export interface IControllerTerminalCallerOptions {
|
|
20
|
+
registry: ControllerMcpCallerRegistry;
|
|
21
|
+
/** The caller's process and its parent chain, proved to hold the request's loopback connection. */
|
|
22
|
+
proveAncestry: typeof proveConnectionProcessAncestry;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Who presents a terminal's MCP caller credential as the terminal (AGL-1891): only a process that runs inside it. The
|
|
27
|
+
* terminal's PTY child gets the credential in its environment, and every process started from it inherits it, a daemon
|
|
28
|
+
* too (Claude Code's background-job daemon started from a terminal shell carries it for its whole life, and so does
|
|
29
|
+
* every job it runs). So a call presenting it is admitted only from a bridge that told the controller its PID at its
|
|
30
|
+
* process enrollment (`{ bridgePid }`), holds the call's own loopback connection, and still descends from the PTY child
|
|
31
|
+
* the terminal's start recorded (PID and start), all read from the kernel at that call. Any other presenter calls
|
|
32
|
+
* anonymously, as with an unknown credential: a daemon that left the terminal (re-parented to init), a process that
|
|
33
|
+
* never ran in it, a bridge whose PID another process claims, and every process once the recorded child is gone.
|
|
34
|
+
*/
|
|
35
|
+
export class ControllerTerminalCallers {
|
|
36
|
+
/**
|
|
37
|
+
* The bridges whose enrollment proved them inside a terminal, by the key of the terminal's PTY child. Only a hint of
|
|
38
|
+
* whose connection a call may come over: admission proves every call afresh.
|
|
39
|
+
*/
|
|
40
|
+
private readonly bridges = new Map<string, IControllerMcpCallerProcess[]>();
|
|
41
|
+
|
|
42
|
+
constructor(private readonly options: IControllerTerminalCallerOptions) {}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* At a bridge's process enrollment the fleet answered no credential (so the bridge calls with the one it inherited):
|
|
46
|
+
* remembers the bridge as a process of the terminal whose recorded PTY child it descends from, once it proved that it
|
|
47
|
+
* holds the enrollment's connection. A bridge inside no running terminal, or one whose proof is refused, is remembered
|
|
48
|
+
* nowhere. Only an unexpected failure to read the processes throws.
|
|
49
|
+
*/
|
|
50
|
+
public async observe(bridgePidArg: number, connectionArg: IWorkerCallerConnection | undefined): Promise<void> {
|
|
51
|
+
const terminals = this.options.registry.terminalProcesses();
|
|
52
|
+
const live = new Set(terminals.map(processKey));
|
|
53
|
+
// A terminal that stopped revoked its credential: its bridges are forgotten with it.
|
|
54
|
+
for (const key of [...this.bridges.keys()]) if (!live.has(key)) this.bridges.delete(key);
|
|
55
|
+
if (terminals.length === 0) return;
|
|
56
|
+
let chain: IWorkerProcessSnapshot[];
|
|
57
|
+
try { chain = await this.options.proveAncestry(connectionArg, bridgePidArg, ancestryDepth); } catch (errorArg) {
|
|
58
|
+
if (isWorkerCallerProofRefusal(errorArg)) return;
|
|
59
|
+
throw errorArg;
|
|
60
|
+
}
|
|
61
|
+
const terminal = chain.find(processArg => terminals.some(terminalArg => sameProcess(processArg, terminalArg)));
|
|
62
|
+
if (!terminal) return;
|
|
63
|
+
const bridge = { pid: chain[0]!.pid, startTicks: chain[0]!.startTicks };
|
|
64
|
+
const key = processKey(terminal);
|
|
65
|
+
const remembered = (this.bridges.get(key) ?? []).filter(entryArg => entryArg.pid !== bridge.pid);
|
|
66
|
+
this.bridges.set(key, [bridge, ...remembered].slice(0, maximumBridgesPerTerminal));
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* The terminal caller `callerArg` (what `credentialArg` resolves to) when the call's connection is held by a bridge
|
|
71
|
+
* its enrollment proved inside that terminal, the same process still, and that bridge still descends from the
|
|
72
|
+
* terminal's recorded PTY child, unchanged since the spawn; the anonymous caller otherwise.
|
|
73
|
+
*/
|
|
74
|
+
public async admit(credentialArg: string, callerArg: IControllerMcpCaller, connectionArg: IWorkerCallerConnection | undefined): Promise<IControllerMcpCaller> {
|
|
75
|
+
const terminal = this.options.registry.terminalProcess(credentialArg);
|
|
76
|
+
if (!terminal) return anonymousControllerMcpCaller;
|
|
77
|
+
for (const bridge of [...this.bridges.get(processKey(terminal)) ?? []]) {
|
|
78
|
+
let chain: IWorkerProcessSnapshot[];
|
|
79
|
+
try { chain = await this.options.proveAncestry(connectionArg, bridge.pid, ancestryDepth); } catch (errorArg) {
|
|
80
|
+
// Not this bridge's connection, or the bridge is gone: the next remembered one may hold it.
|
|
81
|
+
if (isWorkerCallerProofRefusal(errorArg)) continue;
|
|
82
|
+
throw errorArg;
|
|
83
|
+
}
|
|
84
|
+
// The connection is this PID's: it is the presenter, and no other bridge is asked.
|
|
85
|
+
if (sameProcess(chain[0]!, bridge) && chain.some(processArg => sameProcess(processArg, terminal))
|
|
86
|
+
&& this.options.registry.resolve(credentialArg) === callerArg) return callerArg;
|
|
87
|
+
return anonymousControllerMcpCaller;
|
|
88
|
+
}
|
|
89
|
+
return anonymousControllerMcpCaller;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
@@ -11,6 +11,8 @@ import {
|
|
|
11
11
|
import type { IControllerResourceDocument } from './interfaces.projects.js';
|
|
12
12
|
import { ControllerTerminalMirror } from './classes.terminalmirror.js';
|
|
13
13
|
import { createSanitizedRuntimeEnvironment } from './functions.runtimeenvironment.js';
|
|
14
|
+
import { isWorkerCallerProofRefusal, readWorkerProcessSnapshot, type IWorkerProcessSnapshot } from './functions.workerprocessproof.js';
|
|
15
|
+
import type { IControllerMcpCallerProcess } from './classes.mcpcallerregistry.js';
|
|
14
16
|
import {
|
|
15
17
|
assertControllerTerminalAgentSessionIsFree,
|
|
16
18
|
newControllerTerminalAgentSessionId,
|
|
@@ -130,6 +132,17 @@ export interface IControllerTerminalManagerOptions {
|
|
|
130
132
|
resourceId: string;
|
|
131
133
|
agentSessionId?: string;
|
|
132
134
|
}) => string | undefined;
|
|
135
|
+
/**
|
|
136
|
+
* Records the PTY child of a start whose environment carries a minted caller credential (AGL-1891): that credential,
|
|
137
|
+
* and the child's PID and start time, read from the kernel right after the spawn. Every process started in the
|
|
138
|
+
* terminal inherits the credential, a daemon that leaves the terminal as well, so the controller honours it only for
|
|
139
|
+
* a process that descends from this child; a start that records none (its child already exited) leaves the credential
|
|
140
|
+
* honoured for no process.
|
|
141
|
+
*/
|
|
142
|
+
recordCallerProcess?: (inputArg: {
|
|
143
|
+
credential: string;
|
|
144
|
+
process: IControllerMcpCallerProcess;
|
|
145
|
+
}) => void;
|
|
133
146
|
/**
|
|
134
147
|
* Starts a Claude chat's use of the provider pool it runs on (`IControllerTerminalAgentMetadata.provider`) through
|
|
135
148
|
* AGL's inference proxy. Without it, a chat on a pool does not start.
|
|
@@ -680,7 +693,8 @@ export class ControllerTerminalManager {
|
|
|
680
693
|
/**
|
|
681
694
|
* The sanitized shell environment plus this start's caller credential. The credential is written
|
|
682
695
|
* explicitly here and is never inheritable, so a value present in the controller's own
|
|
683
|
-
* environment can never reach a child and impersonate a terminal.
|
|
696
|
+
* environment can never reach a child and impersonate a terminal. Inside the terminal every
|
|
697
|
+
* process inherits it; the controller honours it only for those that still run there (AGL-1891).
|
|
684
698
|
*/
|
|
685
699
|
private spawnEnvironmentFor(
|
|
686
700
|
resourceArg: IControllerResourceDocument,
|
|
@@ -738,6 +752,8 @@ export class ControllerTerminalManager {
|
|
|
738
752
|
else ptyProcess.resume();
|
|
739
753
|
},
|
|
740
754
|
});
|
|
755
|
+
// A start's caller credential is in no other environment: its PTY child is the process it is honoured inside.
|
|
756
|
+
const callerCredential = inputArg.environment[controllerMcpCallerCredentialEnvironmentVariable];
|
|
741
757
|
let execution: plugins.smartshell.IExecResultPtyStreaming;
|
|
742
758
|
try {
|
|
743
759
|
execution = await this.ptySpawner.execSpawnStreamingInteractiveControlPty(
|
|
@@ -817,10 +833,34 @@ export class ControllerTerminalManager {
|
|
|
817
833
|
await this.closeEntry(createdEntry, Date.now() + this.cleanupTimeoutMs);
|
|
818
834
|
throw new Error('The project stopped accepting terminal operations.');
|
|
819
835
|
}
|
|
836
|
+
if (callerCredential !== undefined) {
|
|
837
|
+
try {
|
|
838
|
+
await this.recordCallerProcess(callerCredential, execution.ptyProcess.pid);
|
|
839
|
+
} catch (errorArg) {
|
|
840
|
+
await this.closeEntry(createdEntry, Date.now() + this.cleanupTimeoutMs);
|
|
841
|
+
throw errorArg;
|
|
842
|
+
}
|
|
843
|
+
}
|
|
820
844
|
this.options.onTerminalsChanged(inputArg.projectId);
|
|
821
845
|
return toTerminalDto(createdEntry.terminal);
|
|
822
846
|
}
|
|
823
847
|
|
|
848
|
+
/**
|
|
849
|
+
* Records this start's PTY child as the process its caller credential `credentialArg` is honoured inside (AGL-1891):
|
|
850
|
+
* read once from the kernel right after the spawn, and only while it is this controller's own child. A child that
|
|
851
|
+
* already exited is not recorded, which leaves the credential honoured for no process.
|
|
852
|
+
*/
|
|
853
|
+
private async recordCallerProcess(credentialArg: string, pidArg: number): Promise<void> {
|
|
854
|
+
if (!this.options.recordCallerProcess) return;
|
|
855
|
+
let child: IWorkerProcessSnapshot;
|
|
856
|
+
try { child = await readWorkerProcessSnapshot(pidArg); } catch (errorArg) {
|
|
857
|
+
if (isWorkerCallerProofRefusal(errorArg)) return;
|
|
858
|
+
throw errorArg;
|
|
859
|
+
}
|
|
860
|
+
if (child.parentPid !== process.pid) return;
|
|
861
|
+
this.options.recordCallerProcess({ credential: credentialArg, process: { pid: child.pid, startTicks: child.startTicks } });
|
|
862
|
+
}
|
|
863
|
+
|
|
824
864
|
/** OSC titles are untrusted display metadata: bounded, coalesced and never persisted. */
|
|
825
865
|
private acceptAutomaticTitle(entryArg: ITerminalEntry, titleArg: string): void {
|
|
826
866
|
if (this.state !== 'open' || !entryArg.automaticTitle || entryArg.closing || entryArg.finalState !== 'running') return;
|
|
@@ -3,6 +3,7 @@ import type { IFleetEvent } from '../ts_interfaces/fleetworkers.js';
|
|
|
3
3
|
import type { ITicketChange, ITicketObservation } from './classes.ticketstore.js';
|
|
4
4
|
import { quoteAgentMessageHeaderText } from './functions.agentmessage.js';
|
|
5
5
|
import { boardPumpActor } from './fleet/classes.boardpump.js';
|
|
6
|
+
import { workerIdentityActor } from './fleet/classes.ticketeffects.js';
|
|
6
7
|
|
|
7
8
|
export interface ITicketNotifierOptions {
|
|
8
9
|
/** Steers the live native session (`harness:nativeId`) with the notice; rejects when it was not admitted. */
|
|
@@ -124,8 +125,8 @@ export const ticketNoticeText = (changesArg: ITicketChange[], recipientArg: stri
|
|
|
124
125
|
/**
|
|
125
126
|
* Tells assignees about changes others make to their tickets, and the session that asked a question about its answer. Changes to one ticket for one session gather for a
|
|
126
127
|
* short window and then go out as one notice, at most one per interval, steered into the live session. A session
|
|
127
|
-
* never hears of its own changes, and nobody hears of the board pump's
|
|
128
|
-
* fleet event stream.
|
|
128
|
+
* never hears of its own changes, and nobody hears of the board pump's or of a notice that a worker calls without its
|
|
129
|
+
* identity. Every change but the driver's own also goes to the fleet event stream.
|
|
129
130
|
*/
|
|
130
131
|
export class TicketNotifier {
|
|
131
132
|
private readonly batches = new Map<string, ITicketNoticeBatch>();
|
|
@@ -146,7 +147,8 @@ export class TicketNotifier {
|
|
|
146
147
|
// A reorder is the side effect of another ticket's move; the mover's ticket carries the notice.
|
|
147
148
|
if (change.action === 'reordered') continue;
|
|
148
149
|
// The board pump tells the driver through its own event; steering the assignee would start the work it must leave to the driver.
|
|
149
|
-
|
|
150
|
+
// A notice that a worker calls without its identity is for the owner: it would only wake the worker it is about.
|
|
151
|
+
if (change.actor.id === boardPumpActor.id || change.actor.id === workerIdentityActor.id) continue;
|
|
150
152
|
if (change.actor.role !== 'driver' && change.origin === undefined) {
|
|
151
153
|
// A comment's whole text is the comment, which the driver reads by its id.
|
|
152
154
|
const comment = change.action === 'commented' ? change.changes.comment?.after as ITicketComment | undefined : undefined;
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { controllerFailureJournal } from './classes.failurejournal.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* AGL-1907: the one background run of the order spread (`ts_migration/v43_ticketorderstride.ts`) a controller starts
|
|
5
|
+
* once it is ready; nothing waits for it, since the store is correct on dense orders. Its transactions run one after
|
|
6
|
+
* another, each once the last one settled, with no pause between them. A transaction that loses a write conflict to a
|
|
7
|
+
* mutation runs again, by the driver within its deadline and by the run after it, until it reached three deadlines in
|
|
8
|
+
* a row without a commit; that, and any other failure, ends the run, logged and kept in the failure journal once, and
|
|
9
|
+
* the next controller start continues from where it stopped. `stop`
|
|
10
|
+
* ends it before the controller's database closes.
|
|
11
|
+
*/
|
|
12
|
+
export class TicketOrderSpread {
|
|
13
|
+
private readonly abort = new AbortController();
|
|
14
|
+
private running?: Promise<void>;
|
|
15
|
+
|
|
16
|
+
constructor(private readonly runArg: (signalArg: AbortSignal) => Promise<string[]>) {}
|
|
17
|
+
|
|
18
|
+
/** Starts the run; once, a later call does nothing. A spread stopped before it started runs nothing. */
|
|
19
|
+
public start(): void {
|
|
20
|
+
if (this.abort.signal.aborted) return;
|
|
21
|
+
this.running ??= this.run();
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** Settles once the run ended, whether it finished, failed or was stopped; at once when it never started. Never rejects. */
|
|
25
|
+
public get done(): Promise<void> {
|
|
26
|
+
return this.running ?? Promise.resolve();
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Ends the run and waits for it: the transaction in flight rolls back, or commits when its commit was already sent,
|
|
31
|
+
* and no other starts.
|
|
32
|
+
*/
|
|
33
|
+
public async stop(): Promise<void> {
|
|
34
|
+
this.abort.abort(new Error('The controller is stopping.'));
|
|
35
|
+
await this.done;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
private async run(): Promise<void> {
|
|
39
|
+
const signal = this.abort.signal;
|
|
40
|
+
try {
|
|
41
|
+
const spread = await this.runArg(signal);
|
|
42
|
+
// Once, nearly every ticket of the board: the count, not the keys.
|
|
43
|
+
if (spread.length) console.log(`Spread the orders of ${spread.length} tickets a stride apart, in board order (AGL-1907).`);
|
|
44
|
+
} catch (errorArg) {
|
|
45
|
+
if (signal.aborted && errorArg === signal.reason) return;
|
|
46
|
+
const reference = controllerFailureJournal.record({ operation: 'tickets.orderSpread', cause: errorArg });
|
|
47
|
+
console.error(`Spreading the ticket orders failed (ref ${reference}); the next controller start continues it (AGL-1907):`, errorArg);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|