agl 40.5.0 → 40.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.md +16 -0
- package/dist_serve/bundle.js +598 -598
- package/dist_serve/bundle.js.map +2 -2
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/aglapi/aglapisurface.document.js +2 -2
- package/dist_ts/classes.aglhome.d.ts +5 -0
- package/dist_ts/classes.aglhome.js +2 -1
- package/dist_ts/classes.authstore.js +5 -1
- package/dist_ts/classes.controller.js +11 -2
- package/dist_ts/classes.inferencegateway.d.ts +35 -1
- package/dist_ts/classes.inferencegateway.js +86 -26
- package/dist_ts/classes.inferenceservice.d.ts +4 -2
- package/dist_ts/classes.inferenceservice.js +5 -4
- package/dist_ts/classes.proxycontrol.d.ts +58 -4
- package/dist_ts/classes.proxycontrol.js +23 -44
- package/dist_ts/classes.proxycontrolclient.d.ts +15 -5
- package/dist_ts/classes.proxycontrolclient.js +67 -17
- package/dist_ts/classes.proxydaemon.d.ts +22 -3
- package/dist_ts/classes.proxydaemon.js +73 -12
- package/dist_ts/classes.proxyleaseholder.d.ts +43 -14
- package/dist_ts/classes.proxyleaseholder.js +66 -22
- package/dist_ts/classes.proxystore.d.ts +9 -0
- package/dist_ts/classes.proxystore.js +15 -7
- package/dist_ts/classes.workercallerbindings.d.ts +66 -0
- package/dist_ts/classes.workercallerbindings.js +211 -0
- package/dist_ts/classes.workercallers.d.ts +63 -2
- package/dist_ts/classes.workercallers.js +430 -65
- package/dist_ts/fleet/classes.workerlaunches.d.ts +12 -5
- package/dist_ts/fleet/classes.workerlaunches.js +25 -19
- package/dist_ts/fleet/classes.workers.d.ts +17 -0
- package/dist_ts/fleet/classes.workers.js +74 -1
- package/dist_ts/functions.inferencecli.d.ts +3 -2
- package/dist_ts/functions.inferencecli.js +67 -5
- package/dist_ts/functions.inferencecliarguments.d.ts +6 -2
- package/dist_ts/functions.inferencecliarguments.js +29 -2
- package/dist_ts/functions.inferencelaunch.d.ts +24 -0
- package/dist_ts/functions.inferencelaunch.js +42 -2
- package/dist_ts/functions.proxycontract.d.ts +11 -0
- package/dist_ts/functions.proxycontract.js +12 -1
- package/dist_ts/functions.proxysocket.d.ts +6 -0
- package/dist_ts/functions.proxysocket.js +46 -0
- package/dist_ts/functions.workerprocessproof.d.ts +6 -0
- package/dist_ts/functions.workerprocessproof.js +28 -4
- package/dist_ts_authswitch/authority-contract.d.ts +30 -5
- package/dist_ts_authswitch/authority-contract.js +7 -1
- package/dist_ts_authswitch/classes.authoritycli.js +6 -2
- package/dist_ts_authswitch/classes.authorityclient.js +15 -5
- package/dist_ts_authswitch/classes.claudenative.d.ts +7 -1
- package/dist_ts_authswitch/classes.claudenative.js +21 -6
- package/dist_ts_authswitch/classes.claudesignin.d.ts +14 -2
- package/dist_ts_authswitch/classes.claudesignin.js +68 -22
- package/dist_ts_authswitch/classes.codexmanaged.d.ts +8 -0
- package/dist_ts_authswitch/classes.codexmanaged.js +33 -18
- package/dist_ts_authswitch/classes.harnessprocesses.d.ts +8 -0
- package/dist_ts_authswitch/classes.harnessprocesses.js +41 -9
- package/dist_ts_interfaces/interfaces.d.ts +1 -1
- package/dist_ts_interfaces/interfaces.js +1 -1
- package/dist_ts_interfaces/workercallers.d.ts +43 -4
- package/package.json +1 -1
- package/readme.md +103 -14
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/aglapi/aglapisurface.document.ts +2 -2
- package/ts/classes.aglhome.ts +6 -0
- package/ts/classes.authstore.ts +4 -0
- package/ts/classes.controller.ts +10 -1
- package/ts/classes.inferencegateway.ts +104 -23
- package/ts/classes.inferenceservice.ts +6 -3
- package/ts/classes.proxycontrol.ts +76 -39
- package/ts/classes.proxycontrolclient.ts +72 -20
- package/ts/classes.proxydaemon.ts +85 -15
- package/ts/classes.proxyleaseholder.ts +84 -28
- package/ts/classes.proxystore.ts +16 -5
- package/ts/classes.workercallerbindings.ts +147 -0
- package/ts/classes.workercallers.ts +410 -59
- package/ts/fleet/classes.workerlaunches.ts +23 -15
- package/ts/fleet/classes.workers.ts +51 -1
- package/ts/functions.inferencecli.ts +61 -7
- package/ts/functions.inferencecliarguments.ts +32 -1
- package/ts/functions.inferencelaunch.ts +49 -2
- package/ts/functions.proxycontract.ts +12 -0
- package/ts/functions.proxysocket.ts +39 -0
- package/ts/functions.workerprocessproof.ts +27 -4
- package/ts_authswitch/authority-contract.ts +34 -5
- package/ts_authswitch/classes.authoritycli.ts +5 -1
- package/ts_authswitch/classes.authorityclient.ts +25 -9
- package/ts_authswitch/classes.claudenative.ts +23 -7
- package/ts_authswitch/classes.claudesignin.ts +70 -23
- package/ts_authswitch/classes.codexmanaged.ts +30 -16
- package/ts_authswitch/classes.harnessprocesses.ts +39 -8
- package/ts_authswitch/native-contracts.md +1 -1
- package/ts_interfaces/interfaces.ts +1 -1
- package/ts_interfaces/workercallers.ts +32 -4
- package/ts_web/00_commitinfo_data.ts +1 -1
- package/ts_web/accounts.plugins.ts +5 -1
- package/ts_web/classes.accountsreadmodel.ts +4 -1
|
@@ -3,12 +3,13 @@ import { commitinfo } from './00_commitinfo_data.js';
|
|
|
3
3
|
import type { IAuthoritySocketPaths } from './classes.authorityclient.js';
|
|
4
4
|
import { InferenceService } from './classes.inferenceservice.js';
|
|
5
5
|
import { inferenceGatewayCredentialDigest, type IInferenceGatewayGrant, type IInferenceGatewayLease,
|
|
6
|
-
type IInferenceGatewayLimits } from './classes.inferencegateway.js';
|
|
6
|
+
type IInferenceGatewayLimits, type TInferenceGatewayRoute } from './classes.inferencegateway.js';
|
|
7
7
|
import { InferenceRoutingError, type IInferenceRoutingPolicy } from './classes.inferencerouter.js';
|
|
8
|
-
import { proxyClientNamePattern, type TInferenceProvider,
|
|
8
|
+
import { aglProxyConfinedClient, aglProxyConfinedOrigin, proxyClientNamePattern, type TInferenceProvider,
|
|
9
9
|
type TProxyCredentialDelivery } from './functions.proxycontract.js';
|
|
10
|
-
import { AglProxyControlRefusal, AglProxyControlServer, type
|
|
11
|
-
type
|
|
10
|
+
import { AglProxyControlRefusal, AglProxyControlServer, type IAglProxyConfinedGrant, type IAglProxyConnection,
|
|
11
|
+
type IAglProxyStatus, type IReq_AglProxyCatalog, type IReq_AglProxyReclaim,
|
|
12
|
+
type TAglProxyLeaseCatalog } from './classes.proxycontrol.js';
|
|
12
13
|
import { ProxyStore, type IProxySecretCodec } from './classes.proxystore.js';
|
|
13
14
|
import type { InferenceUsageStore } from './classes.inferenceusage.js';
|
|
14
15
|
import { prepareProxyCredentialDirectory, proxyCredentialFile, removeProxyCredentialFile,
|
|
@@ -27,6 +28,11 @@ export interface IAglProxyDaemonOptions {
|
|
|
27
28
|
limits?: IInferenceGatewayLimits;
|
|
28
29
|
/** How long a lease kept across a restart waits for its holder; `proxyLeaseReclaimWindowMs` unless a test shortens it. */
|
|
29
30
|
leaseReclaimWindowMs?: number;
|
|
31
|
+
/**
|
|
32
|
+
* The confined listener's socket, alone in its directory (`IAGLHomePaths.proxyConfinedSocket`); unset, the proxy has no
|
|
33
|
+
* confined listener and refuses confined runs.
|
|
34
|
+
*/
|
|
35
|
+
confinedSocketPath?: string;
|
|
30
36
|
}
|
|
31
37
|
|
|
32
38
|
/**
|
|
@@ -45,9 +51,19 @@ const catalogRefusal = (errorArg: unknown): AglProxyControlRefusal => new AglPro
|
|
|
45
51
|
errorArg instanceof InferenceRoutingError ? errorArg.code : undefined,
|
|
46
52
|
);
|
|
47
53
|
|
|
54
|
+
/** The policy of every confined run's credential: the Claude pool, unrestricted, as Claude Code's own lease. */
|
|
55
|
+
const confinedPolicy: IInferenceRoutingPolicy = Object.freeze({ provider: 'anthropic' });
|
|
56
|
+
|
|
57
|
+
/** A confined run's id, as the operator names it: the first 12 hex characters of its credential's digest. */
|
|
58
|
+
const confinedRunId = (digestArg: string): string => digestArg.slice(0, 12);
|
|
59
|
+
|
|
48
60
|
/** A lease the proxy serves: held by the control connection it was issued or reclaimed on, or awaiting its holder. */
|
|
49
61
|
interface IHeldLease {
|
|
50
62
|
grant: IInferenceGatewayGrant;
|
|
63
|
+
route: TInferenceGatewayRoute;
|
|
64
|
+
createdAt: Date;
|
|
65
|
+
/** The connection the lease lives as long as, while a holder has it. */
|
|
66
|
+
holder?: IAglProxyConnection;
|
|
51
67
|
/** Set while the lease, kept across a restart, waits for its holder; it ends when this timer fires. */
|
|
52
68
|
reclaimTimer?: ReturnType<typeof setTimeout>;
|
|
53
69
|
}
|
|
@@ -61,9 +77,10 @@ interface IClientLease {
|
|
|
61
77
|
}
|
|
62
78
|
|
|
63
79
|
/**
|
|
64
|
-
* The permanent proxy: one loopback listener, one credential per client,
|
|
65
|
-
* a client its credential
|
|
66
|
-
*
|
|
80
|
+
* The permanent proxy: one loopback listener, one credential per client, a confined listener for Claude Code in
|
|
81
|
+
* containers, and a private control socket that hands a client its credential, a child a lease and a confined run its
|
|
82
|
+
* credential. The control socket is claimed first; it is the proof that no other proxy runs for this AGL home, so the
|
|
83
|
+
* proxy's database is opened by this process alone.
|
|
67
84
|
*/
|
|
68
85
|
export class AglProxyDaemon {
|
|
69
86
|
private readonly control: AglProxyControlServer;
|
|
@@ -107,7 +124,8 @@ export class AglProxyDaemon {
|
|
|
107
124
|
port: this.options.port, signatureKey: identity.signatureKey, affinity: this.store.affinity(),
|
|
108
125
|
placements: this.store.placements(), usageLedger: this.usage, spendStore: this.store.spend(),
|
|
109
126
|
...(this.options.providerFetch ? { providerFetch: this.options.providerFetch } : {}),
|
|
110
|
-
...(this.options.limits ? { limits: this.options.limits } : {})
|
|
127
|
+
...(this.options.limits ? { limits: this.options.limits } : {}),
|
|
128
|
+
...(this.options.confinedSocketPath === undefined ? {} : { confinedSocketPath: this.options.confinedSocketPath }) });
|
|
111
129
|
const baseUrl = await this.service.start(signal);
|
|
112
130
|
const deliveries = new Set(await this.store.fileDeliveries());
|
|
113
131
|
for (const credential of await this.store.credentials()) {
|
|
@@ -128,6 +146,8 @@ export class AglProxyDaemon {
|
|
|
128
146
|
catalog: (requestArg, connectionArg) => this.catalog(requestArg, connectionArg),
|
|
129
147
|
lease: (requestArg, connectionArg) => this.lease(requestArg.policy, requestArg.client, connectionArg),
|
|
130
148
|
reclaim: async (requestArg, connectionArg) => this.reclaim(requestArg.token, connectionArg),
|
|
149
|
+
confined: (connectionArg) => this.confined(connectionArg),
|
|
150
|
+
confinedEnd: async (requestArg) => this.confinedEnd(requestArg.id),
|
|
131
151
|
usage: async (requestArg, connectionArg) => 'since' in requestArg ? this.usage!.changes(requestArg)
|
|
132
152
|
: await this.usage!.snapshot(requestArg, connectionArg.signal),
|
|
133
153
|
});
|
|
@@ -145,7 +165,11 @@ export class AglProxyDaemon {
|
|
|
145
165
|
}),
|
|
146
166
|
leases: status.leases, inFlight: status.inFlight, limits: status.limits, authority: status.authority,
|
|
147
167
|
accounts: status.accounts, usageObservations: status.usageObservations, nextPlacement: status.nextPlacement,
|
|
148
|
-
recentPlacements: status.recentPlacements,
|
|
168
|
+
recentPlacements: status.recentPlacements, confinedSocketPath: this.options.confinedSocketPath ?? null,
|
|
169
|
+
confinedRuns: [...this.leases].filter(([, heldArg]) => heldArg.route === 'confined')
|
|
170
|
+
.sort(([, left], [, right]) => left.createdAt.getTime() - right.createdAt.getTime())
|
|
171
|
+
.map(([digestArg, heldArg]) => ({ id: confinedRunId(digestArg), createdAt: heldArg.createdAt.toISOString(),
|
|
172
|
+
held: heldArg.reclaimTimer === undefined })),
|
|
149
173
|
};
|
|
150
174
|
}
|
|
151
175
|
|
|
@@ -265,11 +289,11 @@ export class AglProxyDaemon {
|
|
|
265
289
|
// The lease continues the sessions of the client it is taken for (see `IReq_AglProxyLease`).
|
|
266
290
|
try { lease = await service.open(policyArg, { sessionScope: clientArg }); }
|
|
267
291
|
catch (errorArg) { throw new AglProxyControlRefusal(errorArg instanceof Error ? errorArg.message : 'The lease could not be opened.'); }
|
|
268
|
-
const held: IHeldLease = { grant: lease };
|
|
292
|
+
const held: IHeldLease = { grant: lease, route: 'loopback', createdAt: new Date(), holder: connectionArg };
|
|
269
293
|
this.leases.set(lease.digest, held);
|
|
270
294
|
try {
|
|
271
295
|
await this.trackLeaseWrite(this.store!.recordLease({ digest: lease.digest, client: clientArg, policy: policyArg,
|
|
272
|
-
createdAt:
|
|
296
|
+
createdAt: held.createdAt, route: 'loopback' }));
|
|
273
297
|
} catch (errorArg) {
|
|
274
298
|
this.endLease(lease.digest, held);
|
|
275
299
|
throw new AglProxyControlRefusal(errorArg instanceof Error ? errorArg.message : 'The lease could not be recorded.');
|
|
@@ -294,7 +318,50 @@ export class AglProxyDaemon {
|
|
|
294
318
|
* an unknown credential from an expired or a held one. The credential is looked up by its SHA-256 digest: a match
|
|
295
319
|
* proves the caller holds the credential the lease was issued with.
|
|
296
320
|
*/
|
|
297
|
-
|
|
321
|
+
/**
|
|
322
|
+
* A confined run's credential (see `IReq_AglProxyConfined`): a lease of the Claude pool that continues the confined
|
|
323
|
+
* client's sessions and is accepted on the confined listener alone. It is kept, as its digest, before it is handed out,
|
|
324
|
+
* and ends when its holder's connection closes or the operator ends it (`confinedEnd`). No model list is read for it:
|
|
325
|
+
* Claude Code reads the pool's models through the confined listener, and the pool is judged at each request.
|
|
326
|
+
*/
|
|
327
|
+
private async confined(connectionArg: IAglProxyConnection): Promise<IAglProxyConfinedGrant> {
|
|
328
|
+
const socketPath = this.options.confinedSocketPath;
|
|
329
|
+
if (socketPath === undefined) throw new AglProxyControlRefusal('This AGL proxy has no confined listener.');
|
|
330
|
+
let lease: IInferenceGatewayLease;
|
|
331
|
+
try { lease = await this.service!.open(confinedPolicy, { sessionScope: aglProxyConfinedClient, route: 'confined' }); }
|
|
332
|
+
catch (errorArg) {
|
|
333
|
+
throw new AglProxyControlRefusal(errorArg instanceof Error ? errorArg.message : 'The confined credential could not be issued.');
|
|
334
|
+
}
|
|
335
|
+
const held: IHeldLease = { grant: lease, route: 'confined', createdAt: new Date(), holder: connectionArg };
|
|
336
|
+
this.leases.set(lease.digest, held);
|
|
337
|
+
try {
|
|
338
|
+
await this.trackLeaseWrite(this.store!.recordLease({ digest: lease.digest, client: aglProxyConfinedClient,
|
|
339
|
+
policy: confinedPolicy, createdAt: held.createdAt, route: 'confined' }));
|
|
340
|
+
} catch (errorArg) {
|
|
341
|
+
this.endLease(lease.digest, held);
|
|
342
|
+
throw new AglProxyControlRefusal(errorArg instanceof Error ? errorArg.message : 'The confined credential could not be recorded.');
|
|
343
|
+
}
|
|
344
|
+
connectionArg.onClose(() => this.endLease(lease.digest, held));
|
|
345
|
+
return { id: confinedRunId(lease.digest), token: lease.token, socketPath, baseUrl: aglProxyConfinedOrigin };
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
/**
|
|
349
|
+
* Ends a confined run's credential at once, held or waiting for its holder: the gateway refuses it from now on, and
|
|
350
|
+
* its requests in flight end. The holder's connection is closed with the reason, so it stops its container instead of
|
|
351
|
+
* taking the credential back.
|
|
352
|
+
*/
|
|
353
|
+
private confinedEnd(idArg: string): { id: string } {
|
|
354
|
+
const matches = [...this.leases].filter(([digestArg, heldArg]) => heldArg.route === 'confined'
|
|
355
|
+
&& confinedRunId(digestArg) === idArg);
|
|
356
|
+
if (matches.length !== 1) throw new AglProxyControlRefusal(`The AGL proxy serves no confined run ${idArg}.`);
|
|
357
|
+
const [[digest, held]] = matches;
|
|
358
|
+
const holder = held.holder;
|
|
359
|
+
this.endLease(digest, held);
|
|
360
|
+
holder?.end(`The operator ended confined run ${idArg} (\`agl proxy confined end\`).`);
|
|
361
|
+
return { id: idArg };
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
private async reclaim(tokenArg: string, connectionArg: IAglProxyConnection): Promise<IReq_AglProxyReclaim['response']> {
|
|
298
365
|
const digest = inferenceGatewayCredentialDigest(tokenArg);
|
|
299
366
|
const held = this.leases.get(digest);
|
|
300
367
|
if (!held?.reclaimTimer || connectionArg.signal.aborted) {
|
|
@@ -309,8 +376,10 @@ export class AglProxyDaemon {
|
|
|
309
376
|
this.endLease(digest, held);
|
|
310
377
|
throw new AglProxyControlRefusal(errorArg instanceof Error ? errorArg.message : 'The lease could not be taken back.');
|
|
311
378
|
}
|
|
379
|
+
held.holder = connectionArg;
|
|
312
380
|
connectionArg.onClose(() => this.endLease(digest, held));
|
|
313
|
-
return { baseUrl:
|
|
381
|
+
return held.route === 'confined' ? { baseUrl: aglProxyConfinedOrigin, socketPath: this.options.confinedSocketPath! }
|
|
382
|
+
: { baseUrl: held.grant.baseUrl };
|
|
314
383
|
}
|
|
315
384
|
|
|
316
385
|
/**
|
|
@@ -338,14 +407,14 @@ export class AglProxyDaemon {
|
|
|
338
407
|
let grant: IInferenceGatewayGrant;
|
|
339
408
|
try {
|
|
340
409
|
if (!kept.orphanedAt) await this.store!.recordLeaseOrphaned(kept.digest, orphanedAt);
|
|
341
|
-
grant = await service.restore(kept.policy, { digest: kept.digest, sessionScope: kept.client });
|
|
410
|
+
grant = await service.restore(kept.policy, { digest: kept.digest, sessionScope: kept.client, route: kept.route });
|
|
342
411
|
} catch (errorArg) {
|
|
343
412
|
// A lease the gateway no longer admits (its policy or limits changed) cannot be served; it ends here.
|
|
344
413
|
this.reportLease(kept.digest, 'is not served again', errorArg);
|
|
345
414
|
await this.forgetLease(kept.digest);
|
|
346
415
|
continue;
|
|
347
416
|
}
|
|
348
|
-
const held: IHeldLease = { grant };
|
|
417
|
+
const held: IHeldLease = { grant, route: kept.route, createdAt: kept.createdAt };
|
|
349
418
|
held.reclaimTimer = setTimeout(() => this.endLease(kept.digest, held), Math.max(0, deadline - Date.now()));
|
|
350
419
|
this.leases.set(kept.digest, held);
|
|
351
420
|
}
|
|
@@ -358,6 +427,7 @@ export class AglProxyDaemon {
|
|
|
358
427
|
private endLease(digestArg: string, heldArg: IHeldLease): void {
|
|
359
428
|
if (this.abort.signal.aborted || this.leases.get(digestArg) !== heldArg) return;
|
|
360
429
|
clearTimeout(heldArg.reclaimTimer);
|
|
430
|
+
heldArg.holder = undefined;
|
|
361
431
|
this.leases.delete(digestArg);
|
|
362
432
|
heldArg.grant.close();
|
|
363
433
|
void this.forgetLease(digestArg);
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as plugins from './plugins.js';
|
|
2
2
|
import type { IInferenceRoutingPolicy } from './classes.inferencerouter.js';
|
|
3
|
-
import type { TAglProxyLeaseCatalog } from './classes.proxycontrol.js';
|
|
3
|
+
import type { IReq_AglProxyReclaim, TAglProxyLeaseCatalog } from './classes.proxycontrol.js';
|
|
4
4
|
import { AglProxyControlClient, AglProxyRefusedError, type IAglProxyLeaseConnection } from './classes.proxycontrolclient.js';
|
|
5
5
|
|
|
6
6
|
/** The delays between attempts to take a lease back from a restarting proxy; the last one repeats. */
|
|
@@ -22,33 +22,30 @@ export class AglProxyLeaseLostError extends Error {
|
|
|
22
22
|
}
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
-
|
|
25
|
+
/** How a holder reaches its proxy and how long it keeps trying to take its lease back. */
|
|
26
|
+
export interface IAglProxyHolderOptions {
|
|
26
27
|
socketPath: string;
|
|
28
|
+
reclaimDelaysMs?: readonly number[];
|
|
29
|
+
reclaimDeadlineMs?: number;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface IAglProxyLeaseHolderOptions extends IAglProxyHolderOptions {
|
|
27
33
|
policy: IInferenceRoutingPolicy;
|
|
28
34
|
/** The proxy client whose sessions the lease continues (see `IReq_AglProxyLease`). */
|
|
29
35
|
client: string;
|
|
30
|
-
reclaimDelaysMs?: readonly number[];
|
|
31
|
-
reclaimDeadlineMs?: number;
|
|
32
36
|
}
|
|
33
37
|
|
|
34
38
|
/**
|
|
35
|
-
* A proxy
|
|
36
|
-
* connection; when that connection closes without the holder closing it -- the proxy
|
|
37
|
-
* holder reconnects with a bounded backoff and takes the same
|
|
38
|
-
* with
|
|
39
|
-
* time, is lost for good: `lost` settles with the reason, for the holder to
|
|
39
|
+
* A proxy credential its holder keeps for as long as it needs it: a lease, or a confined run's credential. It lives as
|
|
40
|
+
* long as the holder's control connection; when that connection closes without the holder closing it -- the proxy
|
|
41
|
+
* stopped or restarted -- the holder reconnects with a bounded backoff and takes the same credential back, so the child
|
|
42
|
+
* configured with it keeps working. A credential the proxy refuses to hand back, one the proxy ended itself (it says
|
|
43
|
+
* why), or a proxy that does not come back in time, is lost for good: `lost` settles with the reason, for the holder to
|
|
44
|
+
* show rather than leave its child failing.
|
|
40
45
|
*/
|
|
41
|
-
|
|
42
|
-
/** Takes a new lease of the proxy on `socketPathArg`. */
|
|
43
|
-
public static async take(optionsArg: IAglProxyLeaseHolderOptions, signalArg: AbortSignal): Promise<AglProxyLeaseHolder> {
|
|
44
|
-
const control = new AglProxyControlClient(optionsArg.socketPath);
|
|
45
|
-
const lease = await control.lease(optionsArg.policy, optionsArg.client, signalArg);
|
|
46
|
-
return new AglProxyLeaseHolder(control, lease, optionsArg);
|
|
47
|
-
}
|
|
48
|
-
|
|
46
|
+
class AglProxyHeldCredential {
|
|
49
47
|
public readonly baseUrl: string;
|
|
50
48
|
public readonly token: string;
|
|
51
|
-
public readonly catalog: TAglProxyLeaseCatalog;
|
|
52
49
|
/** Settles with the reason once the lease is lost for good; never for a lease its holder closes. */
|
|
53
50
|
public readonly lost: Promise<AglProxyLeaseLostError>;
|
|
54
51
|
private stateValue: TAglProxyLeaseState = 'held';
|
|
@@ -56,18 +53,23 @@ export class AglProxyLeaseHolder {
|
|
|
56
53
|
private readonly abort = new AbortController();
|
|
57
54
|
private readonly resolveLost: (errorArg: AglProxyLeaseLostError) => void;
|
|
58
55
|
|
|
59
|
-
|
|
56
|
+
protected constructor(
|
|
60
57
|
private readonly control: AglProxyControlClient,
|
|
61
|
-
|
|
62
|
-
private readonly options:
|
|
58
|
+
issuedArg: { baseUrl: string; token: string } & IAglProxyLeaseConnection,
|
|
59
|
+
private readonly options: IAglProxyHolderOptions,
|
|
63
60
|
) {
|
|
64
|
-
this.baseUrl =
|
|
65
|
-
this.token =
|
|
66
|
-
this.catalog = leaseArg.catalog;
|
|
61
|
+
this.baseUrl = issuedArg.baseUrl;
|
|
62
|
+
this.token = issuedArg.token;
|
|
67
63
|
const lost = Promise.withResolvers<AglProxyLeaseLostError>();
|
|
68
64
|
this.lost = lost.promise;
|
|
69
65
|
this.resolveLost = lost.resolve;
|
|
70
|
-
this.watch(
|
|
66
|
+
this.watch(issuedArg);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Why a credential taken back cannot continue where its child was configured, or `undefined` when it can. */
|
|
70
|
+
protected relocation(reclaimedArg: IReq_AglProxyReclaim['response']): string | undefined {
|
|
71
|
+
return reclaimedArg.baseUrl === this.baseUrl ? undefined
|
|
72
|
+
: `The AGL proxy restarted at ${reclaimedArg.baseUrl} instead of ${this.baseUrl}, where this lease is configured.`;
|
|
71
73
|
}
|
|
72
74
|
|
|
73
75
|
public get state(): TAglProxyLeaseState {
|
|
@@ -84,8 +86,10 @@ export class AglProxyLeaseHolder {
|
|
|
84
86
|
|
|
85
87
|
private watch(connectionArg: IAglProxyLeaseConnection): void {
|
|
86
88
|
this.connection = connectionArg;
|
|
87
|
-
void connectionArg.closed.then(() => {
|
|
89
|
+
void connectionArg.closed.then((endedArg) => {
|
|
88
90
|
if (this.connection !== connectionArg || this.stateValue !== 'held') return;
|
|
91
|
+
// The proxy ended the credential itself and said why: there is nothing to take back.
|
|
92
|
+
if (endedArg !== undefined) { this.lose(endedArg); return; }
|
|
89
93
|
this.stateValue = 'reclaiming';
|
|
90
94
|
void this.reclaim();
|
|
91
95
|
});
|
|
@@ -109,9 +113,10 @@ export class AglProxyLeaseHolder {
|
|
|
109
113
|
const reclaimed = await this.control.reclaim(this.token,
|
|
110
114
|
AbortSignal.any([this.abort.signal, AbortSignal.timeout(Math.max(1, deadline - Date.now()))]));
|
|
111
115
|
if (this.stateValue !== 'reclaiming') { reclaimed.close(); return; }
|
|
112
|
-
|
|
116
|
+
const relocation = this.relocation(reclaimed);
|
|
117
|
+
if (relocation !== undefined) {
|
|
113
118
|
reclaimed.close();
|
|
114
|
-
this.lose(
|
|
119
|
+
this.lose(relocation);
|
|
115
120
|
return;
|
|
116
121
|
}
|
|
117
122
|
this.stateValue = 'held';
|
|
@@ -134,3 +139,54 @@ export class AglProxyLeaseHolder {
|
|
|
134
139
|
this.resolveLost(new AglProxyLeaseLostError(messageArg));
|
|
135
140
|
}
|
|
136
141
|
}
|
|
142
|
+
|
|
143
|
+
/** A proxy lease for one child, with the pool's models as the proxy read them when it issued the lease. */
|
|
144
|
+
export class AglProxyLeaseHolder extends AglProxyHeldCredential {
|
|
145
|
+
/** Takes a new lease of the proxy on `socketPathArg`. */
|
|
146
|
+
public static async take(optionsArg: IAglProxyLeaseHolderOptions, signalArg: AbortSignal): Promise<AglProxyLeaseHolder> {
|
|
147
|
+
const control = new AglProxyControlClient(optionsArg.socketPath);
|
|
148
|
+
const lease = await control.lease(optionsArg.policy, optionsArg.client, signalArg);
|
|
149
|
+
return new AglProxyLeaseHolder(control, lease, optionsArg);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
public readonly catalog: TAglProxyLeaseCatalog;
|
|
153
|
+
|
|
154
|
+
private constructor(controlArg: AglProxyControlClient,
|
|
155
|
+
leaseArg: { baseUrl: string; token: string; catalog: TAglProxyLeaseCatalog } & IAglProxyLeaseConnection,
|
|
156
|
+
optionsArg: IAglProxyLeaseHolderOptions) {
|
|
157
|
+
super(controlArg, leaseArg, optionsArg);
|
|
158
|
+
this.catalog = leaseArg.catalog;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* A confined run's credential (see `IReq_AglProxyConfined`), held for as long as its container runs. It is configured
|
|
164
|
+
* with the confined listener's socket, so one taken back on another socket cannot continue.
|
|
165
|
+
*/
|
|
166
|
+
export class AglProxyConfinedHolder extends AglProxyHeldCredential {
|
|
167
|
+
/** Takes a new confined run's credential of the proxy on `optionsArg.socketPath`, its control socket. */
|
|
168
|
+
public static async take(optionsArg: IAglProxyHolderOptions, signalArg: AbortSignal): Promise<AglProxyConfinedHolder> {
|
|
169
|
+
const control = new AglProxyControlClient(optionsArg.socketPath);
|
|
170
|
+
const grant = await control.confined(signalArg);
|
|
171
|
+
return new AglProxyConfinedHolder(control, grant, optionsArg);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** The run's id, as `agl proxy confined end` and `agl proxy status` name it. */
|
|
175
|
+
public readonly id: string;
|
|
176
|
+
/** The confined listener's socket, on the host. */
|
|
177
|
+
public readonly confinedSocketPath: string;
|
|
178
|
+
|
|
179
|
+
private constructor(controlArg: AglProxyControlClient,
|
|
180
|
+
grantArg: { id: string; baseUrl: string; token: string; socketPath: string } & IAglProxyLeaseConnection,
|
|
181
|
+
optionsArg: IAglProxyHolderOptions) {
|
|
182
|
+
super(controlArg, grantArg, optionsArg);
|
|
183
|
+
this.id = grantArg.id;
|
|
184
|
+
this.confinedSocketPath = grantArg.socketPath;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
protected override relocation(reclaimedArg: IReq_AglProxyReclaim['response']): string | undefined {
|
|
188
|
+
return reclaimedArg.socketPath === this.confinedSocketPath ? undefined
|
|
189
|
+
: `The AGL proxy restarted with its confined listener at ${reclaimedArg.socketPath ?? 'none'} instead of `
|
|
190
|
+
+ `${this.confinedSocketPath}, which this run's container has mounted.`;
|
|
191
|
+
}
|
|
192
|
+
}
|
package/ts/classes.proxystore.ts
CHANGED
|
@@ -5,9 +5,10 @@ import { InferenceSessionModel } from './classes.inferencesession.js';
|
|
|
5
5
|
import { InferencePlacementModel, InferencePlacementStore } from './classes.inferenceplacement.js';
|
|
6
6
|
import { InferenceUsageModel, InferenceUsageStore } from './classes.inferenceusage.js';
|
|
7
7
|
import { InferenceSpendModel, InferenceSpendStore } from './classes.inferencespend.js';
|
|
8
|
-
import { assertInferenceRoutingPolicy } from './classes.inferencegateway.js';
|
|
8
|
+
import { assertInferenceRoutingPolicy, type TInferenceGatewayRoute } from './classes.inferencegateway.js';
|
|
9
9
|
import type { IInferenceRoutingPolicy } from './classes.inferencerouter.js';
|
|
10
10
|
import {
|
|
11
|
+
aglProxyConfinedClient,
|
|
11
12
|
defaultProxyClientProvider,
|
|
12
13
|
defaultProxyClients,
|
|
13
14
|
inferenceProviders,
|
|
@@ -149,6 +150,11 @@ export interface IProxyLeaseDocument {
|
|
|
149
150
|
* until its holder takes it back, which removes it. Absent while a holder has the lease.
|
|
150
151
|
*/
|
|
151
152
|
orphanedAt?: Date;
|
|
153
|
+
/**
|
|
154
|
+
* `confined` for a confined run's credential, accepted on the confined listener alone, with the Claude pool's policy
|
|
155
|
+
* and the confined client's sessions; absent for a lease of the loopback listener.
|
|
156
|
+
*/
|
|
157
|
+
route?: 'confined';
|
|
152
158
|
}
|
|
153
159
|
|
|
154
160
|
const isDate = (valueArg: unknown): valueArg is Date => valueArg instanceof Date && Number.isFinite(valueArg.getTime());
|
|
@@ -171,10 +177,12 @@ const assertLeasePolicy = (valueArg: unknown): IProxyLeasePolicyDocument => {
|
|
|
171
177
|
const assertLease: (valueArg: unknown) => asserts valueArg is IProxyLeaseDocument = (valueArg) => {
|
|
172
178
|
const value = valueArg as Partial<IProxyLeaseDocument> | null;
|
|
173
179
|
if (!value || typeof value !== 'object'
|
|
174
|
-
|| Object.keys(value).length !== (Object.hasOwn(value, 'orphanedAt')
|
|
180
|
+
|| Object.keys(value).length !== 4 + Number(Object.hasOwn(value, 'orphanedAt')) + Number(Object.hasOwn(value, 'route'))
|
|
175
181
|
|| typeof value.id !== 'string' || !/^[a-f0-9]{64}$/.test(value.id)
|
|
176
182
|
|| typeof value.client !== 'string' || !proxyClientNamePattern.test(value.client)
|
|
177
|
-
|| !isDate(value.createdAt) || (Object.hasOwn(value, 'orphanedAt') && !isDate(value.orphanedAt))
|
|
183
|
+
|| !isDate(value.createdAt) || (Object.hasOwn(value, 'orphanedAt') && !isDate(value.orphanedAt))
|
|
184
|
+
|| (Object.hasOwn(value, 'route') && (value.route !== 'confined' || value.client !== aglProxyConfinedClient
|
|
185
|
+
|| JSON.stringify(value.policy) !== JSON.stringify({ provider: 'anthropic' })))) {
|
|
178
186
|
throw new Error('Invalid proxy lease.');
|
|
179
187
|
}
|
|
180
188
|
assertLeasePolicy(value.policy);
|
|
@@ -189,6 +197,7 @@ export class ProxyLeaseModel extends plugins.nosqldb.SmartDataDbDoc<ProxyLeaseMo
|
|
|
189
197
|
@plugins.nosqldb.svDb() public policy!: IProxyLeasePolicyDocument;
|
|
190
198
|
@plugins.nosqldb.svDb() public createdAt!: Date;
|
|
191
199
|
@plugins.nosqldb.svDb() public orphanedAt?: Date;
|
|
200
|
+
@plugins.nosqldb.svDb() public route?: 'confined';
|
|
192
201
|
}
|
|
193
202
|
|
|
194
203
|
const proxyModels = [ProxyIdentityModel, ProxyClientModel, ProxyDeliveryModel, ProxyLeaseModel, InferenceAffinityModel,
|
|
@@ -226,6 +235,8 @@ export interface IProxyLease {
|
|
|
226
235
|
createdAt: Date;
|
|
227
236
|
/** See `IProxyLeaseDocument.orphanedAt`. */
|
|
228
237
|
orphanedAt?: Date;
|
|
238
|
+
/** The listener the lease's credential is accepted on; see `IProxyLeaseDocument.route`. */
|
|
239
|
+
route: TInferenceGatewayRoute;
|
|
229
240
|
}
|
|
230
241
|
|
|
231
242
|
/**
|
|
@@ -381,7 +392,7 @@ export class ProxyStore {
|
|
|
381
392
|
};
|
|
382
393
|
if (await ProxyLeaseModel.exact.count({}) >= maximumProxyLeases) throw new Error('The proxy keeps at most 128 leases.');
|
|
383
394
|
const inserted = await ProxyLeaseModel.exact.insert({ id: leaseArg.digest, client: leaseArg.client, policy,
|
|
384
|
-
createdAt: new Date(leaseArg.createdAt.getTime()) });
|
|
395
|
+
createdAt: new Date(leaseArg.createdAt.getTime()), ...(leaseArg.route === 'confined' ? { route: 'confined' as const } : {}) });
|
|
385
396
|
if (inserted.status !== 'inserted' && inserted.status !== 'already_committed') throw new Error('The proxy lease could not be recorded.');
|
|
386
397
|
}
|
|
387
398
|
|
|
@@ -391,7 +402,7 @@ export class ProxyStore {
|
|
|
391
402
|
return stored.map((entryArg) => {
|
|
392
403
|
const lease = ProxyLeaseModel.exact.toPersisted(entryArg);
|
|
393
404
|
return { digest: lease.id, client: lease.client, policy: lease.policy, createdAt: lease.createdAt,
|
|
394
|
-
...(lease.orphanedAt === undefined ? {} : { orphanedAt: lease.orphanedAt }) };
|
|
405
|
+
route: lease.route ?? 'loopback', ...(lease.orphanedAt === undefined ? {} : { orphanedAt: lease.orphanedAt }) };
|
|
395
406
|
});
|
|
396
407
|
}
|
|
397
408
|
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import * as plugins from './plugins.js';
|
|
2
|
+
import type { IWorkerBridgeCommand, IWorkerProcessSnapshot } from './functions.workerprocessproof.js';
|
|
3
|
+
|
|
4
|
+
/** The Claude process a kept binding was last proved against: its PID, start and the digest of its whole snapshot. */
|
|
5
|
+
export interface IWorkerCallerBindingProcess {
|
|
6
|
+
pid: number;
|
|
7
|
+
startTicks: string;
|
|
8
|
+
snapshotDigest: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* What the controller keeps of a fleet Claude worker's current caller launch, so a restarted controller proves the
|
|
13
|
+
* worker's bridge again instead of forgetting it. Never a credential, never the launch id or generation themselves:
|
|
14
|
+
* the overlay holds those, and a bridge presents them; the record keeps their digests to find and check the launch.
|
|
15
|
+
*/
|
|
16
|
+
export interface IWorkerCallerBindingDocument {
|
|
17
|
+
/** The fleet worker's id: a worker keeps one binding, its current launch's. */
|
|
18
|
+
id: string;
|
|
19
|
+
projectId: string;
|
|
20
|
+
harnessId: 'claude';
|
|
21
|
+
/** SHA-256 digests (base64url) of the launch id and of its generation. */
|
|
22
|
+
launchDigest: string;
|
|
23
|
+
generationDigest: string;
|
|
24
|
+
/** The bridge command approved when the launch was prepared, and the Claude Code executable it was proved against. */
|
|
25
|
+
command: IWorkerBridgeCommand;
|
|
26
|
+
claudeExecutable: string;
|
|
27
|
+
revision: number;
|
|
28
|
+
/** Once bound: the background job, its session, and the job's process when one was proved. */
|
|
29
|
+
binding?: { nativeSessionId: string; jobId: string; process?: IWorkerCallerBindingProcess };
|
|
30
|
+
updateId: string;
|
|
31
|
+
}
|
|
32
|
+
export type TWorkerCallerBindingRecord = Omit<IWorkerCallerBindingDocument, 'updateId'>;
|
|
33
|
+
|
|
34
|
+
/** Where the controller keeps the bindings of fleet Claude launches. */
|
|
35
|
+
export interface IWorkerCallerBindingStore {
|
|
36
|
+
/** The binding kept for the launch whose id has the digest `launchDigestArg`. */
|
|
37
|
+
find(launchDigestArg: string): Promise<TWorkerCallerBindingRecord | undefined>;
|
|
38
|
+
/** Keeps `recordArg` as its worker's binding; another launch's binding of the worker is replaced. */
|
|
39
|
+
save(recordArg: TWorkerCallerBindingRecord): Promise<void>;
|
|
40
|
+
/** Deletes the worker's binding; with `launchDigestArg`, only while it is that launch's. */
|
|
41
|
+
delete(workerIdArg: string, launchDigestArg?: string): Promise<void>;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export const workerCallerDigest = (valueArg: string): string => plugins.crypto.createHash('sha256').update(valueArg).digest('base64url');
|
|
45
|
+
export const workerProcessSnapshotDigest = (snapshotArg: IWorkerProcessSnapshot): string => workerCallerDigest(JSON.stringify([
|
|
46
|
+
snapshotArg.pid, snapshotArg.parentPid, snapshotArg.startTicks, snapshotArg.executable, snapshotArg.arguments]));
|
|
47
|
+
|
|
48
|
+
const digest = plugins.z.string().regex(/^[A-Za-z0-9_-]{43}$/u);
|
|
49
|
+
const absolutePath = plugins.z.string().min(2).max(4096).refine((valueArg) => plugins.path.isAbsolute(valueArg) && !valueArg.includes('\0'));
|
|
50
|
+
const documentSchema = plugins.z.object({
|
|
51
|
+
id: plugins.z.string().min(3).max(1024), projectId: plugins.z.string().regex(/^[A-Za-z0-9_.:-]{1,256}$/u),
|
|
52
|
+
harnessId: plugins.z.literal('claude'), launchDigest: digest, generationDigest: digest,
|
|
53
|
+
command: plugins.z.object({ executable: absolutePath, entry: absolutePath,
|
|
54
|
+
arguments: plugins.z.array(plugins.z.string().max(4096)).max(64) }).strict(),
|
|
55
|
+
claudeExecutable: absolutePath, revision: plugins.z.number().int().min(0).max(Number.MAX_SAFE_INTEGER),
|
|
56
|
+
binding: plugins.z.object({ nativeSessionId: plugins.z.string().uuid(), jobId: plugins.z.string().regex(/^[A-Za-z0-9_.:-]{1,256}$/u),
|
|
57
|
+
process: plugins.z.object({ pid: plugins.z.number().int().min(2), startTicks: plugins.z.string().regex(/^\d{1,32}$/u),
|
|
58
|
+
snapshotDigest: digest }).strict().optional() }).strict().optional(),
|
|
59
|
+
updateId: plugins.z.string().uuid(),
|
|
60
|
+
}).strict();
|
|
61
|
+
|
|
62
|
+
@plugins.nosqldb.namedIndex({ name: 'worker_caller_launch', key: { launchDigest: 1 }, options: { unique: true } })
|
|
63
|
+
@plugins.nosqldb.managed({ collectionName: 'agl_worker_caller_bindings' })
|
|
64
|
+
@plugins.nosqldb.exactPersistence({ assertDocument: (input: unknown) => {
|
|
65
|
+
const doc = documentSchema.parse(input);
|
|
66
|
+
if (!doc.id.startsWith(`${doc.projectId}:`)) throw new Error('Worker caller binding identity mismatch.');
|
|
67
|
+
if (doc.binding && doc.revision < 1) throw new Error('Worker caller binding revision mismatch.');
|
|
68
|
+
} })
|
|
69
|
+
export class WorkerCallerBindingModel extends plugins.nosqldb.SmartDataDbDoc<WorkerCallerBindingModel, IWorkerCallerBindingDocument> {
|
|
70
|
+
declare static exact: plugins.nosqldb.TExact<WorkerCallerBindingModel>;
|
|
71
|
+
@plugins.nosqldb.unI() public id!: string;
|
|
72
|
+
@plugins.nosqldb.svDb() public projectId!: string;
|
|
73
|
+
@plugins.nosqldb.svDb() public harnessId!: 'claude';
|
|
74
|
+
@plugins.nosqldb.svDb() public launchDigest!: string;
|
|
75
|
+
@plugins.nosqldb.svDb() public generationDigest!: string;
|
|
76
|
+
@plugins.nosqldb.svDb() public command!: IWorkerBridgeCommand;
|
|
77
|
+
@plugins.nosqldb.svDb() public claudeExecutable!: string;
|
|
78
|
+
@plugins.nosqldb.svDb() public revision!: number;
|
|
79
|
+
@plugins.nosqldb.svDb() public binding?: IWorkerCallerBindingDocument['binding'];
|
|
80
|
+
@plugins.nosqldb.svDb() public updateId!: string;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const ambiguous = (errorArg: unknown): boolean => errorArg instanceof plugins.nosqldb.SmartdataExactPersistenceError && errorArg.code === 'ambiguous_write';
|
|
84
|
+
const record = (documentArg: IWorkerCallerBindingDocument): TWorkerCallerBindingRecord => {
|
|
85
|
+
const { updateId: _updateId, ...rest } = structuredClone(documentArg);
|
|
86
|
+
return rest;
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
/** The kept bindings in the controller's store; the auth store owns the model manager and database. Writes run one at a time. */
|
|
90
|
+
export class WorkerCallerBindingStore implements IWorkerCallerBindingStore {
|
|
91
|
+
private mutationTail: Promise<void> = Promise.resolve();
|
|
92
|
+
|
|
93
|
+
public async find(launchDigestArg: string): Promise<TWorkerCallerBindingRecord | undefined> {
|
|
94
|
+
const stored = await WorkerCallerBindingModel.exact.findStoredOne({ launchDigest: launchDigestArg });
|
|
95
|
+
return stored ? record(WorkerCallerBindingModel.exact.toPersisted(stored)) : undefined;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
public save(recordArg: TWorkerCallerBindingRecord): Promise<void> {
|
|
99
|
+
return this.mutate(async () => {
|
|
100
|
+
const candidate: IWorkerCallerBindingDocument = { ...structuredClone(recordArg), updateId: plugins.crypto.randomUUID() };
|
|
101
|
+
const stored = await WorkerCallerBindingModel.exact.findStoredOne({ id: candidate.id });
|
|
102
|
+
try {
|
|
103
|
+
if (stored) {
|
|
104
|
+
const current = WorkerCallerBindingModel.exact.toPersisted(stored);
|
|
105
|
+
if (current.launchDigest === candidate.launchDigest) {
|
|
106
|
+
// A launch's binding only gains or renews its job; it is never unbound again.
|
|
107
|
+
if (current.binding && !candidate.binding) throw new Error('A kept worker caller binding cannot lose its job.');
|
|
108
|
+
const result = await WorkerCallerBindingModel.exact.transition({ current: stored, change: (model) => { Object.assign(model, candidate); } });
|
|
109
|
+
if (result.status !== 'transitioned') throw new Error('The kept worker caller binding changed concurrently.');
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
const deleted = await WorkerCallerBindingModel.exact.delete({ current: stored });
|
|
113
|
+
if (deleted.status !== 'deleted') throw new Error('The kept worker caller binding changed concurrently.');
|
|
114
|
+
}
|
|
115
|
+
const inserted = await WorkerCallerBindingModel.exact.insert(candidate);
|
|
116
|
+
if (WorkerCallerBindingModel.exact.toPersisted(inserted.document).updateId !== candidate.updateId) {
|
|
117
|
+
throw new Error('Another worker caller binding holds this launch.');
|
|
118
|
+
}
|
|
119
|
+
} catch (error) {
|
|
120
|
+
if (!ambiguous(error)) throw error;
|
|
121
|
+
const after = await WorkerCallerBindingModel.exact.findStoredOne({ id: candidate.id });
|
|
122
|
+
if (!after || WorkerCallerBindingModel.exact.toPersisted(after).updateId !== candidate.updateId) {
|
|
123
|
+
throw new Error('Keeping the worker caller binding has an uncertain outcome.', { cause: error });
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
public delete(workerIdArg: string, launchDigestArg?: string): Promise<void> {
|
|
130
|
+
return this.mutate(async () => {
|
|
131
|
+
const stored = await WorkerCallerBindingModel.exact.findStoredOne({ id: workerIdArg });
|
|
132
|
+
if (!stored || launchDigestArg !== undefined && WorkerCallerBindingModel.exact.toPersisted(stored).launchDigest !== launchDigestArg) return;
|
|
133
|
+
try {
|
|
134
|
+
const result = await WorkerCallerBindingModel.exact.delete({ current: stored });
|
|
135
|
+
if (result.status !== 'deleted') throw new Error('The kept worker caller binding changed concurrently.');
|
|
136
|
+
} catch (error) {
|
|
137
|
+
if (!ambiguous(error) || await WorkerCallerBindingModel.exact.findStoredOne({ id: workerIdArg })) throw error;
|
|
138
|
+
}
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
private mutate(operationArg: () => Promise<void>): Promise<void> {
|
|
143
|
+
const task = this.mutationTail.then(operationArg);
|
|
144
|
+
this.mutationTail = task.then(() => undefined, () => undefined);
|
|
145
|
+
return task;
|
|
146
|
+
}
|
|
147
|
+
}
|