agl 41.1.0 → 42.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.md +48 -0
- package/dist_serve/bundle.js +2190 -2193
- package/dist_serve/bundle.js.map +2 -2
- package/dist_serve/bundle.js.third-party-notices.json +3 -68
- package/dist_serve/bundle.js.third-party-notices.txt +4 -137
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/aglapi/aglapisurface.document.js +4 -4
- package/dist_ts/classes.authmodels.d.ts +2 -8
- package/dist_ts/classes.authmodels.js +60 -23
- package/dist_ts/classes.authstore.d.ts +22 -0
- package/dist_ts/classes.authstore.js +152 -23
- package/dist_ts/classes.cli.d.ts +3 -6
- package/dist_ts/classes.cli.js +38 -51
- package/dist_ts/classes.config.d.ts +17 -3
- package/dist_ts/classes.config.js +55 -18
- package/dist_ts/classes.controller.d.ts +12 -12
- package/dist_ts/classes.controller.js +46 -117
- package/dist_ts/classes.controllermanagement.d.ts +42 -9
- package/dist_ts/classes.controllermanagement.js +72 -23
- package/dist_ts/classes.controllerservice.d.ts +3 -31
- package/dist_ts/classes.controllerservice.js +3 -47
- package/dist_ts/classes.inferencegateway.js +4 -3
- package/dist_ts/classes.inferenceusage.d.ts +19 -7
- package/dist_ts/classes.inferenceusage.js +49 -13
- package/dist_ts/classes.inferenceusagetap.d.ts +15 -3
- package/dist_ts/classes.inferenceusagetap.js +16 -9
- package/dist_ts/classes.mcpcontrollerhost.js +9 -3
- package/dist_ts/classes.processinspection.d.ts +8 -1
- package/dist_ts/classes.processinspection.js +92 -2
- package/dist_ts/classes.proxycontrol.js +6 -1
- package/dist_ts/classes.proxydaemon.d.ts +2 -2
- package/dist_ts/classes.proxydaemon.js +1 -1
- package/dist_ts/classes.proxystore.d.ts +2 -11
- package/dist_ts/classes.proxystore.js +3 -25
- package/dist_ts/classes.tpmsecretcodec.d.ts +13 -0
- package/dist_ts/classes.tpmsecretcodec.js +28 -0
- package/dist_ts/classes.trustedproxysecrets.d.ts +91 -0
- package/dist_ts/classes.trustedproxysecrets.js +338 -0
- package/dist_ts/classes.upgradecoordinator.d.ts +14 -1
- package/dist_ts/classes.upgradecoordinator.js +21 -3
- package/dist_ts/classes.upgradetransaction.d.ts +32 -0
- package/dist_ts/classes.upgradetransaction.js +101 -4
- package/dist_ts/classes.usagefeed.js +3 -2
- package/dist_ts/functions.controlleradmission.d.ts +67 -0
- package/dist_ts/functions.controlleradmission.js +127 -0
- package/dist_ts/functions.remotecli.d.ts +27 -0
- package/dist_ts/functions.remotecli.js +236 -0
- package/dist_ts/functions.retiredenvironment.d.ts +11 -2
- package/dist_ts/functions.retiredenvironment.js +22 -5
- package/dist_ts/index.d.ts +1 -0
- package/dist_ts/index.js +2 -1
- package/dist_ts/interfaces.auth.d.ts +29 -4
- package/dist_ts/interfaces.auth.js +5 -1
- package/dist_ts/interfaces.config.d.ts +14 -0
- package/dist_ts/interfaces.config.js +9 -2
- package/dist_ts/interfaces.lifecycle.d.ts +3 -0
- package/dist_ts_authswitch/classes.authoritydaemon.js +6 -1
- package/dist_ts_interfaces/interfaces.d.ts +2 -2
- package/dist_ts_interfaces/interfaces.js +1 -1
- package/dist_ts_interfaces/management.d.ts +2 -2
- package/dist_ts_interfaces/management.js +2 -2
- package/dist_ts_interfaces/metrics.d.ts +29 -7
- package/dist_ts_interfaces/metrics.js +14 -1
- package/dist_ts_interfaces/requests.d.ts +1 -2
- package/dist_ts_interfaces/requests.js +1 -1
- package/dist_ts_interfaces/requests.upgrade.d.ts +0 -12
- package/package.json +12 -11
- package/readme.md +99 -28
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/aglapi/aglapisurface.document.ts +3 -3
- package/ts/classes.authmodels.ts +68 -25
- package/ts/classes.authstore.ts +151 -24
- package/ts/classes.cli.ts +38 -61
- package/ts/classes.config.ts +70 -21
- package/ts/classes.controller.ts +50 -163
- package/ts/classes.controllermanagement.ts +123 -49
- package/ts/classes.controllerservice.ts +3 -56
- package/ts/classes.inferencegateway.ts +3 -2
- package/ts/classes.inferenceusage.ts +58 -19
- package/ts/classes.inferenceusagetap.ts +27 -8
- package/ts/classes.mcpcontrollerhost.ts +10 -5
- package/ts/classes.processinspection.ts +84 -3
- package/ts/classes.proxycontrol.ts +2 -0
- package/ts/classes.proxydaemon.ts +3 -2
- package/ts/classes.proxystore.ts +4 -25
- package/ts/classes.tpmsecretcodec.ts +26 -0
- package/ts/classes.trustedproxysecrets.ts +317 -0
- package/ts/classes.upgradecoordinator.ts +33 -3
- package/ts/classes.upgradetransaction.ts +130 -1
- package/ts/classes.usagefeed.ts +2 -1
- package/ts/functions.controlleradmission.ts +168 -0
- package/ts/functions.remotecli.ts +264 -0
- package/ts/functions.retiredenvironment.ts +25 -3
- package/ts/index.ts +1 -0
- package/ts/interfaces.auth.ts +36 -3
- package/ts/interfaces.config.ts +15 -0
- package/ts/interfaces.lifecycle.ts +3 -0
- package/ts_authswitch/classes.authoritydaemon.ts +2 -0
- package/ts_interfaces/interfaces.ts +2 -2
- package/ts_interfaces/management.ts +2 -2
- package/ts_interfaces/metrics.ts +34 -7
- package/ts_interfaces/requests.ts +0 -10
- package/ts_interfaces/requests.upgrade.ts +0 -16
- package/ts_web/00_commitinfo_data.ts +1 -1
- package/ts_web/classes.controllersocketclient.ts +5 -2
- package/ts_web/elements.metricsboard.ts +68 -18
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
} from '../ts_interfaces/index.js';
|
|
17
17
|
import {
|
|
18
18
|
controllerProcessIdentityIsLive,
|
|
19
|
+
installedControllerHoldsPort,
|
|
19
20
|
signalVerifiedControllerProcess,
|
|
20
21
|
} from './classes.processinspection.js';
|
|
21
22
|
import { AglControllerService } from './classes.controllerservice.js';
|
|
@@ -38,8 +39,13 @@ export const isControllerManagementProtocolVersion = (valueArg: unknown): valueA
|
|
|
38
39
|
typeof valueArg === 'number' && Number.isSafeInteger(valueArg) && valueArg >= 1
|
|
39
40
|
);
|
|
40
41
|
|
|
41
|
-
/**
|
|
42
|
+
/**
|
|
43
|
+
* A controller a management call is addressed to: what it declares about its management endpoint, which decides
|
|
44
|
+
* whether this build manages it at all, and its package and version, which only a refusal names.
|
|
45
|
+
*/
|
|
42
46
|
export interface IControllerManagementTarget {
|
|
47
|
+
packageName: string;
|
|
48
|
+
packageVersion: string;
|
|
43
49
|
/**
|
|
44
50
|
* The management protocol the controller declares (`controllerManagementProtocolVersion`): the status's own
|
|
45
51
|
* `managementProtocolVersion` once the controller runs, or what the installed package that runs it declares in
|
|
@@ -48,6 +54,51 @@ export interface IControllerManagementTarget {
|
|
|
48
54
|
managementProtocolVersion?: number;
|
|
49
55
|
}
|
|
50
56
|
|
|
57
|
+
/**
|
|
58
|
+
* The oldest management protocol this build manages a controller through. It speaks only the loopback management
|
|
59
|
+
* endpoint: a controller that declares none (AGL 41.0.x or older, reachable only over its TypedSocket 8 socket, which
|
|
60
|
+
* this build cannot speak) is refused by name, never reached some other way.
|
|
61
|
+
*/
|
|
62
|
+
export const controllerManagementProtocolFloor = 1;
|
|
63
|
+
|
|
64
|
+
/** What a refusal of an older controller tells its operator, in the CLI and in the upgrade log. */
|
|
65
|
+
export const controllerManagementUpgradeOrder = 'AGL 41.0.x or older must upgrade to 41.1.x before 42.';
|
|
66
|
+
|
|
67
|
+
/** The first release whose controller has the loopback management endpoint. */
|
|
68
|
+
export const controllerManagementEndpointRelease = '41.1.0';
|
|
69
|
+
|
|
70
|
+
/** Whether a declaration is one this build manages a controller through. */
|
|
71
|
+
export const isManagedControllerManagementProtocol = (valueArg: unknown): boolean => (
|
|
72
|
+
isControllerManagementProtocolVersion(valueArg) && valueArg >= controllerManagementProtocolFloor
|
|
73
|
+
);
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* The controller a refusal names: a release by package and version, when its declaration or an upgrade transaction
|
|
77
|
+
* names one, or only the port, when nothing does.
|
|
78
|
+
*/
|
|
79
|
+
export type TUnmanagedController =
|
|
80
|
+
| Pick<IControllerManagementTarget, 'packageName' | 'packageVersion'>
|
|
81
|
+
| { port: number };
|
|
82
|
+
|
|
83
|
+
/** The refusal of a controller this build cannot manage; its version appears only here. */
|
|
84
|
+
export const unmanagedControllerError = (controllerArg: TUnmanagedController): ControllerManagementError => (
|
|
85
|
+
new ControllerManagementError(
|
|
86
|
+
'port' in controllerArg
|
|
87
|
+
? `The controller on port ${controllerArg.port} has no loopback management endpoint: it is a controller older `
|
|
88
|
+
+ `than ${controllerManagementEndpointRelease}, which this release does not manage. ${controllerManagementUpgradeOrder}`
|
|
89
|
+
: `The controller of ${controllerArg.packageName}@${controllerArg.packageVersion} predates the loopback management `
|
|
90
|
+
+ `endpoint, through which alone this release manages a controller. ${controllerManagementUpgradeOrder}`,
|
|
91
|
+
'management_protocol_unsupported',
|
|
92
|
+
)
|
|
93
|
+
);
|
|
94
|
+
|
|
95
|
+
/** Refuses, by name, a controller whose declaration this build cannot manage it through. */
|
|
96
|
+
export const assertManagedController = (targetArg: IControllerManagementTarget): void => {
|
|
97
|
+
if (!isManagedControllerManagementProtocol(targetArg.managementProtocolVersion)) {
|
|
98
|
+
throw unmanagedControllerError(targetArg);
|
|
99
|
+
}
|
|
100
|
+
};
|
|
101
|
+
|
|
51
102
|
/**
|
|
52
103
|
* A status read: the identity the status must report, and the declaration that decides how it is read, which comes
|
|
53
104
|
* from the package expected to run the controller, since there is no status yet. Every caller states it, absent
|
|
@@ -55,6 +106,13 @@ export interface IControllerManagementTarget {
|
|
|
55
106
|
*/
|
|
56
107
|
export interface IControllerStatusQuery extends IControllerStatusExpectation {
|
|
57
108
|
managementProtocolVersion: number | undefined;
|
|
109
|
+
/**
|
|
110
|
+
* Where `packageName` and `packageVersion` come from: an upgrade transaction (`transaction`), which names the
|
|
111
|
+
* release it left on the port, or only the caller's own expectation (`caller`, the CLI's own build). A controller
|
|
112
|
+
* without the management endpoint is refused by the transaction's release, or else as one older than
|
|
113
|
+
* `controllerManagementEndpointRelease`, never by a version nobody read from it.
|
|
114
|
+
*/
|
|
115
|
+
identity: 'transaction' | 'caller';
|
|
58
116
|
}
|
|
59
117
|
|
|
60
118
|
const harnessStates = ['starting', 'ready', 'stopping', 'stopped', 'failed'] as const;
|
|
@@ -224,19 +282,6 @@ export const assertControllerStatus = (
|
|
|
224
282
|
return readableStatus as IControllerStatus;
|
|
225
283
|
};
|
|
226
284
|
|
|
227
|
-
/**
|
|
228
|
-
* How this build reaches a controller's management methods, decided by the controller's declaration alone: over its
|
|
229
|
-
* loopback management endpoint when it declares a management protocol, which every later release keeps whatever
|
|
230
|
-
* transport its browser speaks, and over its controller socket when it declares none (a release older than the
|
|
231
|
-
* endpoint, such as a source an upgrade recovery restores), which serves them in every release up to this one.
|
|
232
|
-
* There is no second attempt over the other transport.
|
|
233
|
-
*/
|
|
234
|
-
export const controllerManagementTransport = (
|
|
235
|
-
targetArg: IControllerManagementTarget,
|
|
236
|
-
): 'http' | 'socket' => (
|
|
237
|
-
isControllerManagementProtocolVersion(targetArg.managementProtocolVersion) ? 'http' : 'socket'
|
|
238
|
-
);
|
|
239
|
-
|
|
240
285
|
/** A management request the controller answered with an error, or whose answer this build cannot read. */
|
|
241
286
|
export class ControllerManagementError extends Error {
|
|
242
287
|
constructor(messageArg: string, public readonly code?: string) {
|
|
@@ -281,6 +326,12 @@ const readControllerManagementAnswer = (
|
|
|
281
326
|
throw new ControllerManagementError(text, code);
|
|
282
327
|
};
|
|
283
328
|
|
|
329
|
+
/**
|
|
330
|
+
* The `Allow` header of the 405 with which the controller of every release before the loopback management endpoint
|
|
331
|
+
* refuses a `POST` (its `requestAdmission` admits only GET and HEAD; TypedServer 14.2.1 sends the header as given).
|
|
332
|
+
*/
|
|
333
|
+
const preEndpointControllerAllow = 'GET, HEAD';
|
|
334
|
+
|
|
284
335
|
/** Sends one envelope to the management endpoint on 127.0.0.1 and returns the answer's bytes. */
|
|
285
336
|
const postControllerManagementEnvelope = (
|
|
286
337
|
portArg: number,
|
|
@@ -305,6 +356,16 @@ const postControllerManagementEnvelope = (
|
|
|
305
356
|
}, (responseArg) => {
|
|
306
357
|
const statusCode = responseArg.statusCode ?? 0;
|
|
307
358
|
const contentType = responseArg.headers['content-type'];
|
|
359
|
+
if (statusCode === 405 && responseArg.headers.allow === preEndpointControllerAllow) {
|
|
360
|
+
// The endpoint takes every POST it admits, so this is a listener without it that answers as the controller of a
|
|
361
|
+
// release older than the endpoint does; whether it is one, `requestManagedController` confirms.
|
|
362
|
+
responseArg.resume();
|
|
363
|
+
reject(new ControllerManagementError(
|
|
364
|
+
'The listener answered as a controller without the loopback management endpoint.',
|
|
365
|
+
'management_endpoint_missing',
|
|
366
|
+
));
|
|
367
|
+
return;
|
|
368
|
+
}
|
|
308
369
|
if (statusCode !== 200) {
|
|
309
370
|
responseArg.resume();
|
|
310
371
|
reject(new ControllerManagementError(
|
|
@@ -384,41 +445,38 @@ export const requestControllerManagement = async <T extends TControllerManagemen
|
|
|
384
445
|
return readControllerManagementAnswer(answer, identity);
|
|
385
446
|
};
|
|
386
447
|
|
|
387
|
-
/**
|
|
388
|
-
|
|
448
|
+
/**
|
|
449
|
+
* Sends one management request to a controller that declares the endpoint. One that turns out to have none is
|
|
450
|
+
* refused as `refusalArg` names it: by default the target, whose identity is what it declared or reported.
|
|
451
|
+
*
|
|
452
|
+
* It has none only when both signs of an older controller hold: the listener refused the `POST` with that
|
|
453
|
+
* controller's 405 (`Allow: GET, HEAD`), and the process holding the port's listening socket runs an installed AGL
|
|
454
|
+
* package's controller for this port (`installedControllerHoldsPort`). Any other listener, a 405 of its own included,
|
|
455
|
+
* is a refusal of HTTP (`http_refused`), which callers treat as a busy port, never as an AGL controller.
|
|
456
|
+
*/
|
|
457
|
+
const requestManagedController = async <T extends TControllerManagementTypedRequest>(
|
|
389
458
|
portArg: number,
|
|
459
|
+
controllerArg: IControllerManagementTarget,
|
|
390
460
|
methodArg: T['method'],
|
|
391
461
|
requestArg: T['request'],
|
|
392
462
|
optionsArg: { timeoutMs: number },
|
|
463
|
+
refusalArg: TUnmanagedController = controllerArg,
|
|
393
464
|
): Promise<unknown> => {
|
|
394
|
-
|
|
395
|
-
let socket: plugins.typedsocket.TypedSocket | undefined;
|
|
465
|
+
assertManagedController(controllerArg);
|
|
396
466
|
try {
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
} finally {
|
|
408
|
-
await socket?.stop();
|
|
467
|
+
return await requestControllerManagement<T>(portArg, methodArg, requestArg, optionsArg);
|
|
468
|
+
} catch (errorArg) {
|
|
469
|
+
if (errorArg instanceof ControllerManagementError && errorArg.code === 'management_endpoint_missing') {
|
|
470
|
+
if (await installedControllerHoldsPort(portArg)) throw unmanagedControllerError(refusalArg);
|
|
471
|
+
throw new ControllerManagementError(
|
|
472
|
+
'The controller refused the management request with HTTP 405.',
|
|
473
|
+
'http_refused',
|
|
474
|
+
);
|
|
475
|
+
}
|
|
476
|
+
throw errorArg;
|
|
409
477
|
}
|
|
410
478
|
};
|
|
411
479
|
|
|
412
|
-
const requestManagedController = <T extends TControllerManagementTypedRequest>(
|
|
413
|
-
portArg: number,
|
|
414
|
-
controllerArg: IControllerManagementTarget,
|
|
415
|
-
methodArg: T['method'],
|
|
416
|
-
requestArg: T['request'],
|
|
417
|
-
optionsArg: { timeoutMs: number },
|
|
418
|
-
): Promise<unknown> => controllerManagementTransport(controllerArg) === 'http'
|
|
419
|
-
? requestControllerManagement<T>(portArg, methodArg, requestArg, optionsArg)
|
|
420
|
-
: requestControllerManagementOverSocket<T>(portArg, methodArg, requestArg, optionsArg);
|
|
421
|
-
|
|
422
480
|
export const requestControllerUpgradePrepareBegin = async (
|
|
423
481
|
portArg: number,
|
|
424
482
|
controllerArg: IControllerManagementTarget,
|
|
@@ -474,24 +532,40 @@ export const queryControllerStatus = async (
|
|
|
474
532
|
portArg: number,
|
|
475
533
|
queryArg: IControllerStatusQuery,
|
|
476
534
|
timeoutMsArg = 2_500,
|
|
477
|
-
): Promise<IControllerStatus> =>
|
|
478
|
-
|
|
479
|
-
|
|
535
|
+
): Promise<IControllerStatus> => {
|
|
536
|
+
const status = assertControllerStatus(
|
|
537
|
+
await requestManagedController<IReq_ControllerStatus>(
|
|
538
|
+
portArg,
|
|
539
|
+
queryArg,
|
|
540
|
+
'controller.status',
|
|
541
|
+
{},
|
|
542
|
+
{ timeoutMs: timeoutMsArg },
|
|
543
|
+
queryArg.identity === 'transaction' ? queryArg : { port: portArg },
|
|
544
|
+
),
|
|
480
545
|
queryArg,
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
546
|
+
);
|
|
547
|
+
// From here on the controller's own report is the gate, whatever its package declared before it ran.
|
|
548
|
+
assertManagedController(status);
|
|
549
|
+
return status;
|
|
550
|
+
};
|
|
551
|
+
|
|
552
|
+
/** Whether `errorArg` refuses a controller this build cannot manage, which is there all the same. */
|
|
553
|
+
export const isUnmanagedControllerError = (errorArg: unknown): errorArg is ControllerManagementError => (
|
|
554
|
+
errorArg instanceof ControllerManagementError && errorArg.code === 'management_protocol_unsupported'
|
|
486
555
|
);
|
|
487
556
|
|
|
557
|
+
/**
|
|
558
|
+
* Reads the status of a controller if one this build manages answers; `undefined` when none answers or its answer is
|
|
559
|
+
* not one. A controller that answers but that this build cannot manage is refused, never reported as absent.
|
|
560
|
+
*/
|
|
488
561
|
export const tryQueryControllerStatus = async (
|
|
489
562
|
portArg: number,
|
|
490
563
|
queryArg: IControllerStatusQuery,
|
|
491
564
|
): Promise<IControllerStatus | undefined> => {
|
|
492
565
|
try {
|
|
493
566
|
return await queryControllerStatus(portArg, queryArg);
|
|
494
|
-
} catch {
|
|
567
|
+
} catch (errorArg) {
|
|
568
|
+
if (isUnmanagedControllerError(errorArg)) throw errorArg;
|
|
495
569
|
return undefined;
|
|
496
570
|
}
|
|
497
571
|
};
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import * as plugins from './plugins.js';
|
|
2
2
|
import { controllerServiceUnitName } from '../ts_interfaces/index.js';
|
|
3
3
|
import { resolveAGLHomePaths } from './classes.aglhome.js';
|
|
4
|
-
import {
|
|
5
|
-
import type { IControllerRuntimeOverrides, TControllerTlsMode } from './interfaces.config.js';
|
|
4
|
+
import type { IControllerRuntimeOverrides } from './interfaces.config.js';
|
|
6
5
|
import { prepareControllerLogFile, resolveControllerLogFilePath } from './functions.controllerlog.js';
|
|
7
6
|
|
|
8
7
|
/**
|
|
@@ -15,50 +14,6 @@ export const controllerServiceStandDownExitStatus = 78;
|
|
|
15
14
|
/** The oldest systemd whose manager reports the loaded `ExecStart` and environment a start is matched against. */
|
|
16
15
|
export const controllerServiceManagerMajor = 254;
|
|
17
16
|
|
|
18
|
-
/**
|
|
19
|
-
* The unit cannot run a controller behind a TLS proxy: that controller needs `AGL_TRUSTED_PROXY_SECRET`, and the
|
|
20
|
-
* unit carries no secret (its file is readable by every local user). `remedy` says how to run it detached instead.
|
|
21
|
-
*/
|
|
22
|
-
export class ControllerServiceTlsProxyRefusal extends Error {
|
|
23
|
-
constructor(remedyArg: string) {
|
|
24
|
-
super(`${controllerServiceUnitName} cannot run AGL behind a TLS proxy: the unit cannot carry `
|
|
25
|
-
+ `${trustedProxySecretEnvironmentVariable}, which that mode needs. ${remedyArg}`);
|
|
26
|
-
this.name = 'ControllerServiceTlsProxyRefusal';
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
/**
|
|
31
|
-
* Whether a failed start was refused because the unit cannot run this controller: the refusal itself, or one inside
|
|
32
|
-
* the `AggregateError` a start throws when releasing its resources failed as well. Such a start fails the same way
|
|
33
|
-
* every time, so the unit stands down rather than being restarted.
|
|
34
|
-
*/
|
|
35
|
-
export const isControllerServiceTlsProxyRefusal = (errorArg: unknown): boolean =>
|
|
36
|
-
errorArg instanceof ControllerServiceTlsProxyRefusal
|
|
37
|
-
|| (errorArg instanceof AggregateError && errorArg.errors.some(isControllerServiceTlsProxyRefusal));
|
|
38
|
-
|
|
39
|
-
/** What to do when a controller behind a TLS proxy meets an installed unit: leave the unit and run detached. */
|
|
40
|
-
export const controllerServiceDetachedRemedy = 'Run it detached: `agl service uninstall`, then `agl start`.';
|
|
41
|
-
/** What to do when `agl service install` refuses such a controller: nothing was written, so run it detached. */
|
|
42
|
-
export const controllerServiceInstallRemedy = 'Run it detached with `agl start`.';
|
|
43
|
-
|
|
44
|
-
/**
|
|
45
|
-
* Refuses a controller the unit could not run: one whose stored runtime configuration is in TLS-proxy mode, or
|
|
46
|
-
* startup options that ask for that mode (`--behind-tls-proxy`, or an HTTPS `--public-origin`, which only that
|
|
47
|
-
* mode serves).
|
|
48
|
-
*/
|
|
49
|
-
export const assertControllerServiceTlsMode = (optionsArg: {
|
|
50
|
-
requested?: Pick<IControllerRuntimeOverrides, 'behindTlsProxy' | 'publicOrigin'>;
|
|
51
|
-
stored?: TControllerTlsMode;
|
|
52
|
-
remedy: string;
|
|
53
|
-
}): void => {
|
|
54
|
-
const origin = optionsArg.requested?.publicOrigin;
|
|
55
|
-
if (
|
|
56
|
-
optionsArg.stored === 'proxy'
|
|
57
|
-
|| optionsArg.requested?.behindTlsProxy === true
|
|
58
|
-
|| (origin !== undefined && URL.canParse(origin) && new URL(origin).protocol === 'https:')
|
|
59
|
-
) throw new ControllerServiceTlsProxyRefusal(optionsArg.remedy);
|
|
60
|
-
};
|
|
61
|
-
|
|
62
17
|
/** The startup options the unit's `ExecStart` carries; `--port` aside, every one is optional. */
|
|
63
18
|
export type IControllerServiceOverrides = Omit<IControllerRuntimeOverrides, 'controllerPort' | 'initialProjectDirectory'>;
|
|
64
19
|
|
|
@@ -163,7 +118,6 @@ export class ControllerServiceEnvironmentRefusal extends Error {
|
|
|
163
118
|
*/
|
|
164
119
|
export const controllerServiceEnvironment = (environmentArg: NodeJS.ProcessEnv): Record<string, string> => {
|
|
165
120
|
const secrets: string[] = [];
|
|
166
|
-
if (environmentArg[trustedProxySecretEnvironmentVariable]) secrets.push(trustedProxySecretEnvironmentVariable);
|
|
167
121
|
for (const name of urlVariables) {
|
|
168
122
|
const value = environmentArg[name];
|
|
169
123
|
if (value && urlCarriesCredential(value)) secrets.push(name);
|
|
@@ -287,7 +241,6 @@ export class AglControllerService {
|
|
|
287
241
|
if (!Number.isInteger(optionsArg.port) || optionsArg.port < 1 || optionsArg.port > 65_535) {
|
|
288
242
|
throw new Error('The controller port must be 1 to 65535.');
|
|
289
243
|
}
|
|
290
|
-
assertControllerServiceTlsMode({ requested: optionsArg.overrides, remedy: controllerServiceInstallRemedy });
|
|
291
244
|
return new plugins.smartdaemon.SystemdServiceDefinition({
|
|
292
245
|
unitName: controllerServiceUnitName,
|
|
293
246
|
scope: 'user',
|
|
@@ -404,9 +357,8 @@ export type TControllerStartRoute =
|
|
|
404
357
|
/**
|
|
405
358
|
* The unit runs the controller only for the AGL home and port it was installed for. A start for that home and
|
|
406
359
|
* port with startup options or environment the unit does not carry is refused rather than run with settings
|
|
407
|
-
* other than the ones asked for; every other home and port starts detached.
|
|
408
|
-
*
|
|
409
|
-
* no `environment`: it restarts the controller as installed, from inside the unit's own environment.
|
|
360
|
+
* other than the ones asked for; every other home and port starts detached. An upgrade's restart passes no
|
|
361
|
+
* `environment`: it restarts the controller as installed, from inside the unit's own environment.
|
|
410
362
|
*/
|
|
411
363
|
export const resolveControllerStartRoute = async (optionsArg: {
|
|
412
364
|
service: AglControllerService;
|
|
@@ -426,11 +378,6 @@ export const resolveControllerStartRoute = async (optionsArg: {
|
|
|
426
378
|
if (installed.aglHome !== optionsArg.aglHome || installed.execStart.port !== optionsArg.port) {
|
|
427
379
|
return { kind: 'detached' };
|
|
428
380
|
}
|
|
429
|
-
assertControllerServiceTlsMode({ requested: optionsArg.requested, remedy: controllerServiceDetachedRemedy });
|
|
430
|
-
if (optionsArg.environment?.[trustedProxySecretEnvironmentVariable]) {
|
|
431
|
-
throw new ControllerServiceTlsProxyRefusal('To start the unit, unset it in this shell; to run behind a TLS proxy, '
|
|
432
|
-
+ 'run it detached: `agl service uninstall`, then `agl start`.');
|
|
433
|
-
}
|
|
434
381
|
const differences: string[] = [];
|
|
435
382
|
for (const key of Object.keys(overrideFlags) as Array<keyof IControllerServiceOverrides>) {
|
|
436
383
|
const requested = optionsArg.requested[key];
|
|
@@ -1009,10 +1009,11 @@ export class InferenceGateway {
|
|
|
1009
1009
|
if (spent) this.options.router.recordSpend(accountId, route.loginId, spent);
|
|
1010
1010
|
else this.options.router.uncountedSpend(accountId, answeredArg.model ?? route.model.modelId);
|
|
1011
1011
|
} : undefined)
|
|
1012
|
-
: new InferenceClaudeUsageTap(answerFormat, (usageArg) => {
|
|
1012
|
+
: new InferenceClaudeUsageTap(answerFormat, (usageArg, cacheStatedArg) => {
|
|
1013
1013
|
if (route.provider === 'anthropic') this.options.router.countTokens(accountId, usageArg);
|
|
1014
1014
|
recordUsage({ inputTokens: usageArg.inputTokens, outputTokens: usageArg.outputTokens,
|
|
1015
|
-
cacheReadTokens:
|
|
1015
|
+
cacheReadTokens: cacheStatedArg.reads ? usageArg.cacheReadTokens : null,
|
|
1016
|
+
cacheWriteTokens: cacheStatedArg.writes ? usageArg.cacheWrite5mTokens + usageArg.cacheWrite1hTokens : null });
|
|
1016
1017
|
// An API key's answer is priced as it ends, and its spend counts against the key's limit from then on.
|
|
1017
1018
|
}, route.provider === 'anthropic' && route.credential === 'api_key' ? (answeredArg) => {
|
|
1018
1019
|
const spent = inferenceAnthropicSpend(answeredArg, route.model.modelId, Date.now(), inferenceAnthropicServerToolTypes(body));
|
|
@@ -1,16 +1,19 @@
|
|
|
1
1
|
import * as plugins from './plugins.js';
|
|
2
2
|
import { inferenceProviders, type TInferenceProvider } from './functions.proxycontract.js';
|
|
3
|
-
import type { IControllerUsageBucket, TControllerUsageSpan } from '../ts_interfaces/index.js';
|
|
3
|
+
import type { IControllerUsageBucket, IControllerUsageCounts, TControllerUsageSpan } from '../ts_interfaces/index.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* The tokens one answer used, by kind. Input counts only the tokens read neither from nor written to the provider's
|
|
7
7
|
* prompt cache; the cache reads and writes are counted on their own, so the four add up to everything the answer used.
|
|
8
|
+
* `cacheReadTokens` and `cacheWriteTokens` are `null` when the answer did not state them, which is not the same as
|
|
9
|
+
* stating none. Whether the input count then holds them depends on the provider (`controllerUsageTotalIsLowerBound`):
|
|
10
|
+
* an OpenAI account's input counts every input token, an Anthropic account's leaves the cache out.
|
|
8
11
|
*/
|
|
9
12
|
export interface IInferenceTokenCounts {
|
|
10
13
|
inputTokens: number;
|
|
11
14
|
outputTokens: number;
|
|
12
|
-
cacheReadTokens: number;
|
|
13
|
-
cacheWriteTokens: number;
|
|
15
|
+
cacheReadTokens: number | null;
|
|
16
|
+
cacheWriteTokens: number | null;
|
|
14
17
|
}
|
|
15
18
|
|
|
16
19
|
/**
|
|
@@ -33,8 +36,12 @@ export interface IInferenceUsageLedger {
|
|
|
33
36
|
record(recordArg: IInferenceUsageRecord): void;
|
|
34
37
|
}
|
|
35
38
|
|
|
39
|
+
/** The counts a bucket stored before unknown cache counts were kept lacks. */
|
|
40
|
+
type TUnknownCountKey = 'cacheReadUnknownCalls' | 'cacheWriteUnknownCalls';
|
|
41
|
+
const unknownCountKeys: readonly TUnknownCountKey[] = ['cacheReadUnknownCalls', 'cacheWriteUnknownCalls'];
|
|
42
|
+
|
|
36
43
|
/** One minute or hour of one model's tokens on one harness, account and session, as the proxy keeps it. */
|
|
37
|
-
export interface IInferenceUsageBucket extends
|
|
44
|
+
export interface IInferenceUsageBucket extends IControllerUsageCounts {
|
|
38
45
|
/** The bucket's digest: SHA-256 of its span, start and dimensions. */
|
|
39
46
|
id: string;
|
|
40
47
|
span: TControllerUsageSpan;
|
|
@@ -45,16 +52,29 @@ export interface IInferenceUsageBucket extends IInferenceTokenCounts {
|
|
|
45
52
|
harness: string | null;
|
|
46
53
|
accountId: string;
|
|
47
54
|
session: string | null;
|
|
48
|
-
/** The answers counted. */
|
|
49
|
-
calls: number;
|
|
50
55
|
/** When a flush last added to the bucket. */
|
|
51
56
|
updatedAt: Date;
|
|
52
57
|
}
|
|
53
58
|
|
|
54
|
-
|
|
59
|
+
/**
|
|
60
|
+
* A stored bucket. One written before unknown cache counts were kept has neither `cacheReadUnknownCalls` nor
|
|
61
|
+
* `cacheWriteUnknownCalls`: it did not record whether its answers stated their cache reads and writes, so each of its
|
|
62
|
+
* answers reads as one that stated neither.
|
|
63
|
+
*/
|
|
64
|
+
export interface IInferenceUsageDocument extends Omit<IInferenceUsageBucket, TUnknownCountKey>,
|
|
65
|
+
Partial<Pick<IInferenceUsageBucket, TUnknownCountKey>> {
|
|
55
66
|
expiresAt: Date;
|
|
56
67
|
}
|
|
57
68
|
|
|
69
|
+
/** A stored bucket's counts, those of a bucket written before unknown cache counts were kept with every answer unknown. */
|
|
70
|
+
const storedCounts = (documentArg: Omit<IInferenceUsageDocument, 'id' | 'span' | 'start' | 'provider' | 'model' | 'harness'
|
|
71
|
+
| 'accountId' | 'session' | 'updatedAt' | 'expiresAt'>): IControllerUsageCounts => ({
|
|
72
|
+
calls: documentArg.calls, inputTokens: documentArg.inputTokens, outputTokens: documentArg.outputTokens,
|
|
73
|
+
cacheReadTokens: documentArg.cacheReadTokens, cacheWriteTokens: documentArg.cacheWriteTokens,
|
|
74
|
+
cacheReadUnknownCalls: documentArg.cacheReadUnknownCalls ?? documentArg.calls,
|
|
75
|
+
cacheWriteUnknownCalls: documentArg.cacheWriteUnknownCalls ?? documentArg.calls,
|
|
76
|
+
});
|
|
77
|
+
|
|
58
78
|
/** A usage flush that failed, as the proxy reports it: no session id, prompt or account id. */
|
|
59
79
|
export interface IInferenceUsageFlushFailure {
|
|
60
80
|
/** The buckets kept for the next flush. */
|
|
@@ -96,10 +116,14 @@ const pageDocuments = 128;
|
|
|
96
116
|
|
|
97
117
|
const validDate = (valueArg: unknown): valueArg is Date => valueArg instanceof Date && Number.isFinite(valueArg.getTime());
|
|
98
118
|
const validCount = (valueArg: unknown): valueArg is number => Number.isSafeInteger(valueArg) && (valueArg as number) >= 0;
|
|
119
|
+
/** A count of a bucket's answers: whole, and at most the answers it counts. */
|
|
120
|
+
const countWithin = (countArg: unknown, callsArg: unknown): boolean => validCount(countArg) && validCount(callsArg) && countArg <= callsArg;
|
|
99
121
|
|
|
100
122
|
const assertDocument: (valueArg: unknown) => asserts valueArg is IInferenceUsageDocument = (valueArg) => {
|
|
101
123
|
const value = valueArg as Partial<IInferenceUsageDocument> | null;
|
|
102
|
-
|
|
124
|
+
// A bucket written before unknown cache counts were kept has neither of them; every bucket written since has both.
|
|
125
|
+
const kept = !!value && typeof value === 'object' && Object.hasOwn(value, 'cacheWriteUnknownCalls');
|
|
126
|
+
if (!value || typeof value !== 'object' || Object.keys(value).length !== 15 + (kept ? unknownCountKeys.length : 0)
|
|
103
127
|
|| typeof value.id !== 'string' || !hex64.test(value.id)
|
|
104
128
|
|| !usageSpans.includes(value.span as TControllerUsageSpan)
|
|
105
129
|
|| !validDate(value.start) || value.start.getTime() % spanMs[value.span as TControllerUsageSpan] !== 0
|
|
@@ -111,6 +135,7 @@ const assertDocument: (valueArg: unknown) => asserts valueArg is IInferenceUsage
|
|
|
111
135
|
|| !validCount(value.calls) || value.calls < 1
|
|
112
136
|
|| !validCount(value.inputTokens) || !validCount(value.outputTokens)
|
|
113
137
|
|| !validCount(value.cacheReadTokens) || !validCount(value.cacheWriteTokens)
|
|
138
|
+
|| (kept && !unknownCountKeys.every((keyArg) => countWithin(value[keyArg], value.calls)))
|
|
114
139
|
|| !validDate(value.updatedAt) || !validDate(value.expiresAt)) {
|
|
115
140
|
throw new Error('Invalid inference usage bucket.');
|
|
116
141
|
}
|
|
@@ -134,6 +159,9 @@ export class InferenceUsageModel extends plugins.nosqldb.SmartDataDbDoc<Inferenc
|
|
|
134
159
|
@plugins.nosqldb.svDb() public outputTokens!: number;
|
|
135
160
|
@plugins.nosqldb.svDb() public cacheReadTokens!: number;
|
|
136
161
|
@plugins.nosqldb.svDb() public cacheWriteTokens!: number;
|
|
162
|
+
/** These two are absent only on a bucket written before unknown cache counts were kept; every write sets them. */
|
|
163
|
+
@plugins.nosqldb.svDb() public cacheReadUnknownCalls?: number;
|
|
164
|
+
@plugins.nosqldb.svDb() public cacheWriteUnknownCalls?: number;
|
|
137
165
|
@plugins.nosqldb.svDb()
|
|
138
166
|
@plugins.nosqldb.index() public updatedAt!: Date;
|
|
139
167
|
@plugins.nosqldb.svDb()
|
|
@@ -259,9 +287,11 @@ export class InferenceUsageStore implements IInferenceUsageLedger {
|
|
|
259
287
|
public record(recordArg: IInferenceUsageRecord): void {
|
|
260
288
|
try {
|
|
261
289
|
if (this.stopped) return;
|
|
262
|
-
const counts = [recordArg.inputTokens, recordArg.outputTokens, recordArg.cacheReadTokens, recordArg.cacheWriteTokens];
|
|
290
|
+
const counts = [recordArg.inputTokens, recordArg.outputTokens, recordArg.cacheReadTokens ?? 0, recordArg.cacheWriteTokens ?? 0];
|
|
263
291
|
if (!inferenceProviders.includes(recordArg.provider) || !usageModelPattern.test(recordArg.model)
|
|
264
292
|
|| (recordArg.harness !== null && !usageHarnessPattern.test(recordArg.harness)) || !hex64.test(recordArg.accountId)
|
|
293
|
+
|| (recordArg.cacheReadTokens !== null && !validCount(recordArg.cacheReadTokens))
|
|
294
|
+
|| (recordArg.cacheWriteTokens !== null && !validCount(recordArg.cacheWriteTokens))
|
|
265
295
|
|| !counts.every(validCount) || counts.every((countArg) => countArg === 0)) {
|
|
266
296
|
this.droppedRecords += 1;
|
|
267
297
|
return;
|
|
@@ -280,12 +310,12 @@ export class InferenceUsageStore implements IInferenceUsageLedger {
|
|
|
280
310
|
this.droppedRecords += 1;
|
|
281
311
|
return;
|
|
282
312
|
}
|
|
313
|
+
const answer = answerCounts(recordArg);
|
|
283
314
|
for (const { span, start, id } of buckets) {
|
|
284
315
|
const held = this.pending.get(id);
|
|
285
|
-
if (held) { addCounts(held,
|
|
316
|
+
if (held) { addCounts(held, answer); continue; }
|
|
286
317
|
this.pending.set(id, { id, span, start: new Date(start), provider: recordArg.provider, model: recordArg.model,
|
|
287
|
-
harness: recordArg.harness, accountId: recordArg.accountId, session,
|
|
288
|
-
outputTokens: recordArg.outputTokens, cacheReadTokens: recordArg.cacheReadTokens, cacheWriteTokens: recordArg.cacheWriteTokens });
|
|
318
|
+
harness: recordArg.harness, accountId: recordArg.accountId, session, ...answer });
|
|
289
319
|
}
|
|
290
320
|
} catch {
|
|
291
321
|
this.droppedRecords += 1;
|
|
@@ -364,7 +394,7 @@ export class InferenceUsageStore implements IInferenceUsageLedger {
|
|
|
364
394
|
// The buckets that were not written join those recorded since, as far as the pending limit allows.
|
|
365
395
|
for (const bucket of failed) {
|
|
366
396
|
const held = this.pending.get(bucket.id);
|
|
367
|
-
if (held) addCounts(held, bucket
|
|
397
|
+
if (held) addCounts(held, bucket);
|
|
368
398
|
else if (this.pending.size < maximumPendingBuckets) this.pending.set(bucket.id, bucket);
|
|
369
399
|
else this.droppedRecords += bucket.calls;
|
|
370
400
|
}
|
|
@@ -388,7 +418,10 @@ export class InferenceUsageStore implements IInferenceUsageLedger {
|
|
|
388
418
|
return;
|
|
389
419
|
}
|
|
390
420
|
const result = await InferenceUsageModel.exact.transition({ current, change: (modelArg) => {
|
|
391
|
-
|
|
421
|
+
// A bucket written before unknown cache counts were kept gets them, its own answers as unknown.
|
|
422
|
+
const counts = storedCounts(modelArg);
|
|
423
|
+
addCounts(counts, bucketArg);
|
|
424
|
+
Object.assign(modelArg, counts);
|
|
392
425
|
modelArg.updatedAt = updatedAt;
|
|
393
426
|
} }, { signal: signalArg });
|
|
394
427
|
if (result.status === 'transitioned') {
|
|
@@ -418,15 +451,21 @@ export class InferenceUsageStore implements IInferenceUsageLedger {
|
|
|
418
451
|
}
|
|
419
452
|
|
|
420
453
|
/** A stored bucket as a reader receives it: its start in Unix milliseconds, without its bookkeeping. */
|
|
421
|
-
const wireBucket = (documentArg:
|
|
454
|
+
const wireBucket = (documentArg: IInferenceUsageDocument): IControllerUsageBucket => ({ id: documentArg.id, span: documentArg.span,
|
|
422
455
|
start: documentArg.start.getTime(), provider: documentArg.provider, model: documentArg.model, harness: documentArg.harness,
|
|
423
|
-
accountId: documentArg.accountId, session: documentArg.session,
|
|
424
|
-
|
|
456
|
+
accountId: documentArg.accountId, session: documentArg.session, ...storedCounts(documentArg) });
|
|
457
|
+
|
|
458
|
+
/** One answer as a bucket counts it: the cache counts it stated, and for each it did not, one answer unknown. */
|
|
459
|
+
const answerCounts = (countsArg: IInferenceTokenCounts): IControllerUsageCounts => ({ calls: 1, inputTokens: countsArg.inputTokens,
|
|
460
|
+
outputTokens: countsArg.outputTokens, cacheReadTokens: countsArg.cacheReadTokens ?? 0, cacheWriteTokens: countsArg.cacheWriteTokens ?? 0,
|
|
461
|
+
cacheReadUnknownCalls: countsArg.cacheReadTokens === null ? 1 : 0, cacheWriteUnknownCalls: countsArg.cacheWriteTokens === null ? 1 : 0 });
|
|
425
462
|
|
|
426
|
-
const addCounts = (targetArg:
|
|
427
|
-
targetArg.calls +=
|
|
463
|
+
const addCounts = (targetArg: IControllerUsageCounts, addArg: IControllerUsageCounts): void => {
|
|
464
|
+
targetArg.calls += addArg.calls;
|
|
428
465
|
targetArg.inputTokens += addArg.inputTokens;
|
|
429
466
|
targetArg.outputTokens += addArg.outputTokens;
|
|
430
467
|
targetArg.cacheReadTokens += addArg.cacheReadTokens;
|
|
431
468
|
targetArg.cacheWriteTokens += addArg.cacheWriteTokens;
|
|
469
|
+
targetArg.cacheReadUnknownCalls += addArg.cacheReadUnknownCalls;
|
|
470
|
+
targetArg.cacheWriteUnknownCalls += addArg.cacheWriteUnknownCalls;
|
|
432
471
|
};
|
|
@@ -29,6 +29,16 @@ interface IStatedUsage {
|
|
|
29
29
|
ephemeral_1h_input_tokens?: number;
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
+
/**
|
|
33
|
+
* Which cache counts an Anthropic Messages answer stated: `reads` its `cache_read_input_tokens`, `writes` its
|
|
34
|
+
* `cache_creation_input_tokens` or its `cache_creation` split. Anthropic documents both as `number or null`, and a
|
|
35
|
+
* translated answer gives null for a count the model did not state.
|
|
36
|
+
*/
|
|
37
|
+
export interface IInferenceClaudeCacheStated {
|
|
38
|
+
reads: boolean;
|
|
39
|
+
writes: boolean;
|
|
40
|
+
}
|
|
41
|
+
|
|
32
42
|
/** What the relay hands each chunk of an answer it passed on, and the end of the answer. */
|
|
33
43
|
export interface IInferenceUsageTap {
|
|
34
44
|
chunk(bytesArg: Uint8Array): void;
|
|
@@ -43,7 +53,9 @@ export interface IInferenceUsageTap {
|
|
|
43
53
|
* A stream (`sse`) states usage in `message_start` (`message.usage`, `message.model`) and in `message_delta` (`usage`,
|
|
44
54
|
* cumulative), so only those events' data lines are parsed and every other line is skipped unread; a cut stream reports
|
|
45
55
|
* what arrived. A JSON answer states `usage` and `model` at its top level and is read at its end. Anything unreadable
|
|
46
|
-
* reports nothing: the estimate is the authority's, and the answer is never at stake.
|
|
56
|
+
* reports nothing: the estimate is the authority's, and the answer is never at stake. The report says which cache
|
|
57
|
+
* counts the answer stated (`IInferenceClaudeCacheStated`): one it did not state is unknown, not none. The usage
|
|
58
|
+
* reported for the estimate counts an unknown one as none, as the account service's token usage has no unknown.
|
|
47
59
|
*
|
|
48
60
|
* `answered`, given for an answer an API key served, takes at the end what the answer stated for its price: the model
|
|
49
61
|
* that answered and its usage object as sent, a stream's merged field by field from `message_start` and the
|
|
@@ -66,7 +78,8 @@ export class InferenceClaudeUsageTap implements IInferenceUsageTap {
|
|
|
66
78
|
private stopped = false;
|
|
67
79
|
private ended = false;
|
|
68
80
|
|
|
69
|
-
constructor(private readonly format: 'sse' | 'json',
|
|
81
|
+
constructor(private readonly format: 'sse' | 'json',
|
|
82
|
+
private readonly report: (usageArg: TTokenUsage, cacheStatedArg: IInferenceClaudeCacheStated) => void,
|
|
70
83
|
private readonly answered?: (answeredArg: IInferenceAnsweredUsage) => void) {}
|
|
71
84
|
|
|
72
85
|
/** Reads a chunk the relay passed on; never throws. */
|
|
@@ -102,7 +115,11 @@ export class InferenceClaudeUsageTap implements IInferenceUsageTap {
|
|
|
102
115
|
}
|
|
103
116
|
this.stopped = true;
|
|
104
117
|
const usage = this.usage();
|
|
105
|
-
if (usage)
|
|
118
|
+
if (usage) {
|
|
119
|
+
this.report(usage, { reads: this.stated.cache_read_input_tokens !== undefined,
|
|
120
|
+
writes: this.stated.cache_creation_input_tokens !== undefined || this.stated.ephemeral_5m_input_tokens !== undefined
|
|
121
|
+
|| this.stated.ephemeral_1h_input_tokens !== undefined });
|
|
122
|
+
}
|
|
106
123
|
} catch { this.stopped = true; } finally {
|
|
107
124
|
try {
|
|
108
125
|
this.answered?.({ ...(this.model === undefined ? {} : { model: this.model }), ...(this.raw ? { usage: this.raw } : {}),
|
|
@@ -207,7 +224,8 @@ const searchCallTypes: ReadonlyMap<string, 'web' | 'file'> = new Map([['web_sear
|
|
|
207
224
|
* `response.output_item.done`; a data-only frame is kept up to the frame limit and parsed only when it names a terminal
|
|
208
225
|
* type, or a finished output item that is a search call. A JSON answer states `usage` at its top level. `input_tokens`
|
|
209
226
|
* counts every input token, so the cached reads (`cached_tokens`) and cache writes (`cache_write_tokens`) of
|
|
210
|
-
* `input_tokens_details` are taken out of it.
|
|
227
|
+
* `input_tokens_details` are taken out of it. An answer that does not state `cached_tokens` or `cache_write_tokens`
|
|
228
|
+
* reports those as unknown (`null`), never as none, and its input count then holds whatever it read or wrote.
|
|
211
229
|
*
|
|
212
230
|
* `answered`, given for an answer an OpenAI API key served, takes at the end what the answer stated for its price
|
|
213
231
|
* (`IInferenceResponsesAnswered`): whether its terminal event or its JSON body was read, arrived too large to read, or
|
|
@@ -380,10 +398,11 @@ export class InferenceResponsesUsageTap implements IInferenceUsageTap {
|
|
|
380
398
|
const output = count(usage?.output_tokens);
|
|
381
399
|
if (input === undefined && output === undefined) return undefined;
|
|
382
400
|
const details = record(usage?.input_tokens_details);
|
|
383
|
-
const cacheRead = count(details?.cached_tokens) ??
|
|
384
|
-
const cacheWrite = count(details?.cache_write_tokens) ??
|
|
385
|
-
const counts: IInferenceTokenCounts = { inputTokens: Math.max(0, (input ?? 0) - cacheRead - cacheWrite),
|
|
401
|
+
const cacheRead = count(details?.cached_tokens) ?? null;
|
|
402
|
+
const cacheWrite = count(details?.cache_write_tokens) ?? null;
|
|
403
|
+
const counts: IInferenceTokenCounts = { inputTokens: Math.max(0, (input ?? 0) - (cacheRead ?? 0) - (cacheWrite ?? 0)),
|
|
386
404
|
outputTokens: output ?? 0, cacheReadTokens: cacheRead, cacheWriteTokens: cacheWrite };
|
|
387
|
-
return counts.inputTokens + counts.outputTokens + counts.cacheReadTokens + counts.cacheWriteTokens
|
|
405
|
+
return counts.inputTokens + counts.outputTokens + (counts.cacheReadTokens ?? 0) + (counts.cacheWriteTokens ?? 0) > 0
|
|
406
|
+
? counts : undefined;
|
|
388
407
|
}
|
|
389
408
|
}
|