@ultimat3/ai 23.0.0 → 25.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +96 -32
- package/README.md +249 -75
- package/package.json +12 -11
- package/src/agent-facts.ts +9 -2
- package/src/agent-transcript.ts +1 -1
- package/src/agent.ts +43 -25
- package/src/budget.ts +178 -51
- package/src/content-blocks.ts +227 -0
- package/src/content-errors.ts +67 -0
- package/src/echo-provider.ts +102 -0
- package/src/embeddings.ts +40 -21
- package/src/error-body.ts +38 -3
- package/src/errors.ts +19 -11
- package/src/eval-baseline.ts +5 -1
- package/src/eval-errors.ts +3 -1
- package/src/evals.ts +1 -1
- package/src/failure-disclosure.ts +90 -0
- package/src/gateway.ts +73 -37
- package/src/hive-pool.ts +24 -16
- package/src/hive-result.ts +4 -1
- package/src/hive.ts +15 -5
- package/src/index.ts +66 -25
- package/src/llm-cache.ts +4 -3
- package/src/llm.ts +27 -65
- package/src/model-resolve.ts +79 -0
- package/src/models.ts +125 -111
- package/src/openai-body.ts +13 -2
- package/src/openai-messages.ts +86 -15
- package/src/openai-provider.ts +28 -31
- package/src/openai-wire.ts +11 -3
- package/src/pg-vector-sql.ts +11 -7
- package/src/pg-vector.ts +21 -10
- package/src/prompt-fence.ts +128 -0
- package/src/prompt.ts +43 -13
- package/src/provider.ts +82 -117
- package/src/rag.ts +17 -9
- package/src/remote-embedder.ts +15 -41
- package/src/respond.ts +86 -0
- package/src/runtime.ts +12 -14
- package/src/scorers.ts +14 -3
- package/src/tool-idempotency.ts +79 -0
- package/src/tools.ts +52 -39
- package/src/vector-scope.ts +46 -4
- package/src/vector.ts +117 -41
- package/src/wire.ts +7 -2
- package/src/fix-line.evals.ts +0 -35
- package/src/fix-line.ts +0 -27
- package/src/openai-models.ts +0 -84
package/src/agent-facts.ts
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
import type { AnyAction } from '@ultimat3/action';
|
|
11
11
|
import type { Money } from '@ultimat3/money';
|
|
12
|
+
import type { ModelSource } from './model-resolve';
|
|
12
13
|
|
|
13
14
|
/** The declared ceilings, flattened so a manifest row is plain JSON. `null` is "not declared". */
|
|
14
15
|
export interface AgentBudgetFact {
|
|
@@ -18,7 +19,7 @@ export interface AgentBudgetFact {
|
|
|
18
19
|
}
|
|
19
20
|
|
|
20
21
|
export interface AgentFact {
|
|
21
|
-
/** The export name registration stamped — the same name
|
|
22
|
+
/** The export name registration stamped — the same name `toolFrom` and `tools/call` answer to. */
|
|
22
23
|
readonly name: string;
|
|
23
24
|
readonly prompt: string;
|
|
24
25
|
readonly promptId: string;
|
|
@@ -27,7 +28,13 @@ export interface AgentFact {
|
|
|
27
28
|
* which agent ran and not which agent it was — the same reason every eval result carries it.
|
|
28
29
|
*/
|
|
29
30
|
readonly promptHash: string;
|
|
30
|
-
|
|
31
|
+
/**
|
|
32
|
+
* The model a call runs on: the declaration's, its prompt's, then the gateway's `defaultModel`.
|
|
33
|
+
* `null` when none names one yet — a call in that state is refused (`X_AI_MODEL_UNRESOLVED`).
|
|
34
|
+
*/
|
|
35
|
+
readonly model: string | null;
|
|
36
|
+
/** Which of those answered; `gateway` and `null` are as of the gateway installed at read time. */
|
|
37
|
+
readonly modelFrom: ModelSource | null;
|
|
31
38
|
readonly maxTurns: number;
|
|
32
39
|
readonly maxToolResultChars: number;
|
|
33
40
|
/** Tool names, sorted — the catalogue this agent may call, which is its blast radius. */
|
package/src/agent-transcript.ts
CHANGED
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
* next message — and a rule split across two call sites is a rule one of them will miss.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
import { RESPOND } from './llm';
|
|
9
8
|
import type { AiContentBlock, AiMessage, GenerateResult } from './provider';
|
|
9
|
+
import { RESPOND } from './respond';
|
|
10
10
|
import type { LlmToolCall, LlmToolResult } from './tools';
|
|
11
11
|
|
|
12
12
|
/**
|
package/src/agent.ts
CHANGED
|
@@ -6,8 +6,8 @@
|
|
|
6
6
|
* ninth kind, and the list is what a reader counts instead of a sentence in one of them that
|
|
7
7
|
* cannot see the rest. A tool-using run is
|
|
8
8
|
* still one server-authoritative operation with an input schema, an output schema and a policy —
|
|
9
|
-
* so this returns an `action`, and inherits `.
|
|
10
|
-
*
|
|
9
|
+
* so this returns an `action`, and inherits `.openapi()`, `.client()`, `.job()`, `.contract()`,
|
|
10
|
+
* its MCP tool (`@ultimat3/mcp`'s `toolFrom`) and its manifest row without a line here.
|
|
11
11
|
*
|
|
12
12
|
* It exists because the alternative is a hand-rolled loop outside the framework, and a hand-rolled
|
|
13
13
|
* loop is where the dangerous mistake lives: taking the ACTOR from the model's output. Here the
|
|
@@ -31,7 +31,7 @@ import type { AgentFact } from './agent-facts';
|
|
|
31
31
|
import { registerAgentFact } from './agent-facts';
|
|
32
32
|
import { assistantTurn, repairTurn, toolResultTurn } from './agent-transcript';
|
|
33
33
|
import type { BudgetLimits } from './budget';
|
|
34
|
-
import {
|
|
34
|
+
import { budgetKeysFor, currentBudget, screenedMoney, withBudget } from './budget';
|
|
35
35
|
import {
|
|
36
36
|
AgentMaxTurnsError,
|
|
37
37
|
AgentToolUnexposedError,
|
|
@@ -40,9 +40,10 @@ import {
|
|
|
40
40
|
LlmTruncatedError,
|
|
41
41
|
} from './errors';
|
|
42
42
|
import type { LlmBudget } from './llm';
|
|
43
|
-
import { answerAttributes
|
|
43
|
+
import { answerAttributes } from './llm';
|
|
44
|
+
import { describedModel, type ModelSource, modelForCall } from './model-resolve';
|
|
44
45
|
import type { ModelId } from './models';
|
|
45
|
-
import {
|
|
46
|
+
import { moreCapableThan } from './models';
|
|
46
47
|
import type { Prompt, PromptVars } from './prompt';
|
|
47
48
|
import type {
|
|
48
49
|
AiMessage,
|
|
@@ -53,7 +54,9 @@ import type {
|
|
|
53
54
|
} from './provider';
|
|
54
55
|
import { isTruncated } from './provider';
|
|
55
56
|
import { assertNoSecrets } from './redaction';
|
|
56
|
-
import {
|
|
57
|
+
import type { Respond } from './respond';
|
|
58
|
+
import { RESPOND, respondFor } from './respond';
|
|
59
|
+
import { aiGateway, aiRedactor, installedGateway } from './runtime';
|
|
57
60
|
import type { AgentTool, LlmTool, LlmToolResult, ProjectableAction } from './tools';
|
|
58
61
|
import { asProjectableAction, runLlmToolCall, toLlmTools, toolLabel } from './tools';
|
|
59
62
|
|
|
@@ -153,7 +156,7 @@ export function agent<
|
|
|
153
156
|
TOutput extends StandardSchemaV1,
|
|
154
157
|
V extends PromptVars,
|
|
155
158
|
>(def: AgentDef<TInput, TOutput, V>): Action<TInput, TOutput> {
|
|
156
|
-
const respond =
|
|
159
|
+
const respond = respondFor(def.output);
|
|
157
160
|
// At declaration, because the tools are values by then and a run that discovers this at the
|
|
158
161
|
// first request has already been declared, registered and projected as if it worked. Asked of
|
|
159
162
|
// the DECLARATION rather than of the projection: `isMcpExposed` needs no name, and a real
|
|
@@ -201,6 +204,16 @@ export function agent<
|
|
|
201
204
|
return built;
|
|
202
205
|
}
|
|
203
206
|
|
|
207
|
+
// `null`, not `undefined`, for a row naming no model yet: the row is a published fact, and an
|
|
208
|
+
// absent key reads as a reader that forgot to ask.
|
|
209
|
+
const modelFact = (described: {
|
|
210
|
+
readonly model: string | undefined;
|
|
211
|
+
readonly from: ModelSource | undefined;
|
|
212
|
+
}): Pick<AgentFact, 'model' | 'modelFrom'> => ({
|
|
213
|
+
model: described.model ?? null,
|
|
214
|
+
modelFrom: described.from ?? null,
|
|
215
|
+
});
|
|
216
|
+
|
|
204
217
|
function factsOf<
|
|
205
218
|
TInput extends StandardSchemaV1,
|
|
206
219
|
TOutput extends StandardSchemaV1,
|
|
@@ -211,7 +224,10 @@ function factsOf<
|
|
|
211
224
|
prompt: def.prompt.ref,
|
|
212
225
|
promptId: def.prompt.id,
|
|
213
226
|
promptHash: def.prompt.hash,
|
|
214
|
-
|
|
227
|
+
// The run's own precedence, so the row names the model a call would use NOW: read when the
|
|
228
|
+
// facts are, at describe time, like every name in the row. `modelFrom` says which place
|
|
229
|
+
// answered — `gateway` (or `null`, none yet) depends on the gateway installed at that moment.
|
|
230
|
+
...modelFact(describedModel(def.model, def.prompt.model, installedGateway()?.defaultModel)),
|
|
215
231
|
maxTurns: def.maxTurns ?? DEFAULT_MAX_TURNS,
|
|
216
232
|
maxToolResultChars: def.maxToolResultChars ?? DEFAULT_TOOL_RESULT_CHARS,
|
|
217
233
|
tools: [...def.tools.map(toolLabel)].sort(),
|
|
@@ -241,13 +257,16 @@ async function run<
|
|
|
241
257
|
V extends PromptVars,
|
|
242
258
|
>(
|
|
243
259
|
def: AgentDef<TInput, TOutput, V>,
|
|
244
|
-
respond:
|
|
260
|
+
respond: Respond,
|
|
245
261
|
adapted: Adapted,
|
|
246
262
|
args: { readonly input: InferOutput<TInput>; readonly ctx: Ctx },
|
|
247
263
|
): Promise<InferOutput<TOutput>> {
|
|
248
264
|
const { prompt } = def;
|
|
249
265
|
const name = prompt.ref;
|
|
250
|
-
|
|
266
|
+
// Screened first: a bad declaration is refused whether or not a gateway exists.
|
|
267
|
+
const limits = limitsOf(def);
|
|
268
|
+
// The declaration, its prompt, then the gateway's `defaultModel` — the app's three places.
|
|
269
|
+
const model = modelForCall('agent', name, def.model, prompt.model);
|
|
251
270
|
const vars = await def.vars({ input: args.input, ctx: args.ctx });
|
|
252
271
|
assertNoSecrets(name, vars);
|
|
253
272
|
const redact = aiRedactor();
|
|
@@ -267,15 +286,12 @@ async function run<
|
|
|
267
286
|
'llm.redacted': rendered !== rawPrompt || system !== prompt.system,
|
|
268
287
|
});
|
|
269
288
|
|
|
270
|
-
// Screened first, as before: a bad declaration is refused whether or not a gateway exists.
|
|
271
|
-
const limits = limitsOf(def);
|
|
272
289
|
// Rooted in the GATEWAY's own ceilings when no scope is open — an empty root ignored them.
|
|
273
290
|
const gateway = aiGateway(name);
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
).derive(limits);
|
|
291
|
+
// Keyed on the CALLER, so the gateway's `actor` / `org` ceilings count this call against them.
|
|
292
|
+
const ledger = (currentBudget() ?? gateway.callLedger(budgetKeysFor(args.ctx.actor))).derive(
|
|
293
|
+
limits,
|
|
294
|
+
);
|
|
279
295
|
const base: GenerateRequest = {
|
|
280
296
|
model,
|
|
281
297
|
...(system === undefined ? {} : { system }),
|
|
@@ -283,7 +299,7 @@ async function run<
|
|
|
283
299
|
maxTokens: def.maxTokens ?? DEFAULT_MAX_TOKENS,
|
|
284
300
|
...(prompt.effort === undefined ? {} : { effort: prompt.effort }),
|
|
285
301
|
...(prompt.thinking === undefined ? {} : { thinking: prompt.thinking }),
|
|
286
|
-
tools: [...adapted.offered, respond],
|
|
302
|
+
tools: [...adapted.offered, respond.tool],
|
|
287
303
|
// The caller's own signal, forwarded to the transport. A disconnect has to reach the socket,
|
|
288
304
|
// not just the top of the next turn: a provider call already in flight is the expensive one.
|
|
289
305
|
signal: args.ctx.signal,
|
|
@@ -322,14 +338,14 @@ async function run<
|
|
|
322
338
|
usage: result.usage,
|
|
323
339
|
cost: result.cost,
|
|
324
340
|
});
|
|
325
|
-
assertAnswerable(result, name);
|
|
341
|
+
assertAnswerable(result, name, base);
|
|
326
342
|
|
|
327
343
|
if (requested.length > 0) {
|
|
328
344
|
// Concurrent, and deliberately unbounded WITHIN one turn: the batch is what a single
|
|
329
345
|
// model turn asked for, each entry is an ordinary `action` carrying its own policy and
|
|
330
346
|
// its own `rateLimit`, and a second ceiling here would be a throttle competing with
|
|
331
|
-
// those. A tool that calls a model still
|
|
332
|
-
// budget holds. Serial cost 5x wall clock for a turn that asked for 5 tools, and nothing
|
|
347
|
+
// those. A tool that calls a model still reserves on the same ledger chain, which
|
|
348
|
+
// debits before it checks, so the budget holds. Serial cost 5x wall clock for a turn that asked for 5 tools, and nothing
|
|
333
349
|
// in the types or the docs ever said so.
|
|
334
350
|
//
|
|
335
351
|
// Order is by INDEX, not by completion: `Promise.all` resolves positionally and each
|
|
@@ -353,7 +369,7 @@ async function run<
|
|
|
353
369
|
continue;
|
|
354
370
|
}
|
|
355
371
|
|
|
356
|
-
const parsed = await validateAsync(def.output,
|
|
372
|
+
const parsed = await validateAsync(def.output, respond.read(result));
|
|
357
373
|
if (parsed.issues === undefined) return parsed.value;
|
|
358
374
|
// A wrong shape gets another turn like any other, because unlike `llm()` this loop has
|
|
359
375
|
// turns left by construction — and unlike a tool result, the correction is the message.
|
|
@@ -397,12 +413,12 @@ async function observeTurn<
|
|
|
397
413
|
}
|
|
398
414
|
|
|
399
415
|
/** Refuse before the answer is read, for the reason `llm()` does: a refusal is a 200 with no answer. */
|
|
400
|
-
function assertAnswerable(result: GenerateResult, name: string): void {
|
|
416
|
+
function assertAnswerable(result: GenerateResult, name: string, request: GenerateRequest): void {
|
|
401
417
|
if (result.stopReason !== 'refusal') return;
|
|
402
418
|
throw new LlmRefusedError({
|
|
403
419
|
prompt: name,
|
|
404
420
|
model: result.model,
|
|
405
|
-
alternative: moreCapableThan(result.model),
|
|
421
|
+
alternative: moreCapableThan(result.model, request),
|
|
406
422
|
category: result.stopDetails?.category,
|
|
407
423
|
explanation: result.stopDetails?.explanation,
|
|
408
424
|
});
|
|
@@ -420,7 +436,9 @@ function limitsOf<
|
|
|
420
436
|
...(budget?.tokensIn === undefined
|
|
421
437
|
? {}
|
|
422
438
|
: { tokensIn: finiteCount(SUBJECT, 'budget.tokensIn', budget.tokensIn) }),
|
|
423
|
-
...(budget?.costPerCall === undefined
|
|
439
|
+
...(budget?.costPerCall === undefined
|
|
440
|
+
? {}
|
|
441
|
+
: { costPerCall: screenedMoney(SUBJECT, 'budget.costPerCall', budget.costPerCall) }),
|
|
424
442
|
// The ledger's `request` scope accumulates across every call made under one ledger, which for
|
|
425
443
|
// a run under `withBudget` is exactly "the whole run".
|
|
426
444
|
...(budget?.tokensPerRun === undefined
|
package/src/budget.ts
CHANGED
|
@@ -10,7 +10,8 @@
|
|
|
10
10
|
// module-scope `new` threw at EVALUATION in a browser bundle, where the bundler stubs
|
|
11
11
|
// `node:async_hooks` to `{}`, and took every importer of `@ultimat3/ai` with it.
|
|
12
12
|
|
|
13
|
-
import {
|
|
13
|
+
import type { Actor } from '@ultimat3/core';
|
|
14
|
+
import { assert, asyncContext, finiteCount } from '@ultimat3/core';
|
|
14
15
|
import type { Money } from '@ultimat3/money';
|
|
15
16
|
import { assertSameCurrency } from '@ultimat3/money';
|
|
16
17
|
import { AiBudgetExceededError } from './errors';
|
|
@@ -64,14 +65,59 @@ export function estimateSpend(request: GenerateRequest): SpendEstimate {
|
|
|
64
65
|
};
|
|
65
66
|
}
|
|
66
67
|
|
|
68
|
+
/**
|
|
69
|
+
* What one `take` found. `spent` is the counter as the store read it, BEFORE this take — so a
|
|
70
|
+
* refusal can say how much was left.
|
|
71
|
+
*/
|
|
72
|
+
export interface BudgetTake {
|
|
73
|
+
readonly taken: boolean;
|
|
74
|
+
readonly spent: number;
|
|
75
|
+
}
|
|
76
|
+
|
|
67
77
|
/** Where cross-request counters live. Swap for Redis in a multi-process deployment. */
|
|
68
78
|
export interface BudgetStore {
|
|
69
79
|
spent(key: string): Promise<number> | number;
|
|
70
80
|
/** `tokens` may be NEGATIVE: releasing a reservation the call never spent is a credit. */
|
|
71
81
|
add(key: string, tokens: number): Promise<void> | void;
|
|
82
|
+
/**
|
|
83
|
+
* Add `tokens` to `key` only if the total stays at or under `limit`, in ONE atomic step — a
|
|
84
|
+
* Redis `EVAL`, a SQL `update … set spent = spent + $2 where key = $1 and spent + $2 <= $3`.
|
|
85
|
+
* Required, with no default built on `spent` + `add`: that pair is the read-then-write two
|
|
86
|
+
* concurrent requests both pass, which is the overspend this member exists to close.
|
|
87
|
+
*/
|
|
88
|
+
take(key: string, tokens: number, limit: number): Promise<BudgetTake> | BudgetTake;
|
|
72
89
|
reset(key?: string): Promise<void> | void;
|
|
73
90
|
}
|
|
74
91
|
|
|
92
|
+
/**
|
|
93
|
+
* The store keys a call's identity is counted under: the acting identity (kind and id — one window
|
|
94
|
+
* per actor, whichever org it acts in) and its org, when it has one. Derived, never taken from a
|
|
95
|
+
* declaration, so every `llm()`, `agent()` and `hive()` binds the gateway's `actor` / `org` ceilings
|
|
96
|
+
* to whoever called it. An anonymous caller is one identity: every anonymous call shares its window.
|
|
97
|
+
*/
|
|
98
|
+
export interface BudgetKeys {
|
|
99
|
+
readonly actorKey?: string;
|
|
100
|
+
readonly orgKey?: string;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export function budgetKeysFor(actor: Actor): BudgetKeys {
|
|
104
|
+
const orgId: string | null | undefined = actor.orgId;
|
|
105
|
+
return {
|
|
106
|
+
actorKey: `actor:${actor.kind}:${actor.id}`,
|
|
107
|
+
...(orgless(orgId) ? {} : { orgKey: `org:${orgId}` }),
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* "Inside no org", in all three spellings `orgId` arrives in: it is a value off the wire (a decoded
|
|
113
|
+
* session row, an app's adapter), so `null` and `''` reach here despite core's
|
|
114
|
+
* `string | undefined`. Testing `undefined` alone turned the other two into the real keys `org:null`
|
|
115
|
+
* and `org:` — ONE org window shared by every org-less caller in the deployment. The same predicate
|
|
116
|
+
* as `@ultimat3/query`'s `cache.ts`, restated because that one is module-private.
|
|
117
|
+
*/
|
|
118
|
+
const orgless = (orgId: string | null | undefined): boolean =>
|
|
119
|
+
orgId === undefined || orgId === null || orgId === '';
|
|
120
|
+
|
|
75
121
|
/**
|
|
76
122
|
* What `reserve` debited, so `record` can reconcile it against the provider's real counts and
|
|
77
123
|
* `release` can give it back. Held by the caller rather than the ledger because one ledger serves
|
|
@@ -81,6 +127,13 @@ export interface BudgetReservation {
|
|
|
81
127
|
readonly tokens: number;
|
|
82
128
|
}
|
|
83
129
|
|
|
130
|
+
/**
|
|
131
|
+
* The per-process default. There is NO window: a counter lives until `reset()` or a restart, so an
|
|
132
|
+
* `actor` / `org` ceiling here is "per process lifetime". It holds one entry per caller that has
|
|
133
|
+
* spent under a DECLARED ceiling (none at all without one), and deliberately no eviction: dropping
|
|
134
|
+
* a counter hands that caller its whole ceiling back, which is the bypass the ceiling exists to
|
|
135
|
+
* stop. A window — per day, per month — is a shared store whose keys expire.
|
|
136
|
+
*/
|
|
84
137
|
export class MemoryBudgetStore implements BudgetStore {
|
|
85
138
|
private readonly counters = new Map<string, number>();
|
|
86
139
|
|
|
@@ -88,8 +141,27 @@ export class MemoryBudgetStore implements BudgetStore {
|
|
|
88
141
|
return this.counters.get(key) ?? 0;
|
|
89
142
|
}
|
|
90
143
|
|
|
144
|
+
/** A counter back at zero is deleted: absent and zero are the same answer, and an entry costs. */
|
|
91
145
|
add(key: string, tokens: number): void {
|
|
92
|
-
this.
|
|
146
|
+
this.write(key, this.spent(key) + tokens);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** Atomic because it is synchronous: nothing else runs between the read and the write. */
|
|
150
|
+
take(key: string, tokens: number, limit: number): BudgetTake {
|
|
151
|
+
const spent = this.spent(key);
|
|
152
|
+
if (spent + tokens > limit) return { taken: false, spent };
|
|
153
|
+
this.write(key, spent + tokens);
|
|
154
|
+
return { taken: true, spent };
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** How many counters are held — one per caller that has spent under a declared ceiling. */
|
|
158
|
+
size(): number {
|
|
159
|
+
return this.counters.size;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
private write(key: string, total: number): void {
|
|
163
|
+
if (total === 0) this.counters.delete(key);
|
|
164
|
+
else this.counters.set(key, total);
|
|
93
165
|
}
|
|
94
166
|
|
|
95
167
|
reset(key?: string): void {
|
|
@@ -98,6 +170,11 @@ export class MemoryBudgetStore implements BudgetStore {
|
|
|
98
170
|
}
|
|
99
171
|
}
|
|
100
172
|
|
|
173
|
+
/** The one way to build the per-process store; the class is a type in the barrel only. */
|
|
174
|
+
export function memoryBudgetStore(): MemoryBudgetStore {
|
|
175
|
+
return new MemoryBudgetStore();
|
|
176
|
+
}
|
|
177
|
+
|
|
101
178
|
export interface BudgetLedgerInput {
|
|
102
179
|
readonly limits: BudgetLimits;
|
|
103
180
|
/** Stable identity keys. Omit a key to skip that scope even when a limit is set. */
|
|
@@ -133,20 +210,13 @@ export class BudgetLedger {
|
|
|
133
210
|
* inside it, and its `request` ceiling was re-granted in full to every one of them.
|
|
134
211
|
*/
|
|
135
212
|
private parent: BudgetLedger | undefined;
|
|
136
|
-
/**
|
|
137
|
-
* Reservations take turns. Check-then-debit spans an `await store.spent()`, and three callers
|
|
138
|
-
* interleaving inside it is the bypass this ledger exists to close — one event loop, so a
|
|
139
|
-
* promise chain IS the lock. A store shared across PROCESSES needs an atomic increment of its
|
|
140
|
-
* own; this closes the parallelism inside one.
|
|
141
|
-
*/
|
|
142
|
-
private turnstile: Promise<unknown> = Promise.resolve();
|
|
143
213
|
|
|
144
214
|
constructor(input: BudgetLedgerInput) {
|
|
145
215
|
// The ceilings are screened where they LAND, because `assertScope` compares with `>`: a `NaN`
|
|
146
216
|
// limit makes `limit - spent` a `NaN`, `want > NaN` false, and the scope silently unlimited —
|
|
147
217
|
// the ceiling does not become wrong, it stops existing. `llm()`, `agent()` and `hive()` screen
|
|
148
218
|
// the same numbers first under the key names their declarations use (`tokensPerRun` is this
|
|
149
|
-
// `request`), so this is the backstop for a `
|
|
219
|
+
// `request`), so this is the backstop for a `providerGateway({ budget })` or a hand-built ledger,
|
|
150
220
|
// where these ARE the names the caller wrote.
|
|
151
221
|
this.limits = assertFiniteLimits(input.limits);
|
|
152
222
|
this.actorKey = input.actorKey;
|
|
@@ -156,35 +226,19 @@ export class BudgetLedger {
|
|
|
156
226
|
}
|
|
157
227
|
|
|
158
228
|
/**
|
|
159
|
-
* Check an estimate against every applicable scope BEFORE the call,
|
|
160
|
-
* the first scope that cannot cover it, naming that scope, so the fix line points
|
|
161
|
-
* rather than four.
|
|
229
|
+
* Check an estimate against every applicable scope BEFORE the call, and DEBIT it in the same
|
|
230
|
+
* step. Throws on the first scope that cannot cover it, naming that scope, so the fix line points
|
|
231
|
+
* at one knob rather than four.
|
|
162
232
|
*
|
|
163
|
-
*
|
|
164
|
-
*
|
|
165
|
-
*
|
|
166
|
-
*
|
|
167
|
-
*
|
|
233
|
+
* Debit-then-check, never check-then-debit: the in-memory scopes are checked and debited with no
|
|
234
|
+
* `await` between (one event loop, so that IS atomic), and the store's scopes go through its
|
|
235
|
+
* atomic `take`. Any read-then-write across an `await` lets concurrent calls — `Promise.all` of
|
|
236
|
+
* derived ledgers, or one request per scope racing one org key — all read the same `spent`, all
|
|
237
|
+
* pass, and all debit a ceiling only one fitted. A refusal at a later scope gives back what the
|
|
238
|
+
* earlier ones took. `record` replaces the estimate with the real counts; `release` gives it back
|
|
239
|
+
* when the call never happened.
|
|
168
240
|
*/
|
|
169
241
|
async reserve(estimate: SpendEstimate): Promise<BudgetReservation> {
|
|
170
|
-
// The ROOT's turnstile, not this ledger's: reservations under one scope take turns even when
|
|
171
|
-
// each call derived its own ledger, which is every `llm()` call. A per-ledger queue serialised
|
|
172
|
-
// nothing once `derive` existed — `Promise.all` of three derived ledgers all read the chain
|
|
173
|
-
// before any of them debited it.
|
|
174
|
-
const gate = this.rootLedger();
|
|
175
|
-
const turn = gate.turnstile.then(() => this.reserveNow(estimate));
|
|
176
|
-
// Chained on a settled shadow: one refusal must not reject every reservation queued behind it.
|
|
177
|
-
gate.turnstile = turn.catch(() => undefined);
|
|
178
|
-
return await turn;
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
private rootLedger(): BudgetLedger {
|
|
182
|
-
let ledger: BudgetLedger = this;
|
|
183
|
-
while (ledger.parent !== undefined) ledger = ledger.parent;
|
|
184
|
-
return ledger;
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
private async reserveNow(estimate: SpendEstimate): Promise<BudgetReservation> {
|
|
188
242
|
// Every ledger in the chain, because each keeps its own counter and the tightest limit is not
|
|
189
243
|
// always the one with the most spent against it.
|
|
190
244
|
for (let l: BudgetLedger | undefined = this; l !== undefined; l = l.parent) {
|
|
@@ -192,19 +246,53 @@ export class BudgetLedger {
|
|
|
192
246
|
}
|
|
193
247
|
// Per call, so nothing is "already spent" against it.
|
|
194
248
|
this.assertScope('tokensIn', this.limits.tokensIn, 0, estimate.inputTokens);
|
|
195
|
-
if (this.limits.actor !== undefined && this.actorKey !== undefined) {
|
|
196
|
-
const spent = await this.store.spent(this.actorKey);
|
|
197
|
-
this.assertScope(`actor:${this.actorKey}`, this.limits.actor, spent, estimate.tokens);
|
|
198
|
-
}
|
|
199
|
-
if (this.limits.org !== undefined && this.orgKey !== undefined) {
|
|
200
|
-
const spent = await this.store.spent(this.orgKey);
|
|
201
|
-
this.assertScope(`org:${this.orgKey}`, this.limits.org, spent, estimate.tokens);
|
|
202
|
-
}
|
|
203
249
|
this.assertCost(estimate.cost);
|
|
204
|
-
|
|
250
|
+
this.debitChain(estimate.tokens);
|
|
251
|
+
const taken: string[] = [];
|
|
252
|
+
try {
|
|
253
|
+
await this.takeScope('actor', this.limits.actor, this.actorKey, estimate.tokens, taken);
|
|
254
|
+
await this.takeScope('org', this.limits.org, this.orgKey, estimate.tokens, taken);
|
|
255
|
+
} catch (error) {
|
|
256
|
+
this.debitChain(-estimate.tokens);
|
|
257
|
+
for (const key of taken) await this.store.add(key, -estimate.tokens);
|
|
258
|
+
throw error;
|
|
259
|
+
}
|
|
205
260
|
return { tokens: estimate.tokens };
|
|
206
261
|
}
|
|
207
262
|
|
|
263
|
+
/**
|
|
264
|
+
* One store scope, and only when its ceiling is DECLARED: every `llm()` / `agent()` / `hive()`
|
|
265
|
+
* carries its caller's keys, so writing per key grew `MemoryBudgetStore` by one entry per caller
|
|
266
|
+
* forever — and cost a shared store two writes a call — for counters no ceiling reads. Records
|
|
267
|
+
* the key it took, so a refusal further on can give it back.
|
|
268
|
+
*/
|
|
269
|
+
private async takeScope(
|
|
270
|
+
scope: 'actor' | 'org',
|
|
271
|
+
limit: number | undefined,
|
|
272
|
+
key: string | undefined,
|
|
273
|
+
tokens: number,
|
|
274
|
+
taken: string[],
|
|
275
|
+
): Promise<void> {
|
|
276
|
+
if (key === undefined || limit === undefined) return;
|
|
277
|
+
const outcome: unknown = await this.store.take(key, tokens, limit);
|
|
278
|
+
// The store is the app's: a shape it did not promise must be a coded refusal, never a
|
|
279
|
+
// `TypeError` from reading `.taken` off `undefined` inside the reservation.
|
|
280
|
+
assert(
|
|
281
|
+
isBudgetTake(outcome),
|
|
282
|
+
`BudgetStore.take("${key}") answered ${typeof outcome}, not { taken: boolean, spent: number }`,
|
|
283
|
+
'return { taken, spent } from take(key, tokens, limit), spent being the counter before this take',
|
|
284
|
+
);
|
|
285
|
+
if (!outcome.taken) {
|
|
286
|
+
throw new AiBudgetExceededError({
|
|
287
|
+
scope: `${scope}:${key}`,
|
|
288
|
+
requested: tokens,
|
|
289
|
+
remaining: limit - outcome.spent,
|
|
290
|
+
limit,
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
taken.push(key);
|
|
294
|
+
}
|
|
295
|
+
|
|
208
296
|
/** Give a reservation back: a provider that threw, a stream abandoned before `done`. */
|
|
209
297
|
async release(reservation: BudgetReservation | undefined): Promise<void> {
|
|
210
298
|
if (reservation === undefined) return;
|
|
@@ -217,6 +305,9 @@ export class BudgetLedger {
|
|
|
217
305
|
* an `llm()` action must not be able to widen the actor or org ceiling it runs inside.
|
|
218
306
|
*/
|
|
219
307
|
derive(limits: BudgetLimits): BudgetLedger {
|
|
308
|
+
// Screened BEFORE tightening, not only by the child's constructor: a `NaN` that won
|
|
309
|
+
// `tighterMoney` would otherwise reach the child as the parent's ceiling, widened.
|
|
310
|
+
assertFiniteLimits(limits);
|
|
220
311
|
const child = new BudgetLedger({
|
|
221
312
|
limits: {
|
|
222
313
|
...pick('request', tighterNumber(this.limits.request, limits.request)),
|
|
@@ -258,11 +349,25 @@ export class BudgetLedger {
|
|
|
258
349
|
*/
|
|
259
350
|
private async debit(tokens: number): Promise<void> {
|
|
260
351
|
if (tokens === 0) return;
|
|
352
|
+
this.debitChain(tokens);
|
|
353
|
+
// The keys `reserve` took — a scope with a declared ceiling — and no others, so a reconcile
|
|
354
|
+
// never creates the counter `takeScope` declined to.
|
|
355
|
+
for (const key of this.meteredKeys()) await this.store.add(key, tokens);
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/** The store keys this ledger meters: a key whose scope declares a ceiling. */
|
|
359
|
+
private meteredKeys(): readonly string[] {
|
|
360
|
+
const keys: string[] = [];
|
|
361
|
+
if (this.actorKey !== undefined && this.limits.actor !== undefined) keys.push(this.actorKey);
|
|
362
|
+
if (this.orgKey !== undefined && this.limits.org !== undefined) keys.push(this.orgKey);
|
|
363
|
+
return keys;
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/** The in-memory half: this ledger's `request` counter and every ancestor's. */
|
|
367
|
+
private debitChain(tokens: number): void {
|
|
261
368
|
for (let l: BudgetLedger | undefined = this; l !== undefined; l = l.parent) {
|
|
262
369
|
l.requestTokens += tokens;
|
|
263
370
|
}
|
|
264
|
-
if (this.actorKey !== undefined) await this.store.add(this.actorKey, tokens);
|
|
265
|
-
if (this.orgKey !== undefined) await this.store.add(this.orgKey, tokens);
|
|
266
371
|
}
|
|
267
372
|
|
|
268
373
|
async report(): Promise<BudgetReport> {
|
|
@@ -301,18 +406,40 @@ export class BudgetLedger {
|
|
|
301
406
|
}
|
|
302
407
|
|
|
303
408
|
/**
|
|
304
|
-
* Every declared
|
|
409
|
+
* Every declared ceiling, proven to be a whole count. A limit is optional and an absent one is
|
|
305
410
|
* "unlimited" by design — which is exactly why a `NaN` one is the dangerous value: it reads as a
|
|
306
|
-
* declared ceiling everywhere (`report()`, a manifest row, a log line) and enforces nothing.
|
|
411
|
+
* declared ceiling everywhere (`report()`, a manifest row, a log line) and enforces nothing. The
|
|
412
|
+
* money one most of all: `tighterMoney` asks `a.minor <= b.minor`, false against a `NaN`, so an
|
|
413
|
+
* unscreened `costPerCall` won every `derive` and WIDENED the stricter ceiling it ran inside.
|
|
414
|
+
* `subject` and `prefix` let a caller name the key its own declaration wrote (`gateway`'s
|
|
415
|
+
* `budget.org`), because a fix line naming a key the app never typed is not a fix.
|
|
307
416
|
*/
|
|
308
|
-
function assertFiniteLimits(
|
|
417
|
+
export function assertFiniteLimits(
|
|
418
|
+
limits: BudgetLimits,
|
|
419
|
+
subject = 'the AI budget',
|
|
420
|
+
prefix = '',
|
|
421
|
+
): BudgetLimits {
|
|
309
422
|
for (const scope of ['request', 'tokensIn', 'actor', 'org'] as const) {
|
|
310
423
|
const limit = limits[scope];
|
|
311
|
-
if (limit !== undefined) finiteCount(
|
|
424
|
+
if (limit !== undefined) finiteCount(subject, `${prefix}${scope}`, limit);
|
|
312
425
|
}
|
|
426
|
+
const money = limits.costPerCall;
|
|
427
|
+
if (money !== undefined) finiteCount(subject, `${prefix}costPerCall.minor`, money.minor);
|
|
313
428
|
return limits;
|
|
314
429
|
}
|
|
315
430
|
|
|
431
|
+
/** One money ceiling screened under `key`, for a declaration whose field names differ from ours. */
|
|
432
|
+
export function screenedMoney(subject: string, key: string, money: Money): Money {
|
|
433
|
+
finiteCount(subject, `${key}.minor`, money.minor);
|
|
434
|
+
return money;
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
function isBudgetTake(value: unknown): value is BudgetTake {
|
|
438
|
+
if (typeof value !== 'object' || value === null) return false;
|
|
439
|
+
const record = value as Record<string, unknown>;
|
|
440
|
+
return typeof record['taken'] === 'boolean' && Number.isFinite(record['spent']);
|
|
441
|
+
}
|
|
442
|
+
|
|
316
443
|
/** Spreadable single-key record, so an absent limit stays absent under exactOptionalPropertyTypes. */
|
|
317
444
|
function pick<K extends string, V>(key: K, value: V | undefined): Partial<Record<K, V>> {
|
|
318
445
|
return value === undefined ? {} : ({ [key]: value } as Record<K, V>);
|