@plurnk/plurnk-providers 1.5.0 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.defaults +41 -34
- package/README.md +15 -0
- package/SPEC.md +242 -89
- package/dist/AiSdkProvider.d.ts +33 -33
- package/dist/AiSdkProvider.d.ts.map +1 -1
- package/dist/AiSdkProvider.js +442 -133
- package/dist/AiSdkProvider.js.map +1 -1
- package/dist/Mock.d.ts +10 -11
- package/dist/Mock.d.ts.map +1 -1
- package/dist/Mock.js +87 -25
- package/dist/Mock.js.map +1 -1
- package/dist/Pool.d.ts +9 -24
- package/dist/Pool.d.ts.map +1 -1
- package/dist/Pool.js +86 -25
- package/dist/Pool.js.map +1 -1
- package/dist/accounting.d.ts +5 -2
- package/dist/accounting.d.ts.map +1 -1
- package/dist/accounting.js +100 -16
- package/dist/accounting.js.map +1 -1
- package/dist/accountingPublic.d.ts +5 -0
- package/dist/accountingPublic.d.ts.map +1 -0
- package/dist/accountingPublic.js +3 -0
- package/dist/accountingPublic.js.map +1 -0
- package/dist/aiSdkTransport.d.ts +9 -2
- package/dist/aiSdkTransport.d.ts.map +1 -1
- package/dist/aiSdkTransport.js +160 -62
- package/dist/aiSdkTransport.js.map +1 -1
- package/dist/capacity.d.ts +26 -0
- package/dist/capacity.d.ts.map +1 -0
- package/dist/capacity.js +90 -0
- package/dist/capacity.js.map +1 -0
- package/dist/catalogProvider.d.ts +8 -3
- package/dist/catalogProvider.d.ts.map +1 -1
- package/dist/catalogProvider.js +45 -41
- package/dist/catalogProvider.js.map +1 -1
- package/dist/compatibleProvider.d.ts.map +1 -1
- package/dist/compatibleProvider.js +26 -12
- package/dist/compatibleProvider.js.map +1 -1
- package/dist/cost.d.ts +10 -10
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +90 -42
- package/dist/cost.js.map +1 -1
- package/dist/env.d.ts +13 -11
- package/dist/env.d.ts.map +1 -1
- package/dist/env.js +83 -46
- package/dist/env.js.map +1 -1
- package/dist/errors.d.ts +17 -3
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +91 -8
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +7 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -3
- package/dist/index.js.map +1 -1
- package/dist/ollama.js +3 -3
- package/dist/ollama.js.map +1 -1
- package/dist/promptTokens.d.ts.map +1 -1
- package/dist/promptTokens.js +7 -4
- package/dist/promptTokens.js.map +1 -1
- package/dist/sdkModels.d.ts +7 -2
- package/dist/sdkModels.d.ts.map +1 -1
- package/dist/sdkModels.js +43 -13
- package/dist/sdkModels.js.map +1 -1
- package/dist/types.d.ts +55 -33
- package/dist/types.d.ts.map +1 -1
- package/dist/usage.d.ts +22 -5
- package/dist/usage.d.ts.map +1 -1
- package/dist/usage.js +169 -83
- package/dist/usage.js.map +1 -1
- package/package.json +18 -7
- package/src/AiSdkProvider.test.ts +964 -206
- package/src/AiSdkProvider.ts +545 -155
- package/src/Mock.test.ts +69 -30
- package/src/Mock.ts +99 -29
- package/src/Pool.test.ts +90 -19
- package/src/Pool.ts +96 -27
- package/src/ProviderRegistry.test.ts +16 -11
- package/src/accounting.test.ts +58 -22
- package/src/accounting.ts +119 -18
- package/src/accountingPublic.ts +9 -0
- package/src/aiSdkTransport.test.ts +42 -49
- package/src/aiSdkTransport.ts +174 -62
- package/src/boundaries.test.ts +2 -0
- package/src/capacity.test.ts +92 -0
- package/src/capacity.ts +140 -0
- package/src/catalogProvider.test.ts +339 -30
- package/src/catalogProvider.ts +65 -47
- package/src/compatibleProvider.test.ts +7 -5
- package/src/compatibleProvider.ts +29 -13
- package/src/cost.test.ts +86 -36
- package/src/cost.ts +111 -50
- package/src/defaults.test.ts +13 -3
- package/src/env.test.ts +103 -25
- package/src/env.ts +153 -65
- package/src/errors.test.ts +80 -2
- package/src/errors.ts +107 -8
- package/src/index.ts +26 -7
- package/src/ollama.test.ts +5 -3
- package/src/ollama.ts +3 -3
- package/src/promptTokens.ts +8 -5
- package/src/sdkModels.test.ts +77 -8
- package/src/sdkModels.ts +51 -15
- package/src/types.ts +112 -51
- package/src/usage.test.ts +112 -116
- package/src/usage.ts +214 -93
package/src/Mock.test.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { strict as assert } from "node:assert";
|
|
|
3
3
|
import Mock from "./Mock.ts";
|
|
4
4
|
import type { Provider } from "./types.ts";
|
|
5
5
|
import type { MockResponse } from "./Mock.ts";
|
|
6
|
+
import { ProviderError } from "./errors.ts";
|
|
6
7
|
|
|
7
8
|
const build = (responses: MockResponse[] = [{ assistant: { content: "hi", reasoning: null } }]) =>
|
|
8
9
|
new Mock({ contextWindow: 100000, responses });
|
|
@@ -12,6 +13,7 @@ const build = (responses: MockResponse[] = [{ assistant: { content: "hi", reason
|
|
|
12
13
|
test("Mock: contextWindow and model are stable across reads", () => {
|
|
13
14
|
const m = build();
|
|
14
15
|
assert.equal(m.contextWindow, 100000);
|
|
16
|
+
assert.equal(m.inputCapacity, null);
|
|
15
17
|
assert.equal(m.contextWindow, 100000);
|
|
16
18
|
assert.equal(m.model, "mock");
|
|
17
19
|
assert.equal(m.model, "mock");
|
|
@@ -34,19 +36,23 @@ test("Mock: prompt counting is exact for its declared mock vocabulary", async ()
|
|
|
34
36
|
});
|
|
35
37
|
});
|
|
36
38
|
|
|
37
|
-
test("Mock: calculateCost returns its deliberate zero estimate", () => {
|
|
38
|
-
const m = build();
|
|
39
|
-
assert.equal(m.calculateCost({ prompt: 100, completion: 20, reasoning: 10, cached: 5, total: 130 }), 0);
|
|
40
|
-
});
|
|
41
|
-
|
|
42
39
|
// — Transport ({§provider-interface}) —
|
|
43
40
|
|
|
44
41
|
test("Mock: generate resolves a valid ProviderResponse shape", async () => {
|
|
45
42
|
const m = build([{ assistant: { content: "hello", reasoning: "cot" } }]);
|
|
46
|
-
const { assistant, assistantRaw } = await m.generate({ messages: [] });
|
|
43
|
+
const { assistant, assistantRaw, accounting } = await m.generate({ messages: [] });
|
|
47
44
|
assert.equal(assistant.content, "hello");
|
|
48
45
|
assert.equal(assistant.reasoning, "cot");
|
|
49
|
-
assert.deepEqual(
|
|
46
|
+
assert.deepEqual(accounting[0]?.usage, {
|
|
47
|
+
inputTokens: 0,
|
|
48
|
+
outputTokens: 0,
|
|
49
|
+
totalTokens: 0,
|
|
50
|
+
});
|
|
51
|
+
assert.deepEqual(accounting[0]?.cost, {
|
|
52
|
+
kind: "estimated",
|
|
53
|
+
amount: { amount: "0", currency: "USD" },
|
|
54
|
+
source: "mock provider fixture",
|
|
55
|
+
});
|
|
50
56
|
assert.equal(assistant.finishReason, "stop");
|
|
51
57
|
assert.equal(assistant.model, "mock");
|
|
52
58
|
assert.equal(assistantRaw, null); // present, defaulted
|
|
@@ -57,16 +63,20 @@ test("Mock: generate applies caller-supplied overrides", async () => {
|
|
|
57
63
|
assistant: {
|
|
58
64
|
content: "x",
|
|
59
65
|
reasoning: null,
|
|
60
|
-
usage: { prompt: 1, completion: 2, reasoning: 0, cached: 0, total: 3 },
|
|
61
66
|
finishReason: "length",
|
|
62
67
|
model: "mock-xl",
|
|
63
68
|
},
|
|
69
|
+
usage: { inputTokens: 1, outputTokens: 2, totalTokens: 3 },
|
|
64
70
|
assistantRaw: { wire: true },
|
|
65
71
|
}]);
|
|
66
|
-
const { assistant, assistantRaw } = await m.generate({ messages: [] });
|
|
72
|
+
const { assistant, assistantRaw, accounting } = await m.generate({ messages: [] });
|
|
67
73
|
assert.equal(assistant.finishReason, "length");
|
|
68
74
|
assert.equal(assistant.model, "mock-xl");
|
|
69
|
-
assert.deepEqual(
|
|
75
|
+
assert.deepEqual(accounting[0]?.usage, {
|
|
76
|
+
inputTokens: 1,
|
|
77
|
+
outputTokens: 2,
|
|
78
|
+
totalTokens: 3,
|
|
79
|
+
});
|
|
70
80
|
assert.deepEqual(assistantRaw, { wire: true });
|
|
71
81
|
});
|
|
72
82
|
|
|
@@ -116,42 +126,71 @@ test("Mock: remaining decrements as responses are consumed", async () => {
|
|
|
116
126
|
assert.equal(m.remaining, 1);
|
|
117
127
|
});
|
|
118
128
|
|
|
119
|
-
test("Mock: exhausted queue throws a
|
|
129
|
+
test("Mock: exhausted queue throws a ProviderError carrying its settled accounting", async () => {
|
|
120
130
|
const m = build([]);
|
|
121
|
-
await assert.rejects(
|
|
131
|
+
await assert.rejects(
|
|
132
|
+
() => m.generate({ messages: [] }),
|
|
133
|
+
(error: unknown) => {
|
|
134
|
+
assert.ok(error instanceof ProviderError);
|
|
135
|
+
assert.match(error.message, /exhausted/);
|
|
136
|
+
assert.deepEqual(error.accounting, [{
|
|
137
|
+
provider: "provider:mock",
|
|
138
|
+
model: "mock",
|
|
139
|
+
outcome: "error",
|
|
140
|
+
cost: {
|
|
141
|
+
kind: "unknown",
|
|
142
|
+
reason: "mock provider exhausted before producing a response",
|
|
143
|
+
},
|
|
144
|
+
}]);
|
|
145
|
+
return true;
|
|
146
|
+
},
|
|
147
|
+
);
|
|
122
148
|
});
|
|
123
149
|
|
|
124
150
|
// -- {§provider-generation-envelope} --
|
|
125
151
|
|
|
126
|
-
test("the
|
|
127
|
-
|
|
128
|
-
|
|
152
|
+
test("the generation-envelope getters are on the Provider interface", () => {
|
|
153
|
+
const previous = {
|
|
154
|
+
output: process.env.PLURNK_PROVIDERS_OUTPUT_BUDGET,
|
|
155
|
+
reasoning: process.env.PLURNK_PROVIDERS_REASONING_BUDGET,
|
|
156
|
+
};
|
|
129
157
|
try {
|
|
130
|
-
process.env.
|
|
158
|
+
process.env.PLURNK_PROVIDERS_OUTPUT_BUDGET = "35%";
|
|
159
|
+
process.env.PLURNK_PROVIDERS_REASONING_BUDGET = "10%";
|
|
131
160
|
const p: Provider = new Mock({ contextWindow: 49152, responses: [] });
|
|
132
|
-
assert.equal(p.
|
|
161
|
+
assert.equal(p.outputBudget, 17_203);
|
|
162
|
+
assert.equal(p.reasoningBudget, 4_915);
|
|
163
|
+
assert.equal(p.inputCapacity, 31_949);
|
|
133
164
|
} finally {
|
|
134
|
-
if (
|
|
165
|
+
if (previous.output === undefined) delete process.env.PLURNK_PROVIDERS_OUTPUT_BUDGET;
|
|
166
|
+
else process.env.PLURNK_PROVIDERS_OUTPUT_BUDGET = previous.output;
|
|
167
|
+
if (previous.reasoning === undefined) delete process.env.PLURNK_PROVIDERS_REASONING_BUDGET;
|
|
168
|
+
else process.env.PLURNK_PROVIDERS_REASONING_BUDGET = previous.reasoning;
|
|
135
169
|
}
|
|
136
170
|
});
|
|
137
171
|
|
|
138
|
-
test("Mock resolves
|
|
139
|
-
const
|
|
140
|
-
|
|
172
|
+
test("Mock resolves percentage and absolute generation budgets against its window", () => {
|
|
173
|
+
const previous = {
|
|
174
|
+
output: process.env.PLURNK_PROVIDERS_OUTPUT_BUDGET,
|
|
175
|
+
reasoning: process.env.PLURNK_PROVIDERS_REASONING_BUDGET,
|
|
176
|
+
};
|
|
141
177
|
try {
|
|
142
|
-
process.env.
|
|
143
|
-
process.env.
|
|
178
|
+
process.env.PLURNK_PROVIDERS_OUTPUT_BUDGET = "35%";
|
|
179
|
+
process.env.PLURNK_PROVIDERS_REASONING_BUDGET = "8192";
|
|
144
180
|
const m = new Mock({ contextWindow: 49152, responses: [] });
|
|
145
|
-
assert.equal(m.
|
|
146
|
-
assert.equal(m.
|
|
181
|
+
assert.equal(m.outputBudget, 17_203);
|
|
182
|
+
assert.equal(m.reasoningBudget, 8_192);
|
|
147
183
|
} finally {
|
|
148
|
-
if (
|
|
149
|
-
|
|
184
|
+
if (previous.output === undefined) delete process.env.PLURNK_PROVIDERS_OUTPUT_BUDGET;
|
|
185
|
+
else process.env.PLURNK_PROVIDERS_OUTPUT_BUDGET = previous.output;
|
|
186
|
+
if (previous.reasoning === undefined) delete process.env.PLURNK_PROVIDERS_REASONING_BUDGET;
|
|
187
|
+
else process.env.PLURNK_PROVIDERS_REASONING_BUDGET = previous.reasoning;
|
|
150
188
|
}
|
|
151
189
|
});
|
|
152
190
|
|
|
153
|
-
test("no
|
|
191
|
+
test("no generation-budget env leaves a bare Mock unbounded", () => {
|
|
154
192
|
const m = new Mock({ contextWindow: 49152, responses: [] });
|
|
155
|
-
assert.equal(m.
|
|
156
|
-
assert.equal(m.
|
|
193
|
+
assert.equal(m.outputBudget, null);
|
|
194
|
+
assert.equal(m.reasoningBudget, null);
|
|
195
|
+
assert.equal(m.inputCapacity, null);
|
|
157
196
|
});
|
package/src/Mock.ts
CHANGED
|
@@ -5,15 +5,15 @@
|
|
|
5
5
|
// Provider contract. Production providers don't expose the `ops` escape
|
|
6
6
|
// hatch — that's an intg-only convenience.
|
|
7
7
|
|
|
8
|
-
import type {
|
|
9
|
-
import
|
|
10
|
-
import {
|
|
8
|
+
import type { ChatMessage, FinishReason, GrammarEvidence, PromptTokenMeasurement, Provider, ProviderAssistant, ProviderCost, ProviderEncryptedReasoningItem, ProviderRequestAccounting, ProviderRequestCapacity, ProviderResponse, ProviderUsage } from "./types.ts";
|
|
9
|
+
import { resolveGenerationEnvelopeFromEnv } from "./env.ts";
|
|
10
|
+
import { validateProviderRequestAccounting } from "./accounting.ts";
|
|
11
|
+
import { ProviderError } from "./errors.ts";
|
|
12
|
+
import { assessRequestCapacity, effectiveInputCapacity, effectiveOutputBudget, effectiveReasoningBudget } from "./capacity.ts";
|
|
11
13
|
|
|
12
14
|
export type MockAssistant = {
|
|
13
15
|
content: string;
|
|
14
16
|
reasoning: string | null;
|
|
15
|
-
// Partial — omitted fields fall back to DEFAULT_USAGE (e.g. reasoning: 0).
|
|
16
|
-
usage?: Partial<ProviderUsage>;
|
|
17
17
|
finishReason?: FinishReason;
|
|
18
18
|
model?: string;
|
|
19
19
|
// Provider-normalized encrypted reasoning fixture.
|
|
@@ -28,7 +28,9 @@ export type MockAssistant = {
|
|
|
28
28
|
export type MockResponse = {
|
|
29
29
|
assistant: MockAssistant;
|
|
30
30
|
assistantRaw?: unknown;
|
|
31
|
-
|
|
31
|
+
// Partial — omitted fields fall back to the deliberate zero fixture.
|
|
32
|
+
usage?: Partial<ProviderUsage>;
|
|
33
|
+
cost?: ProviderCost;
|
|
32
34
|
grammarEvidence?: GrammarEvidence;
|
|
33
35
|
};
|
|
34
36
|
|
|
@@ -36,34 +38,44 @@ export type MockResponse = {
|
|
|
36
38
|
export type MockReturnedAssistant = ProviderAssistant & { ops?: unknown[] };
|
|
37
39
|
export type MockReturnedResponse = ProviderResponse & { assistant: MockReturnedAssistant };
|
|
38
40
|
|
|
39
|
-
const DEFAULT_USAGE: ProviderUsage = {
|
|
41
|
+
const DEFAULT_USAGE: ProviderUsage = {
|
|
42
|
+
inputTokens: 0,
|
|
43
|
+
outputTokens: 0,
|
|
44
|
+
totalTokens: 0,
|
|
45
|
+
};
|
|
40
46
|
type MockGenerateArgs = Omit<Parameters<Provider["generate"]>[0], "workerId"> & { workerId?: string };
|
|
41
47
|
|
|
42
48
|
export default class Mock implements Provider {
|
|
43
49
|
#contextWindow: number | null;
|
|
44
|
-
#
|
|
45
|
-
#
|
|
50
|
+
#outputBudget: number | null;
|
|
51
|
+
#reasoningBudget: number | null;
|
|
46
52
|
#queue: MockResponse[];
|
|
47
53
|
|
|
48
|
-
// {§provider-generation-envelope}
|
|
49
|
-
//
|
|
50
|
-
//
|
|
51
|
-
//
|
|
52
|
-
//
|
|
53
|
-
// without the reserves in most base tests, so absent env → null → the
|
|
54
|
-
// consumer's no-cap path, and the ~100 `new Mock({ contextWindow, responses })`
|
|
55
|
-
// sites stay untouched. Mock has no alias identity, so it reads the BARE knobs.
|
|
54
|
+
// {§provider-generation-envelope} Mock resolves the same generation
|
|
55
|
+
// envelope as a real provider, but tolerates absent policy because it is
|
|
56
|
+
// also the universal test fixture. No output budget means its context
|
|
57
|
+
// window alone cannot determine an input capacity. Mock has no alias
|
|
58
|
+
// identity, so it reads the bare knobs.
|
|
56
59
|
constructor({ contextWindow, responses }: { contextWindow: number | null; responses: MockResponse[] }) {
|
|
57
60
|
this.#contextWindow = contextWindow;
|
|
58
|
-
const
|
|
59
|
-
this.#
|
|
60
|
-
this.#
|
|
61
|
+
const envelope = resolveGenerationEnvelopeFromEnv(process.env, contextWindow);
|
|
62
|
+
this.#outputBudget = envelope.outputBudget;
|
|
63
|
+
this.#reasoningBudget = envelope.reasoningBudget;
|
|
61
64
|
this.#queue = [...responses];
|
|
62
65
|
}
|
|
63
66
|
|
|
64
67
|
get contextWindow(): number | null { return this.#contextWindow; }
|
|
65
|
-
get
|
|
66
|
-
get
|
|
68
|
+
get maxInputTokens(): number | null { return null; }
|
|
69
|
+
get maxOutputTokens(): number | null { return null; }
|
|
70
|
+
get outputBudget(): number | null { return this.#outputBudget; }
|
|
71
|
+
get reasoningBudget(): number | null { return this.#reasoningBudget; }
|
|
72
|
+
get inputCapacity(): number | null {
|
|
73
|
+
return effectiveInputCapacity({
|
|
74
|
+
contextWindow: this.#contextWindow,
|
|
75
|
+
maxInputTokens: this.maxInputTokens,
|
|
76
|
+
outputBudget: this.#outputBudget,
|
|
77
|
+
});
|
|
78
|
+
}
|
|
67
79
|
get model(): string { return "mock"; }
|
|
68
80
|
|
|
69
81
|
// Mock's deliberately simple vocabulary defines each two content code units
|
|
@@ -77,22 +89,79 @@ export default class Mock implements Provider {
|
|
|
77
89
|
};
|
|
78
90
|
}
|
|
79
91
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
92
|
+
async assessRequestCapacity(
|
|
93
|
+
messages: readonly ChatMessage[],
|
|
94
|
+
maxOutputTokens?: number,
|
|
95
|
+
): Promise<ProviderRequestCapacity> {
|
|
96
|
+
const outputBudget = effectiveOutputBudget({
|
|
97
|
+
requested: maxOutputTokens,
|
|
98
|
+
configured: this.#outputBudget,
|
|
99
|
+
maxOutputTokens: null,
|
|
100
|
+
contextWindow: this.#contextWindow,
|
|
101
|
+
});
|
|
102
|
+
const reasoningBudget = effectiveReasoningBudget({
|
|
103
|
+
configured: this.#reasoningBudget,
|
|
104
|
+
outputBudget,
|
|
105
|
+
});
|
|
106
|
+
return assessRequestCapacity({
|
|
107
|
+
contextWindow: this.#contextWindow,
|
|
108
|
+
maxInputTokens: null,
|
|
109
|
+
maxOutputTokens: null,
|
|
110
|
+
outputBudget,
|
|
111
|
+
reasoningBudget,
|
|
112
|
+
measurement: await this.countPromptTokens(messages),
|
|
113
|
+
});
|
|
114
|
+
}
|
|
83
115
|
|
|
84
|
-
async generate({ signal, grammar }: MockGenerateArgs): Promise<MockReturnedResponse> {
|
|
116
|
+
async generate({ messages, maxOutputTokens, signal, grammar, observeRequest }: MockGenerateArgs): Promise<MockReturnedResponse> {
|
|
85
117
|
// Honor abort before consuming the queue — an aborted call makes no
|
|
86
118
|
// "wire call" and must not exhaust a queued response
|
|
87
119
|
// ({§provider-failure-normalization}).
|
|
88
120
|
signal?.throwIfAborted();
|
|
121
|
+
const capacity = await this.assessRequestCapacity(messages, maxOutputTokens);
|
|
122
|
+
if (capacity.decision === "reject") {
|
|
123
|
+
throw new ProviderError("mock", "capacity_exceeded", "Mock request exceeds its exact input capacity.", {
|
|
124
|
+
capacity,
|
|
125
|
+
extensions: { capacityStage: "preflight", capacity },
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
const settle = await observeRequest?.({ provider: "provider:mock", model: this.model });
|
|
89
129
|
const next = this.#queue.shift();
|
|
90
|
-
if (next === undefined)
|
|
130
|
+
if (next === undefined) {
|
|
131
|
+
const accounting = validateProviderRequestAccounting({
|
|
132
|
+
provider: "provider:mock",
|
|
133
|
+
model: this.model,
|
|
134
|
+
outcome: "error",
|
|
135
|
+
cost: { kind: "unknown", reason: "mock provider exhausted before producing a response" },
|
|
136
|
+
});
|
|
137
|
+
await settle?.(accounting);
|
|
138
|
+
throw new ProviderError(
|
|
139
|
+
"mock",
|
|
140
|
+
"invalid_response",
|
|
141
|
+
"Mock provider exhausted: no more queued responses",
|
|
142
|
+
{ accounting: [accounting], capacity },
|
|
143
|
+
);
|
|
144
|
+
}
|
|
91
145
|
const a = next.assistant;
|
|
146
|
+
const usage: ProviderUsage = {
|
|
147
|
+
...DEFAULT_USAGE,
|
|
148
|
+
...next.usage,
|
|
149
|
+
};
|
|
150
|
+
const requestAccounting: ProviderRequestAccounting = validateProviderRequestAccounting({
|
|
151
|
+
provider: "provider:mock",
|
|
152
|
+
model: a.model ?? this.model,
|
|
153
|
+
outcome: "response",
|
|
154
|
+
usage,
|
|
155
|
+
cost: next.cost ?? {
|
|
156
|
+
kind: "estimated",
|
|
157
|
+
amount: { amount: "0", currency: "USD" },
|
|
158
|
+
source: "mock provider fixture",
|
|
159
|
+
},
|
|
160
|
+
});
|
|
161
|
+
await settle?.(requestAccounting);
|
|
92
162
|
const assistant: MockReturnedAssistant = {
|
|
93
163
|
content: a.content,
|
|
94
164
|
reasoning: a.reasoning,
|
|
95
|
-
usage: { ...DEFAULT_USAGE, ...a.usage },
|
|
96
165
|
...(a.reasoningEncrypted !== undefined ? { reasoningEncrypted: a.reasoningEncrypted } : {}),
|
|
97
166
|
finishReason: a.finishReason ?? "stop",
|
|
98
167
|
model: a.model ?? "mock",
|
|
@@ -105,7 +174,8 @@ export default class Mock implements Provider {
|
|
|
105
174
|
return {
|
|
106
175
|
assistant,
|
|
107
176
|
assistantRaw: next.assistantRaw ?? null,
|
|
108
|
-
|
|
177
|
+
accounting: [requestAccounting],
|
|
178
|
+
capacity,
|
|
109
179
|
...(grammarEvidence !== undefined ? { grammarEvidence } : {}),
|
|
110
180
|
};
|
|
111
181
|
}
|
package/src/Pool.test.ts
CHANGED
|
@@ -4,16 +4,43 @@ import Pool from "./Pool.ts";
|
|
|
4
4
|
import { ProviderError } from "./errors.ts";
|
|
5
5
|
import type { PromptTokenMeasurement, Provider, ProviderResponse } from "./types.ts";
|
|
6
6
|
import { resetEmittedWarnings } from "./warnings.ts";
|
|
7
|
+
import { effectiveInputCapacity } from "./capacity.ts";
|
|
7
8
|
|
|
8
9
|
test.afterEach(() => { resetEmittedWarnings(); });
|
|
9
10
|
|
|
10
|
-
const RESP
|
|
11
|
+
const RESP: ProviderResponse = {
|
|
12
|
+
assistant: {
|
|
13
|
+
content: "ok",
|
|
14
|
+
reasoning: null,
|
|
15
|
+
finishReason: "stop",
|
|
16
|
+
model: "gemma",
|
|
17
|
+
},
|
|
18
|
+
assistantRaw: null,
|
|
19
|
+
accounting: [{
|
|
20
|
+
provider: "provider:test",
|
|
21
|
+
model: "gemma",
|
|
22
|
+
outcome: "response",
|
|
23
|
+
usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
|
|
24
|
+
cost: { kind: "unknown", reason: "test fixture has no cost" },
|
|
25
|
+
}],
|
|
26
|
+
capacity: {
|
|
27
|
+
decision: "admit",
|
|
28
|
+
contextWindow: 48_000,
|
|
29
|
+
maxInputTokens: null,
|
|
30
|
+
maxOutputTokens: null,
|
|
31
|
+
outputBudget: 12_000,
|
|
32
|
+
reasoningBudget: null,
|
|
33
|
+
inputCapacity: 36_000,
|
|
34
|
+
prompt: { kind: "exact", tokens: 0, source: "test:exact" },
|
|
35
|
+
},
|
|
36
|
+
};
|
|
11
37
|
|
|
12
38
|
type FakeOpts = {
|
|
13
39
|
model?: string; window?: number | null; servedModel?: string;
|
|
14
|
-
constrainsOutput?: boolean;
|
|
15
|
-
|
|
16
|
-
|
|
40
|
+
constrainsOutput?: boolean; requiresOutputBudget?: boolean;
|
|
41
|
+
maxInputTokens?: number | null; maxOutputTokens?: number | null;
|
|
42
|
+
outputBudget?: number | null; reasoningBudget?: number | null;
|
|
43
|
+
tokenize?: boolean; throws?: Error;
|
|
17
44
|
promptMeasurement?: PromptTokenMeasurement;
|
|
18
45
|
};
|
|
19
46
|
// A fake backend that records which workers it served, and optionally throws.
|
|
@@ -22,18 +49,38 @@ const backend = (opts: FakeOpts = {}) => {
|
|
|
22
49
|
const b: Provider = {
|
|
23
50
|
model: opts.model ?? "gemma",
|
|
24
51
|
contextWindow: opts.window === undefined ? 48000 : opts.window,
|
|
52
|
+
maxInputTokens: opts.maxInputTokens ?? null,
|
|
53
|
+
maxOutputTokens: opts.maxOutputTokens ?? null,
|
|
54
|
+
outputBudget: opts.outputBudget ?? null,
|
|
55
|
+
reasoningBudget: opts.reasoningBudget ?? null,
|
|
56
|
+
inputCapacity: effectiveInputCapacity({
|
|
57
|
+
contextWindow: opts.window === undefined ? 48_000 : opts.window,
|
|
58
|
+
maxInputTokens: opts.maxInputTokens ?? null,
|
|
59
|
+
outputBudget: opts.outputBudget ?? null,
|
|
60
|
+
}),
|
|
25
61
|
...(opts.servedModel !== undefined ? { servedModel: opts.servedModel } : {}),
|
|
26
62
|
...(opts.constrainsOutput !== undefined ? { constrainsOutput: opts.constrainsOutput } : {}),
|
|
27
|
-
...(opts.
|
|
28
|
-
...(opts.reasoningReserve !== undefined ? { reasoningReserve: opts.reasoningReserve } : {}),
|
|
29
|
-
...(opts.completionReserve !== undefined ? { completionReserve: opts.completionReserve } : {}),
|
|
63
|
+
...(opts.requiresOutputBudget !== undefined ? { requiresOutputBudget: opts.requiresOutputBudget } : {}),
|
|
30
64
|
...(opts.tokenize ? { tokenize: async (t: string) => [t.length] } : {}),
|
|
31
65
|
countPromptTokens: async (messages) => opts.promptMeasurement ?? ({
|
|
32
66
|
kind: "exact",
|
|
33
67
|
tokens: messages.reduce((sum, { content }) => sum + content.length, 0),
|
|
34
68
|
source: "test:exact",
|
|
35
69
|
}),
|
|
36
|
-
|
|
70
|
+
assessRequestCapacity: async (messages, maxOutputTokens) => ({
|
|
71
|
+
decision: "admit",
|
|
72
|
+
contextWindow: opts.window === undefined ? 48_000 : opts.window,
|
|
73
|
+
maxInputTokens: opts.maxInputTokens ?? null,
|
|
74
|
+
maxOutputTokens: opts.maxOutputTokens ?? null,
|
|
75
|
+
outputBudget: maxOutputTokens ?? opts.outputBudget ?? null,
|
|
76
|
+
reasoningBudget: opts.reasoningBudget ?? null,
|
|
77
|
+
inputCapacity: null,
|
|
78
|
+
prompt: opts.promptMeasurement ?? {
|
|
79
|
+
kind: "exact",
|
|
80
|
+
tokens: messages.reduce((sum, { content }) => sum + content.length, 0),
|
|
81
|
+
source: "test:exact",
|
|
82
|
+
},
|
|
83
|
+
}),
|
|
37
84
|
generate: async (args: Parameters<Provider["generate"]>[0]): Promise<ProviderResponse> => {
|
|
38
85
|
served.push(args.workerId);
|
|
39
86
|
if (opts.throws !== undefined) throw opts.throws;
|
|
@@ -68,20 +115,22 @@ test("Pool: any unknown (null) window makes the pool null - no improvised cap",
|
|
|
68
115
|
assert.equal(new Pool([backend({ window: 48000 }).b, backend({ window: null }).b]).contextWindow, null);
|
|
69
116
|
});
|
|
70
117
|
|
|
71
|
-
test("Pool:
|
|
72
|
-
const big = backend({ window: 48000,
|
|
73
|
-
const small = backend({ window: 32000,
|
|
118
|
+
test("Pool: physical limits and budgets aggregate to independent safe floors", () => {
|
|
119
|
+
const big = backend({ window: 48000, maxInputTokens: 40_000, maxOutputTokens: 16_000, outputBudget: 12_000, reasoningBudget: 4_800 }).b;
|
|
120
|
+
const small = backend({ window: 32000, maxInputTokens: 24_000, maxOutputTokens: 12_000, outputBudget: 8_000, reasoningBudget: 3_200 }).b;
|
|
74
121
|
const p = new Pool([big, small]);
|
|
75
122
|
assert.equal(p.contextWindow, 32000);
|
|
76
|
-
assert.equal(p.
|
|
77
|
-
assert.equal(p.
|
|
123
|
+
assert.equal(p.maxInputTokens, 24_000);
|
|
124
|
+
assert.equal(p.maxOutputTokens, 12_000);
|
|
125
|
+
assert.equal(p.outputBudget, 8_000);
|
|
126
|
+
assert.equal(p.reasoningBudget, 3_200);
|
|
78
127
|
});
|
|
79
128
|
|
|
80
|
-
test("Pool: capabilities aggregate conservatively (constrainsOutput all-true,
|
|
129
|
+
test("Pool: capabilities aggregate conservatively (constrainsOutput all-true, requiresOutputBudget any-true)", () => {
|
|
81
130
|
assert.equal(new Pool([backend({ constrainsOutput: true }).b, backend({ constrainsOutput: true }).b]).constrainsOutput, true);
|
|
82
131
|
assert.equal(new Pool([backend({ constrainsOutput: true }).b, backend({ constrainsOutput: false }).b]).constrainsOutput, undefined);
|
|
83
|
-
assert.equal(new Pool([backend({
|
|
84
|
-
assert.equal(new Pool([backend({}).b]).
|
|
132
|
+
assert.equal(new Pool([backend({ requiresOutputBudget: false }).b, backend({ requiresOutputBudget: true }).b]).requiresOutputBudget, true);
|
|
133
|
+
assert.equal(new Pool([backend({}).b]).requiresOutputBudget, undefined);
|
|
85
134
|
});
|
|
86
135
|
|
|
87
136
|
test("Pool: servedModel is the common id, undefined when they differ", () => {
|
|
@@ -94,12 +143,11 @@ test("Pool: tokenize is exposed iff every backend has it", () => {
|
|
|
94
143
|
assert.equal(new Pool([backend({ tokenize: true }).b, backend({ tokenize: false }).b]).tokenize, undefined);
|
|
95
144
|
});
|
|
96
145
|
|
|
97
|
-
test("Pool: prompt counting
|
|
98
|
-
const p = new Pool([backend(
|
|
146
|
+
test("Pool: prompt counting delegates conservatively to its backends", async () => {
|
|
147
|
+
const p = new Pool([backend().b]);
|
|
99
148
|
assert.deepEqual(await p.countPromptTokens([{ role: "user", content: "abcd" }]), {
|
|
100
149
|
kind: "exact", tokens: 4, source: "pool:test:exact",
|
|
101
150
|
});
|
|
102
|
-
assert.equal(p.calculateCost({ prompt: 0, completion: 0, reasoning: 0, cached: 0, total: 0 }), 42);
|
|
103
151
|
});
|
|
104
152
|
|
|
105
153
|
test("Pool: prompt evidence is conservative across every routable backend", async () => {
|
|
@@ -118,6 +166,29 @@ test("Pool: prompt evidence is conservative across every routable backend", asyn
|
|
|
118
166
|
source: "pool:test:a,heuristic:chars2",
|
|
119
167
|
detail: "at least one interchangeable backend has only an estimate: unknown framing",
|
|
120
168
|
});
|
|
169
|
+
|
|
170
|
+
const unavailable = backend({ promptMeasurement: {
|
|
171
|
+
kind: "unavailable", source: "test:none", detail: "counter offline",
|
|
172
|
+
} }).b;
|
|
173
|
+
assert.deepEqual(await new Pool([exact, unavailable]).countPromptTokens([]), {
|
|
174
|
+
kind: "unavailable",
|
|
175
|
+
source: "pool:test:a,test:none",
|
|
176
|
+
detail: "at least one interchangeable backend cannot measure the request: counter offline",
|
|
177
|
+
});
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
test("Pool: request capacity uses the smallest complete backend envelope", async () => {
|
|
181
|
+
const prompt = { kind: "exact", tokens: 11, source: "test:exact" } as const;
|
|
182
|
+
const narrowInput = backend({ window: 100, outputBudget: 90, promptMeasurement: prompt }).b;
|
|
183
|
+
const narrowContext = backend({ window: 50, outputBudget: 10, promptMeasurement: prompt }).b;
|
|
184
|
+
const pool = new Pool([narrowInput, narrowContext]);
|
|
185
|
+
|
|
186
|
+
assert.equal(pool.inputCapacity, 10);
|
|
187
|
+
const capacity = await pool.assessRequestCapacity([]);
|
|
188
|
+
assert.equal(capacity.contextWindow, 50);
|
|
189
|
+
assert.equal(capacity.outputBudget, 10);
|
|
190
|
+
assert.equal(capacity.inputCapacity, 10, "independent minima do not synthesize a nonexistent 40-token envelope");
|
|
191
|
+
assert.equal(capacity.decision, "reject");
|
|
121
192
|
});
|
|
122
193
|
|
|
123
194
|
// --- dispatch: round-robin + affinity ---
|