@plurnk/plurnk-providers 6.1.1 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.defaults +5 -4
- package/README.md +1 -1
- package/SPEC.md +10 -8
- package/dist/InferenceAdmission.d.ts +10 -1
- package/dist/InferenceAdmission.d.ts.map +1 -1
- package/dist/InferenceAdmission.js +79 -15
- package/dist/InferenceAdmission.js.map +1 -1
- package/dist/ModelProvider.d.ts +3 -1
- package/dist/ModelProvider.d.ts.map +1 -1
- package/dist/ModelProvider.js +8 -5
- package/dist/ModelProvider.js.map +1 -1
- package/dist/catalogProvider.d.ts +2 -2
- package/dist/catalogProvider.d.ts.map +1 -1
- package/dist/catalogProvider.js +2 -1
- package/dist/catalogProvider.js.map +1 -1
- package/dist/compatibleProvider.d.ts.map +1 -1
- package/dist/compatibleProvider.js +6 -3
- package/dist/compatibleProvider.js.map +1 -1
- package/dist/env.d.ts +3 -1
- package/dist/env.d.ts.map +1 -1
- package/dist/env.js +24 -2
- package/dist/env.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/modelAdapters.d.ts +2 -2
- package/dist/modelAdapters.d.ts.map +1 -1
- package/dist/modelAdapters.js +53 -4
- package/dist/modelAdapters.js.map +1 -1
- package/dist/pi-transport.d.ts +5 -3
- package/dist/pi-transport.d.ts.map +1 -1
- package/dist/pi-transport.js +248 -51
- package/dist/pi-transport.js.map +1 -1
- package/dist/types.d.ts +2 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +3 -3
package/.env.defaults
CHANGED
|
@@ -16,8 +16,9 @@
|
|
|
16
16
|
# PLURNK_BASEURL_<alias>=<url>
|
|
17
17
|
|
|
18
18
|
# --- Capacity ---
|
|
19
|
-
# Simultaneous inference attempts per resolved endpoint in this process; -1 = unrestricted
|
|
20
|
-
#
|
|
19
|
+
# Simultaneous inference attempts per resolved endpoint in this process; -1 = unrestricted, or
|
|
20
|
+
# bounded automatically by probed physical slot capacity (e.g. llama-server GET /props total_slots).
|
|
21
|
+
# Positive limits queue excess calls and are capped by probed physical slots.
|
|
21
22
|
PLURNK_PROVIDERS_MAX_CONCURRENCY=-1
|
|
22
23
|
# PLURNK_PROVIDERS_MAX_CONCURRENCY_local=1
|
|
23
24
|
# Reserved output INCLUDING reasoning: positive tokens or context percentage in (0,100).
|
|
@@ -77,9 +78,9 @@ PLURNK_PROVIDERS_DRY_MULTIPLIER=0
|
|
|
77
78
|
# --- Cache and billing ---
|
|
78
79
|
# Stable Session affinity on supported routes: 1 = on; 0 = off. Unknown routes get no guessed field.
|
|
79
80
|
PLURNK_PROVIDERS_CACHE_AFFINITY=1
|
|
80
|
-
# Explicit cache writes on explicit-breakpoint routes (Anthropic, OpenRouter Anthropic, Bedrock Claude):
|
|
81
|
+
# Explicit cache writes on explicit-breakpoint routes (Anthropic, MiniMax Anthropic, OpenRouter Anthropic, Bedrock Claude, OpenAI GPT-5.6+):
|
|
81
82
|
# conversation = system + rolling conversation/log prefix before volatile turn status;
|
|
82
|
-
# stable-system = reusable system boundary only; off = no explicit writes. Distinct from affinity and affects billing.
|
|
83
|
+
# stable-system = reusable system boundary only; off = no explicit writes (suppresses paid implicit writes on OpenAI GPT-5.6+). Distinct from affinity and affects billing.
|
|
83
84
|
PLURNK_PROVIDERS_CACHE_WRITE_POLICY=conversation
|
|
84
85
|
# Provider service tier; unset = provider default. Usually set per alias: routing can change cost.
|
|
85
86
|
# Native routes send it only where the SDK documents one ({§provider-request-authority}); others warn.
|
package/README.md
CHANGED
|
@@ -58,7 +58,7 @@ In-package `openai` and `ollama` adapters probe live context windows, `llama-ser
|
|
|
58
58
|
| --- | --- |
|
|
59
59
|
| Effort & reasoning budget | `adaptive` requests dynamic reasoning or falls back to `PLURNK_PROVIDERS_EFFORT_FALLBACK` (`high`); explicit fixed efforts must be route-supported. Unset reasoning budget leaves reasoning enabled. |
|
|
60
60
|
| Output & floor (`PLURNK_PROVIDERS_OUTPUT_FLOOR`) | Shipped `25%` floor sets the input wall (`contextWindow - floor`); packets past the output budget flex output down to the remainder ({§provider-capacity-admission}). |
|
|
61
|
-
| Concurrency & deadlines |
|
|
61
|
+
| Concurrency & deadlines | Per-endpoint FIFO `MAX_CONCURRENCY` gate ({§provider-inference-admission}), automatically bounded by probed `llama-server` `total_slots` and resized live across `.env` reloads; one whole-call generation deadline across queueing and `Retry-After` retries. |
|
|
62
62
|
| Provider extensions | First-party `@plurnk/*` packages with `plurnk: { "kind": "provider", "name": "..." }` default-export `ProviderExtension` (`model(modelId)`, optional `attributions(context)`, {§extension-discovery}). |
|
|
63
63
|
|
|
64
64
|
## Live conformance & development
|
package/SPEC.md
CHANGED
|
@@ -59,10 +59,10 @@ Every physical attempt settles a `ProviderRequestAccounting` record (`{ provider
|
|
|
59
59
|
`ModelProvider.ts` delegates streaming, protocol framing, vendor request assembly, model compatibility flags (`getBuiltinModels`), header normalization (`headersToRecord`), interruptible backoff (`sleep`), token estimation (`estimateMessageTokens`), and context-overflow pattern matching (`getOverflowPatterns`) to `@earendil-works/pi-ai` across all supported APIs (`anthropic-messages`, `openai-completions`, `openai-responses`, `google-generative-ai`, `bedrock-converse-stream`):
|
|
60
60
|
|
|
61
61
|
- Prompt cache affinity (`PLURNK_PROVIDERS_CACHE_AFFINITY=1`) projects each provider's documented session key (`sessionId` -> `prompt_cache_key` / `x-session-id` / `x-session-affinity`).
|
|
62
|
-
- Explicit prompt-cache write policy (`PLURNK_PROVIDERS_CACHE_WRITE_POLICY`, default `conversation`: `conversation | stable-system | off`) governs explicit cache breakpoints (`cache_control: { type: "ephemeral" }` on Anthropic and OpenRouter Anthropic routes; `cachePoint: { type: "default" }` on Bedrock Claude prompt-caching routes), suppressed on stateless `callKind: "bare"` calls:
|
|
63
|
-
- `off
|
|
62
|
+
- Explicit prompt-cache write policy (`PLURNK_PROVIDERS_CACHE_WRITE_POLICY`, default `conversation`: `conversation | stable-system | off`) governs explicit cache breakpoints (`cache_control: { type: "ephemeral" }` on Anthropic, MiniMax Anthropic, and OpenRouter Anthropic routes; `cachePoint: { type: "default" }` on Bedrock Claude prompt-caching routes; `prompt_cache_options: { mode: "explicit" }` with `prompt_cache_breakpoint: { mode: "explicit" }` on OpenAI GPT-5.6+ routes, including OpenRouter `openai/*`), suppressed on stateless `callKind: "bare"` calls:
|
|
63
|
+
- `off` (and `callKind: "bare"`): zero explicit cache breakpoints; on OpenAI GPT-5.6+ routes `prompt_cache_options: { mode: "explicit" }` is still sent with zero breakpoints so implicit paid cache writes are disabled.
|
|
64
64
|
- `stable-system`: one breakpoint on the final leading `system` instruction.
|
|
65
|
-
- `conversation`: breakpoints on the final leading `system` instruction, the penultimate conversation message, and the
|
|
65
|
+
- `conversation`: splits caller-delimited `userCachePrefixes` on historical `user` messages into separate content blocks so prior turns' cached durable log blocks retain their exact block boundaries. Ephemeral-breakpoint routes (`cache_control` / `cachePoint`) place breakpoints on the final leading `system` instruction, the penultimate conversation message, and the durable prefix of the final `user` message before any volatile per-turn status tail (or the full final `user` message when no volatile tail is delimited), staying within three breakpoints per request; OpenAI GPT-5.6+ explicit routes keep monotonic `prompt_cache_breakpoint` markers on the final leading `system` instruction and each `user` message's durable prefix across turns so prior segment boundaries remain stable while writing only new uncached prefixes.
|
|
66
66
|
- Output token allowance (`responseMax ?? outputBudget`) is enforced both in flight during streaming (aborting a runaway stream once mid-stream reported output tokens or conservative token lower-bound `floor(estimateMessageTokens(partial) / 2)` exceeds the granted allowance) and on final reported `usage.outputTokens`.
|
|
67
67
|
|
|
68
68
|
## §provider-effort Reasoning effort and readable trace
|
|
@@ -84,7 +84,7 @@ Every physical attempt settles a `ProviderRequestAccounting` record (`{ provider
|
|
|
84
84
|
| `PLURNK_PROVIDERS_EFFORT` / `EFFORT_FALLBACK` | `adaptive` / `high` | Portable reasoning effort selector and graded-route fallback |
|
|
85
85
|
| `PLURNK_PROVIDERS_FETCH_TIMEOUT` / `OPERATION_TIMEOUT` | `30000` / `600000` ms | Per-attempt fetch/idle timeout and total operation wall-clock timeout |
|
|
86
86
|
| `PLURNK_PROVIDERS_RETRY_ATTEMPTS` | `3` | Maximum automatic retry attempts for retryable transport failures |
|
|
87
|
-
| `PLURNK_PROVIDERS_MAX_CONCURRENCY` | `-1` (unbounded or positive integer) | Per-endpoint FIFO concurrency gate ({§provider-inference-admission}) |
|
|
87
|
+
| `PLURNK_PROVIDERS_MAX_CONCURRENCY` | `-1` (unbounded when unprobed, or positive integer) | Per-endpoint FIFO concurrency gate, automatically bounded by probed physical endpoint slots ({§provider-inference-admission}) |
|
|
88
88
|
| `PLURNK_PROVIDERS_COST_<CATEGORY>` | empty (non-negative USD per 1M tokens for `INPUT`, `OUTPUT`, `REASONING`, `CACHE_READ`, `CACHE_WRITE`) | Per-category operator rate overrides merged over catalog rates ({§provider-monetary-evidence}) |
|
|
89
89
|
| `PLURNK_PROVIDERS_RAWBODY` | `0` (`0 \| 1`) | Retains redacted raw request/response forensic bodies in `forensicEvidence` |
|
|
90
90
|
|
|
@@ -97,7 +97,7 @@ Every physical attempt settles a `ProviderRequestAccounting` record (`{ provider
|
|
|
97
97
|
| Route family | Fact authority |
|
|
98
98
|
|---|---|
|
|
99
99
|
| Cataloged cloud provider (`catalogProvider.ts`) | `resolveModel(provider, model)` from `@plurnk/plurnk-models`, capped by optional `PLURNK_PROVIDERS_CONTEXT_WINDOW`. Uncataloged models on a catalog provider require an explicit `PLURNK_PROVIDERS_CONTEXT_WINDOW`. Catalog credential names (`info.env`) or `PLURNK_PROVIDERS_PROVIDER_<ID>_API_KEY_ENV` (one exact variable name, never a comma list) supply authentication. |
|
|
100
|
-
| Local / compatible endpoint (`compatibleProvider.ts`) | Probes `GET <baseUrl>/models` for `n_ctx` / `max_model_len` / `context_length`. When llama-server is detected (`meta.n_ctx` or `PLURNK_PROVIDERS_LLAMA_SERVER=1`), probes `/props` for `total_slots` and `eos_token` and measures prompts via `/v1/chat/completions/input_tokens`. |
|
|
100
|
+
| Local / compatible endpoint (`compatibleProvider.ts`) | Probes `GET <baseUrl>/models` for `n_ctx` / `max_model_len` / `context_length`. When llama-server is detected (`meta.n_ctx` or `PLURNK_PROVIDERS_LLAMA_SERVER=1`), probes `/props` for `total_slots` (bounding endpoint inference concurrency via {§provider-inference-admission}) and `eos_token` and measures prompts via `/v1/chat/completions/input_tokens`. |
|
|
101
101
|
| Built-in Ollama rail (`ollama.ts`) | Probes `POST <origin>/api/show` for `*.context_length`, capped by optional `PLURNK_PROVIDERS_CONTEXT_WINDOW`, and dispatches chat completions to `<origin>/v1/chat/completions`. |
|
|
102
102
|
| Operator probe scripts (`scripts/`) | `providers-ping.mjs` probes one cheapest declared route per keyed provider and writes redacted JSON evidence; `token-envelope-probe.mjs` probes explicit aliases across configured or physical output limits. |
|
|
103
103
|
|
|
@@ -132,8 +132,10 @@ All transport and response failures normalize into `ProviderError` carrying an R
|
|
|
132
132
|
|
|
133
133
|
## §provider-inference-admission Inference admission
|
|
134
134
|
|
|
135
|
-
`InferenceAdmission.forEndpoint(endpoint, maxConcurrency)` coordinates in-flight generations across all `Provider` instances targeting the same canonical endpoint URL (`PLURNK_PROVIDERS_MAX_CONCURRENCY`, default `-1` unbounded, or a positive safe integer):
|
|
135
|
+
`InferenceAdmission.forEndpoint(endpoint, maxConcurrency, options?)` coordinates in-flight generations across all `Provider` instances targeting the same canonical endpoint URL (`PLURNK_PROVIDERS_MAX_CONCURRENCY`, default `-1` unbounded, or a positive safe integer):
|
|
136
136
|
|
|
137
|
+
- `effectiveMaxConcurrency(configuredLimit, physicalSlots)` reconciles operator policy with probed physical endpoint slot capacity (`GET /props` `total_slots` on llama-server routes, `null` when unprobed): `-1` adopts `physicalSlots` when probed (remaining `-1` when `null`), and a positive `configuredLimit` resolves to `min(configuredLimit, physicalSlots)` when probed.
|
|
137
138
|
- Waiters acquire slots in strict FIFO order before opening a physical request (`observeRequest` does not fire while queued) and hold the slot until the response stream or error settles.
|
|
138
|
-
- Aborted waiters are removed from the queue immediately without consuming capacity; retry backoff releases the slot before sleeping and re-acquires it for the next attempt.
|
|
139
|
-
-
|
|
139
|
+
- Aborted waiters are removed from the queue immediately in $O(1)$ without consuming capacity or retaining signal listeners; retry backoff releases the slot before sleeping and re-acquires it for the next attempt.
|
|
140
|
+
- Within one configuration generation, conflicting effective allowances across aliases targeting the same canonical endpoint throw at construction; aliases that resolve to the same effective allowance (such as `-1` alongside explicit `1` on a 1-slot llama-server) share the gate.
|
|
141
|
+
- Advancing the configuration generation (`InferenceAdmission.advanceGeneration`) on operator `.env` reload or observing a changed physical `total_slots` under unchanged operator policy resizes the existing endpoint semaphore in place: growing immediately wakes queued FIFO waiters up to the new allowance, and shrinking lets surplus in-flight leases finish naturally before admitting queued work.
|
|
@@ -1,7 +1,16 @@
|
|
|
1
|
+
export interface InferenceAdmissionOptions {
|
|
2
|
+
readonly configuredLimit?: number;
|
|
3
|
+
readonly generation?: number;
|
|
4
|
+
}
|
|
1
5
|
export default class InferenceAdmission {
|
|
2
6
|
#private;
|
|
3
|
-
static
|
|
7
|
+
static get currentGeneration(): number;
|
|
8
|
+
static advanceGeneration(generation?: number): number;
|
|
9
|
+
static forEndpoint(endpoint: string, limit: number, options?: InferenceAdmissionOptions): InferenceAdmission;
|
|
4
10
|
private constructor();
|
|
11
|
+
get limit(): number;
|
|
12
|
+
get activeCount(): number;
|
|
13
|
+
get waitingCount(): number;
|
|
5
14
|
acquire(signal?: AbortSignal): Promise<() => void>;
|
|
6
15
|
}
|
|
7
16
|
//# sourceMappingURL=InferenceAdmission.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"InferenceAdmission.d.ts","sourceRoot":"","sources":["../src/InferenceAdmission.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"InferenceAdmission.d.ts","sourceRoot":"","sources":["../src/InferenceAdmission.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,yBAAyB;IACtC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAChC;AASD,MAAM,CAAC,OAAO,OAAO,kBAAkB;;IAUnC,MAAM,KAAK,iBAAiB,IAAI,MAAM,CAErC;IAID,MAAM,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,CAYpD;IAED,MAAM,CAAC,WAAW,CACd,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,yBAAyB,GACpC,kBAAkB,CAiCpB;IAED,OAAO,eAIN;IAED,IAAI,KAAK,IAAI,MAAM,CAElB;IAED,IAAI,WAAW,IAAI,MAAM,CAExB;IAED,IAAI,YAAY,IAAI,MAAM,CAEzB;IAeD,OAAO,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CA0BjD;CACJ"}
|
|
@@ -1,37 +1,104 @@
|
|
|
1
|
-
|
|
1
|
+
const assertValidLimit = (limit) => {
|
|
2
|
+
if (!Number.isSafeInteger(limit) || limit !== -1 && limit < 1) {
|
|
3
|
+
throw new TypeError("PLURNK_PROVIDERS_MAX_CONCURRENCY must be -1 or a positive safe integer");
|
|
4
|
+
}
|
|
5
|
+
};
|
|
6
|
+
// {§provider-inference-admission} One process-local FIFO allowance per canonical endpoint.
|
|
2
7
|
export default class InferenceAdmission {
|
|
3
8
|
static #endpoints = new Map();
|
|
9
|
+
static #currentGeneration = 0;
|
|
4
10
|
#limit;
|
|
11
|
+
#configuredLimit;
|
|
12
|
+
#generation;
|
|
5
13
|
#waiting = new Set();
|
|
6
14
|
#active = 0;
|
|
7
|
-
static
|
|
8
|
-
|
|
9
|
-
|
|
15
|
+
static get currentGeneration() {
|
|
16
|
+
return this.#currentGeneration;
|
|
17
|
+
}
|
|
18
|
+
// Advance the configuration generation monotonically on operator environment reload so live
|
|
19
|
+
// endpoints resize in place while conflicting aliases within the same generation still fail fast.
|
|
20
|
+
static advanceGeneration(generation) {
|
|
21
|
+
if (generation === undefined) {
|
|
22
|
+
this.#currentGeneration += 1;
|
|
23
|
+
return this.#currentGeneration;
|
|
24
|
+
}
|
|
25
|
+
if (!Number.isSafeInteger(generation) || generation < 0) {
|
|
26
|
+
throw new TypeError("InferenceAdmission generation must be a non-negative safe integer");
|
|
27
|
+
}
|
|
28
|
+
if (generation > this.#currentGeneration) {
|
|
29
|
+
this.#currentGeneration = generation;
|
|
30
|
+
}
|
|
31
|
+
return this.#currentGeneration;
|
|
32
|
+
}
|
|
33
|
+
static forEndpoint(endpoint, limit, options) {
|
|
34
|
+
assertValidLimit(limit);
|
|
35
|
+
const configuredLimit = options?.configuredLimit ?? limit;
|
|
36
|
+
assertValidLimit(configuredLimit);
|
|
37
|
+
const generation = options?.generation ?? this.#currentGeneration;
|
|
38
|
+
if (!Number.isSafeInteger(generation) || generation < 0) {
|
|
39
|
+
throw new TypeError("InferenceAdmission generation must be a non-negative safe integer");
|
|
10
40
|
}
|
|
11
41
|
const key = new URL(endpoint).href.replace(/\/+$/, "");
|
|
12
42
|
const existing = this.#endpoints.get(key);
|
|
13
43
|
if (existing !== undefined) {
|
|
14
|
-
if (existing.#
|
|
15
|
-
|
|
44
|
+
if (generation > existing.#generation) {
|
|
45
|
+
existing.#generation = generation;
|
|
46
|
+
existing.#configuredLimit = configuredLimit;
|
|
47
|
+
existing.#resize(limit);
|
|
48
|
+
return existing;
|
|
49
|
+
}
|
|
50
|
+
if (existing.#limit === limit) {
|
|
51
|
+
if (configuredLimit !== -1)
|
|
52
|
+
existing.#configuredLimit = configuredLimit;
|
|
53
|
+
return existing;
|
|
54
|
+
}
|
|
55
|
+
// Same operator policy on a restarted endpoint whose probed physical slot count changed.
|
|
56
|
+
if (existing.#configuredLimit === configuredLimit) {
|
|
57
|
+
existing.#resize(limit);
|
|
58
|
+
return existing;
|
|
16
59
|
}
|
|
17
|
-
|
|
60
|
+
throw new TypeError(`Provider aliases resolving to ${key} have conflicting PLURNK_PROVIDERS_MAX_CONCURRENCY values ${existing.#limit} and ${limit}; one endpoint has one allowance`);
|
|
18
61
|
}
|
|
19
|
-
const admission = new InferenceAdmission(limit);
|
|
62
|
+
const admission = new InferenceAdmission(limit, configuredLimit, generation);
|
|
20
63
|
this.#endpoints.set(key, admission);
|
|
21
64
|
return admission;
|
|
22
65
|
}
|
|
23
|
-
constructor(limit) {
|
|
66
|
+
constructor(limit, configuredLimit, generation) {
|
|
24
67
|
this.#limit = limit;
|
|
68
|
+
this.#configuredLimit = configuredLimit;
|
|
69
|
+
this.#generation = generation;
|
|
70
|
+
}
|
|
71
|
+
get limit() {
|
|
72
|
+
return this.#limit;
|
|
73
|
+
}
|
|
74
|
+
get activeCount() {
|
|
75
|
+
return this.#active;
|
|
76
|
+
}
|
|
77
|
+
get waitingCount() {
|
|
78
|
+
return this.#waiting.size;
|
|
79
|
+
}
|
|
80
|
+
#resize(limit) {
|
|
81
|
+
this.#limit = limit;
|
|
82
|
+
this.#drainQueue();
|
|
83
|
+
}
|
|
84
|
+
#drainQueue() {
|
|
85
|
+
while (this.#waiting.size > 0 && (this.#limit === -1 || this.#active < this.#limit)) {
|
|
86
|
+
const next = this.#waiting.values().next().value;
|
|
87
|
+
this.#waiting.delete(next);
|
|
88
|
+
next();
|
|
89
|
+
}
|
|
25
90
|
}
|
|
26
91
|
acquire(signal) {
|
|
27
92
|
signal?.throwIfAborted();
|
|
28
93
|
return new Promise((resolve, reject) => {
|
|
94
|
+
let queued = false;
|
|
29
95
|
const abort = () => {
|
|
30
96
|
this.#waiting.delete(enter);
|
|
31
97
|
reject(signal.reason);
|
|
32
98
|
};
|
|
33
99
|
const enter = () => {
|
|
34
|
-
|
|
100
|
+
if (queued)
|
|
101
|
+
signal?.removeEventListener("abort", abort);
|
|
35
102
|
this.#active += 1;
|
|
36
103
|
let released = false;
|
|
37
104
|
resolve(() => {
|
|
@@ -39,16 +106,13 @@ export default class InferenceAdmission {
|
|
|
39
106
|
throw new Error("Inference admission lease was released twice");
|
|
40
107
|
released = true;
|
|
41
108
|
this.#active -= 1;
|
|
42
|
-
|
|
43
|
-
if (next !== undefined) {
|
|
44
|
-
this.#waiting.delete(next);
|
|
45
|
-
next();
|
|
46
|
-
}
|
|
109
|
+
this.#drainQueue();
|
|
47
110
|
});
|
|
48
111
|
};
|
|
49
112
|
if (this.#limit === -1 || this.#active < this.#limit)
|
|
50
113
|
enter();
|
|
51
114
|
else {
|
|
115
|
+
queued = true;
|
|
52
116
|
this.#waiting.add(enter);
|
|
53
117
|
signal?.addEventListener("abort", abort, { once: true });
|
|
54
118
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"InferenceAdmission.js","sourceRoot":"","sources":["../src/InferenceAdmission.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"InferenceAdmission.js","sourceRoot":"","sources":["../src/InferenceAdmission.ts"],"names":[],"mappings":"AAKA,MAAM,gBAAgB,GAAG,CAAC,KAAa,EAAQ,EAAE;IAC7C,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QAC5D,MAAM,IAAI,SAAS,CAAC,wEAAwE,CAAC,CAAC;IAClG,CAAC;AACL,CAAC,CAAC;AAEF,2FAA2F;AAC3F,MAAM,CAAC,OAAO,OAAO,kBAAkB;IACnC,MAAM,CAAU,UAAU,GAAG,IAAI,GAAG,EAA8B,CAAC;IACnE,MAAM,CAAC,kBAAkB,GAAG,CAAC,CAAC;IAE9B,MAAM,CAAS;IACf,gBAAgB,CAAS;IACzB,WAAW,CAAS;IACX,QAAQ,GAAG,IAAI,GAAG,EAAc,CAAC;IAC1C,OAAO,GAAG,CAAC,CAAC;IAEZ,MAAM,KAAK,iBAAiB;QACxB,OAAO,IAAI,CAAC,kBAAkB,CAAC;IACnC,CAAC;IAED,4FAA4F;IAC5F,kGAAkG;IAClG,MAAM,CAAC,iBAAiB,CAAC,UAAmB;QACxC,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC,kBAAkB,IAAI,CAAC,CAAC;YAC7B,OAAO,IAAI,CAAC,kBAAkB,CAAC;QACnC,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;YACtD,MAAM,IAAI,SAAS,CAAC,mEAAmE,CAAC,CAAC;QAC7F,CAAC;QACD,IAAI,UAAU,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACvC,IAAI,CAAC,kBAAkB,GAAG,UAAU,CAAC;QACzC,CAAC;QACD,OAAO,IAAI,CAAC,kBAAkB,CAAC;IACnC,CAAC;IAED,MAAM,CAAC,WAAW,CACd,QAAgB,EAChB,KAAa,EACb,OAAmC;QAEnC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QACxB,MAAM,eAAe,GAAG,OAAO,EAAE,eAAe,IAAI,KAAK,CAAC;QAC1D,gBAAgB,CAAC,eAAe,CAAC,CAAC;QAClC,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,IAAI,CAAC,kBAAkB,CAAC;QAClE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;YACtD,MAAM,IAAI,SAAS,CAAC,mEAAmE,CAAC,CAAC;QAC7F,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1C,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YACzB,IAAI,UAAU,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;gBACpC,QAAQ,CAAC,WAAW,GAAG,UAAU,CAAC;gBAClC,QAAQ,CAAC,gBAAgB,GAAG,eAAe,CAAC;gBAC5C,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBACxB,OAAO,QAAQ,CAAC;YACpB,CAAC;YACD,IAAI,QAAQ,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;gBAC5B,IAAI,eAAe,KAAK,CAAC,CAAC;oBAAE,QAAQ,CAAC,gBAAgB,GAAG,eAAe,CAAC;gBACxE,OAAO,QAAQ,CAAC;YACpB,CAAC;YACD,yFAAyF;YACzF,IAAI,QAAQ,CAAC,gBAAgB,KAAK,eAAe,EAAE,CAAC;gBAChD,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBACxB,OAAO,QAAQ,CAAC;YACpB,CAAC;YACD,MAAM,IAAI,SAAS,CACf,iCAAiC,GAAG,6DAA6D,QAAQ,CAAC,MAAM,QAAQ,KAAK,kCAAkC,CAClK,CAAC;QACN,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,KAAK,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;QAC7E,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACpC,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,YAAoB,KAAa,EAAE,eAAuB,EAAE,UAAkB;QAC1E,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;QACxC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAClC,CAAC;IAED,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAED,IAAI,WAAW;QACX,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,IAAI,YAAY;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;IAC9B,CAAC;IAED,OAAO,CAAC,KAAa;QACjB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,WAAW,EAAE,CAAC;IACvB,CAAC;IAED,WAAW;QACP,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YAClF,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAM,CAAC;YAClD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC3B,IAAI,EAAE,CAAC;QACX,CAAC;IACL,CAAC;IAED,OAAO,CAAC,MAAoB;QACxB,MAAM,EAAE,cAAc,EAAE,CAAC;QACzB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,MAAM,GAAG,KAAK,CAAC;YACnB,MAAM,KAAK,GAAG,GAAS,EAAE;gBACrB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC5B,MAAM,CAAC,MAAO,CAAC,MAAM,CAAC,CAAC;YAC3B,CAAC,CAAC;YACF,MAAM,KAAK,GAAG,GAAS,EAAE;gBACrB,IAAI,MAAM;oBAAE,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACxD,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;gBAClB,IAAI,QAAQ,GAAG,KAAK,CAAC;gBACrB,OAAO,CAAC,GAAG,EAAE;oBACT,IAAI,QAAQ;wBAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;oBAC9E,QAAQ,GAAG,IAAI,CAAC;oBAChB,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;oBAClB,IAAI,CAAC,WAAW,EAAE,CAAC;gBACvB,CAAC,CAAC,CAAC;YACP,CAAC,CAAC;YACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM;gBAAE,KAAK,EAAE,CAAC;iBACzD,CAAC;gBACF,MAAM,GAAG,IAAI,CAAC;gBACd,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACzB,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YAC7D,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;CACJ","sourcesContent":["export interface InferenceAdmissionOptions {\n readonly configuredLimit?: number;\n readonly generation?: number;\n}\n\nconst assertValidLimit = (limit: number): void => {\n if (!Number.isSafeInteger(limit) || limit !== -1 && limit < 1) {\n throw new TypeError(\"PLURNK_PROVIDERS_MAX_CONCURRENCY must be -1 or a positive safe integer\");\n }\n};\n\n// {§provider-inference-admission} One process-local FIFO allowance per canonical endpoint.\nexport default class InferenceAdmission {\n static readonly #endpoints = new Map<string, InferenceAdmission>();\n static #currentGeneration = 0;\n\n #limit: number;\n #configuredLimit: number;\n #generation: number;\n readonly #waiting = new Set<() => void>();\n #active = 0;\n\n static get currentGeneration(): number {\n return this.#currentGeneration;\n }\n\n // Advance the configuration generation monotonically on operator environment reload so live\n // endpoints resize in place while conflicting aliases within the same generation still fail fast.\n static advanceGeneration(generation?: number): number {\n if (generation === undefined) {\n this.#currentGeneration += 1;\n return this.#currentGeneration;\n }\n if (!Number.isSafeInteger(generation) || generation < 0) {\n throw new TypeError(\"InferenceAdmission generation must be a non-negative safe integer\");\n }\n if (generation > this.#currentGeneration) {\n this.#currentGeneration = generation;\n }\n return this.#currentGeneration;\n }\n\n static forEndpoint(\n endpoint: string,\n limit: number,\n options?: InferenceAdmissionOptions,\n ): InferenceAdmission {\n assertValidLimit(limit);\n const configuredLimit = options?.configuredLimit ?? limit;\n assertValidLimit(configuredLimit);\n const generation = options?.generation ?? this.#currentGeneration;\n if (!Number.isSafeInteger(generation) || generation < 0) {\n throw new TypeError(\"InferenceAdmission generation must be a non-negative safe integer\");\n }\n const key = new URL(endpoint).href.replace(/\\/+$/, \"\");\n const existing = this.#endpoints.get(key);\n if (existing !== undefined) {\n if (generation > existing.#generation) {\n existing.#generation = generation;\n existing.#configuredLimit = configuredLimit;\n existing.#resize(limit);\n return existing;\n }\n if (existing.#limit === limit) {\n if (configuredLimit !== -1) existing.#configuredLimit = configuredLimit;\n return existing;\n }\n // Same operator policy on a restarted endpoint whose probed physical slot count changed.\n if (existing.#configuredLimit === configuredLimit) {\n existing.#resize(limit);\n return existing;\n }\n throw new TypeError(\n `Provider aliases resolving to ${key} have conflicting PLURNK_PROVIDERS_MAX_CONCURRENCY values ${existing.#limit} and ${limit}; one endpoint has one allowance`,\n );\n }\n const admission = new InferenceAdmission(limit, configuredLimit, generation);\n this.#endpoints.set(key, admission);\n return admission;\n }\n\n private constructor(limit: number, configuredLimit: number, generation: number) {\n this.#limit = limit;\n this.#configuredLimit = configuredLimit;\n this.#generation = generation;\n }\n\n get limit(): number {\n return this.#limit;\n }\n\n get activeCount(): number {\n return this.#active;\n }\n\n get waitingCount(): number {\n return this.#waiting.size;\n }\n\n #resize(limit: number): void {\n this.#limit = limit;\n this.#drainQueue();\n }\n\n #drainQueue(): void {\n while (this.#waiting.size > 0 && (this.#limit === -1 || this.#active < this.#limit)) {\n const next = this.#waiting.values().next().value!;\n this.#waiting.delete(next);\n next();\n }\n }\n\n acquire(signal?: AbortSignal): Promise<() => void> {\n signal?.throwIfAborted();\n return new Promise((resolve, reject) => {\n let queued = false;\n const abort = (): void => {\n this.#waiting.delete(enter);\n reject(signal!.reason);\n };\n const enter = (): void => {\n if (queued) signal?.removeEventListener(\"abort\", abort);\n this.#active += 1;\n let released = false;\n resolve(() => {\n if (released) throw new Error(\"Inference admission lease was released twice\");\n released = true;\n this.#active -= 1;\n this.#drainQueue();\n });\n };\n if (this.#limit === -1 || this.#active < this.#limit) enter();\n else {\n queued = true;\n this.#waiting.add(enter);\n signal?.addEventListener(\"abort\", abort, { once: true });\n }\n });\n }\n}\n"]}
|
package/dist/ModelProvider.d.ts
CHANGED
|
@@ -11,6 +11,7 @@ export type CacheAffinity = {
|
|
|
11
11
|
readonly name: string;
|
|
12
12
|
};
|
|
13
13
|
export type CacheBreakpoints = Exclude<CacheWritePolicy, "off">;
|
|
14
|
+
export type PromptCacheStyle = "ephemeral" | "openai-explicit";
|
|
14
15
|
export type ModelProviderConfig = {
|
|
15
16
|
inferenceAdmission?: InferenceAdmission;
|
|
16
17
|
model: string;
|
|
@@ -42,6 +43,7 @@ export type ModelProviderConfig = {
|
|
|
42
43
|
source?: string;
|
|
43
44
|
cacheAffinity?: CacheAffinity;
|
|
44
45
|
cacheBreakpoints?: CacheBreakpoints;
|
|
46
|
+
promptCacheStyle?: PromptCacheStyle;
|
|
45
47
|
serviceTier?: string;
|
|
46
48
|
requestControls?: NativeRequestControls;
|
|
47
49
|
apiKeyRejectedMessage?: string;
|
|
@@ -96,6 +98,6 @@ export default class ModelProvider implements Provider {
|
|
|
96
98
|
get requiresOutputBudget(): boolean | undefined;
|
|
97
99
|
countPromptTokens(messages: readonly ChatMessage[], signal?: AbortSignal): Promise<PromptTokenMeasurement>;
|
|
98
100
|
assessRequestCapacity(messages: readonly ChatMessage[], maxOutputTokens?: number, signal?: AbortSignal): Promise<ProviderRequestCapacity>;
|
|
99
|
-
generate({ messages, sessionId, signal, maxOutputTokens, sampling, observeRequest, observeReasoning, callKind,
|
|
101
|
+
generate({ messages, sessionId, signal, maxOutputTokens, sampling, observeRequest, observeReasoning, callKind, userCachePrefixes }: ProviderGenerateArgs): Promise<ProviderResponse>;
|
|
100
102
|
}
|
|
101
103
|
//# sourceMappingURL=ModelProvider.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ModelProvider.d.ts","sourceRoot":"","sources":["../src/ModelProvider.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EACR,WAAW,EACX,MAAM,EACN,aAAa,EACb,sBAAsB,EACtB,QAAQ,EAER,sBAAsB,EACtB,oBAAoB,EACpB,aAAa,EAEb,uBAAuB,EAEvB,gBAAgB,EAChB,aAAa,EAChB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAE7D,OAAO,KAAK,EAAE,gBAAgB,EAAE,aAAa,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAC;AAMxF,OAAO,EAAiB,KAAK,oBAAoB,EAAE,KAAK,2BAA2B,EAAE,MAAM,qBAAqB,CAAC;AAOjH,OAAO,KAAK,kBAAkB,MAAM,yBAAyB,CAAC;AAM9D,MAAM,MAAM,aAAa,GAAG,OAAO,UAAU,CAAC,KAAK,CAAC;AAGpD,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,OAAO,GAAG,UAAU,CAAC;AAE3D,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,OAAO,CAAC;AAE3E,MAAM,MAAM,aAAa,GAAG;IAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAC1F,MAAM,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"ModelProvider.d.ts","sourceRoot":"","sources":["../src/ModelProvider.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EACR,WAAW,EACX,MAAM,EACN,aAAa,EACb,sBAAsB,EACtB,QAAQ,EAER,sBAAsB,EACtB,oBAAoB,EACpB,aAAa,EAEb,uBAAuB,EAEvB,gBAAgB,EAChB,aAAa,EAChB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAE7D,OAAO,KAAK,EAAE,gBAAgB,EAAE,aAAa,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAC;AAMxF,OAAO,EAAiB,KAAK,oBAAoB,EAAE,KAAK,2BAA2B,EAAE,MAAM,qBAAqB,CAAC;AAOjH,OAAO,KAAK,kBAAkB,MAAM,yBAAyB,CAAC;AAM9D,MAAM,MAAM,aAAa,GAAG,OAAO,UAAU,CAAC,KAAK,CAAC;AAGpD,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,OAAO,GAAG,UAAU,CAAC;AAE3D,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,OAAO,CAAC;AAE3E,MAAM,MAAM,aAAa,GAAG;IAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAC1F,MAAM,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;AAChE,MAAM,MAAM,gBAAgB,GAAG,WAAW,GAAG,iBAAiB,CAAC;AAE/D,MAAM,MAAM,mBAAmB,GAAG;IAC9B,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IACxC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,2BAA2B,KAAK,oBAAoB,CAAC;IAC9E,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,eAAe,CAAC,EAAE,WAAW,CAAC,aAAa,CAAC,CAAC;IAC7C,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,gBAAgB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAIrC,cAAc,CAAC,EAAE,YAAY,GAAG,kBAAkB,CAAC;IACnD,sBAAsB,CAAC,EAAE,OAAO,CAAC;IAKjC,yBAAyB,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;IACpD,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,gBAAgB,CAAC,EAAE,YAAY,GAAG,uBAAuB,CAAC;IAC1D,oBAAoB,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,GAAG,IAAI,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjG,sBAAsB,CAAC,EAAE,sBAAsB,CAAC;IAChD,iBAAiB,CAAC,EAAE,CAAC,QAAQ,EAAE,SAAS,WAAW,EAAE,EAAE,MAAM,CAAC,EAAE,WAAW,KAAK,sBAAsB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;IACzI,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,GAAG,SAAS,KAAK,YAAY,CAAC;IAClE,aAAa,CAAC,EAAE,sBAAsB,CAAC;IACvC,MAAM,CAAC,EAAE,MAAM,CAAC;IAGhB,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IAIpC,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,eAAe,CAAC,EAAE,qBAAqB,CAAC;IACxC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAI1B,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,eAAe,CAAC,EAAE,MAAM,CAAC;IAKzB,WAAW,CAAC,EAAE,MAAM,CAAC;IAIrB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAG/B,MAAM,EAAE,aAAa,CAAC;IAGtB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAE7B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAK1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IAKrB,aAAa,EAAE,MAAM,CAAC;IAItB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAQ1B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,OAAO,CAAC,EAAE,OAAO,CAAC;CACrB,CAAC;AAmCF,MAAM,WAAW,qBAAqB;IAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;CAC9B;AA+BD,MAAM,CAAC,OAAO,OAAO,aAAc,YAAW,QAAQ;;IAoDlD,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,2BAA2B,KAAK,oBAAoB,CAAC;IAMvF,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAE/C,YAAY,MAAM,EAAE,mBAAmB,EA6JtC;IAED,IAAI,aAAa,IAAI,MAAM,GAAG,IAAI,CAAgC;IAClE,IAAI,eAAe,IAAI,WAAW,CAAC,aAAa,CAAC,CAAkC;IACnF,IAAI,cAAc,IAAI,MAAM,GAAG,IAAI,CAAiC;IACpE,IAAI,eAAe,IAAI,MAAM,GAAG,IAAI,CAAkC;IACtE,IAAI,YAAY,IAAI,MAAM,GAAG,IAAI,CAA+B;IAChE,IAAI,WAAW,IAAI,MAAM,GAAG,IAAI,CAA8B;IAC9D,IAAI,eAAe,IAAI,MAAM,GAAG,IAAI,CAAkC;IACtE,IAAI,gBAAgB,IAAI,SAAS,MAAM,EAAE,CAAmC;IAC5E,IAAI,aAAa,IAAI,MAAM,GAAG,IAAI,CAMjC;IAED,IAAI,SAAS,IAAI,MAAM,GAAG,IAAI,CAM7B;IACD,IAAI,KAAK,IAAI,MAAM,CAAwB;IAC3C,IAAI,WAAW,IAAI,MAAM,GAAG,IAAI,CAAuD;IACvF,IAAI,IAAI,IAAI,MAAM,GAAG,SAAS,CAAwC;IAEtE,IAAI,WAAW,IAAI,MAAM,GAAG,SAAS,CAA8B;IAEnE,IAAI,oBAAoB,IAAI,OAAO,GAAG,SAAS,CAAuC;IAEhF,iBAAiB,CACnB,QAAQ,EAAE,SAAS,WAAW,EAAE,EAChC,MAAM,CAAC,EAAE,WAAW,GACrB,OAAO,CAAC,sBAAsB,CAAC,CAsDjC;IAEK,qBAAqB,CACvB,QAAQ,EAAE,SAAS,WAAW,EAAE,EAChC,eAAe,CAAC,EAAE,MAAM,EACxB,MAAM,CAAC,EAAE,WAAW,GACrB,OAAO,CAAC,uBAAuB,CAAC,CAoBlC;IA4BK,QAAQ,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,cAAc,EAAE,gBAAgB,EAAE,QAAQ,EAAE,iBAAiB,EAAE,EAAE,oBAAoB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA+azL;CAEJ"}
|
package/dist/ModelProvider.js
CHANGED
|
@@ -18,7 +18,7 @@ import { assessRequestCapacity, effectiveInputCapacity, effectiveInputWall, effe
|
|
|
18
18
|
import ModelRequestBody from "./ModelRequestBody.js";
|
|
19
19
|
import LeadingReasoning from "./LeadingReasoning.js";
|
|
20
20
|
import RequestCapture from "./RequestCapture.js";
|
|
21
|
-
import { executePiTransportStream } from "./pi-transport.js";
|
|
21
|
+
import { executePiTransportStream, formatCacheAffinityValue } from "./pi-transport.js";
|
|
22
22
|
import { executeWithRetry, raceAgainstDeadline } from "./retry-deadline.js";
|
|
23
23
|
import { RETRYABLE_MALFORMED_FINISH_REASONS, readFailureEvidence } from "./stream-forensics.js";
|
|
24
24
|
class ProviderRequestObserverError extends Error {
|
|
@@ -116,6 +116,7 @@ export default class ModelProvider {
|
|
|
116
116
|
#source;
|
|
117
117
|
#cacheAffinity;
|
|
118
118
|
#cacheBreakpoints;
|
|
119
|
+
#promptCacheStyle;
|
|
119
120
|
#serviceTier;
|
|
120
121
|
#resolvedControls;
|
|
121
122
|
#supportsSlotPinning;
|
|
@@ -213,7 +214,8 @@ export default class ModelProvider {
|
|
|
213
214
|
this.#source = config.source ?? "provider";
|
|
214
215
|
this.#cacheAffinity = config.cacheAffinity;
|
|
215
216
|
this.#cacheBreakpoints = config.cacheBreakpoints;
|
|
216
|
-
|
|
217
|
+
this.#promptCacheStyle = config.promptCacheStyle;
|
|
218
|
+
if (this.#providerModel === undefined && (this.#cacheBreakpoints !== undefined || this.#promptCacheStyle !== undefined)) {
|
|
217
219
|
throw new Error(`${this.#source}: prompt cache breakpoints require a model transport`);
|
|
218
220
|
}
|
|
219
221
|
if (this.#providerModel === undefined && this.#additiveReasoningProvider !== undefined) {
|
|
@@ -400,7 +402,7 @@ export default class ModelProvider {
|
|
|
400
402
|
cost: resolveProviderCost(direct, this.#estimateCost(knownUsage)),
|
|
401
403
|
});
|
|
402
404
|
}
|
|
403
|
-
async generate({ messages, sessionId, signal, maxOutputTokens, sampling, observeRequest, observeReasoning, callKind,
|
|
405
|
+
async generate({ messages, sessionId, signal, maxOutputTokens, sampling, observeRequest, observeReasoning, callKind, userCachePrefixes }) {
|
|
404
406
|
// {§provider-interface} The session identity is required.
|
|
405
407
|
if (sessionId === undefined || sessionId.length === 0)
|
|
406
408
|
throw new Error("generate: sessionId is required — the session's stable, opaque identity");
|
|
@@ -439,7 +441,7 @@ export default class ModelProvider {
|
|
|
439
441
|
...(this.#topLogprobs !== null ? { logprobs: true, top_logprobs: this.#topLogprobs } : {}),
|
|
440
442
|
...this.#requestBody.slotBody(sessionId),
|
|
441
443
|
...(this.#cacheAffinity?.target === "body"
|
|
442
|
-
? { [this.#cacheAffinity.name]: sessionId }
|
|
444
|
+
? { [this.#cacheAffinity.name]: formatCacheAffinityValue(this.#cacheAffinity, sessionId) }
|
|
443
445
|
: {}),
|
|
444
446
|
};
|
|
445
447
|
for (const field of ["tools", "tool_choice", "functions", "function_call", "parallel_tool_calls"]) {
|
|
@@ -575,7 +577,8 @@ export default class ModelProvider {
|
|
|
575
577
|
nativeAdaptiveThinking: this.#nativeAdaptiveThinking,
|
|
576
578
|
reasoningRequestBody: this.#reasoningRequestBody,
|
|
577
579
|
cacheBreakpoints: callKind === "bare" ? undefined : this.#cacheBreakpoints,
|
|
578
|
-
|
|
580
|
+
promptCacheStyle: this.#promptCacheStyle,
|
|
581
|
+
userCachePrefixes,
|
|
579
582
|
cacheAffinity: this.#cacheAffinity,
|
|
580
583
|
resolvedControls: this.#resolvedControls,
|
|
581
584
|
rawBody: this.#rawBody,
|