@plurnk/plurnk-providers 6.1.1 → 6.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/.env.defaults CHANGED
@@ -16,8 +16,9 @@
16
16
  # PLURNK_BASEURL_<alias>=<url>
17
17
 
18
18
  # --- Capacity ---
19
- # Simultaneous inference attempts per resolved endpoint in this process; -1 = unrestricted.
20
- # Positive limits queue excess calls. Aliases sharing an endpoint must agree on its limit.
19
+ # Simultaneous inference attempts per resolved endpoint in this process; -1 = unrestricted, or
20
+ # bounded automatically by probed physical slot capacity (e.g. llama-server GET /props total_slots).
21
+ # Positive limits queue excess calls and are capped by probed physical slots.
21
22
  PLURNK_PROVIDERS_MAX_CONCURRENCY=-1
22
23
  # PLURNK_PROVIDERS_MAX_CONCURRENCY_local=1
23
24
  # Reserved output INCLUDING reasoning: positive tokens or context percentage in (0,100).
@@ -77,9 +78,9 @@ PLURNK_PROVIDERS_DRY_MULTIPLIER=0
77
78
  # --- Cache and billing ---
78
79
  # Stable Session affinity on supported routes: 1 = on; 0 = off. Unknown routes get no guessed field.
79
80
  PLURNK_PROVIDERS_CACHE_AFFINITY=1
80
- # Explicit cache writes on explicit-breakpoint routes (Anthropic, OpenRouter Anthropic, Bedrock Claude):
81
+ # Explicit cache writes on explicit-breakpoint routes (Anthropic, MiniMax Anthropic, OpenRouter Anthropic, Bedrock Claude, OpenAI GPT-5.6+):
81
82
  # conversation = system + rolling conversation/log prefix before volatile turn status;
82
- # stable-system = reusable system boundary only; off = no explicit writes. Distinct from affinity and affects billing.
83
+ # stable-system = reusable system boundary only; off = no explicit writes (suppresses paid implicit writes on OpenAI GPT-5.6+). Distinct from affinity and affects billing.
83
84
  PLURNK_PROVIDERS_CACHE_WRITE_POLICY=conversation
84
85
  # Provider service tier; unset = provider default. Usually set per alias: routing can change cost.
85
86
  # Native routes send it only where the SDK documents one ({§provider-request-authority}); others warn.
package/README.md CHANGED
@@ -58,7 +58,7 @@ In-package `openai` and `ollama` adapters probe live context windows, `llama-ser
58
58
  | --- | --- |
59
59
  | Effort & reasoning budget | `adaptive` requests dynamic reasoning or falls back to `PLURNK_PROVIDERS_EFFORT_FALLBACK` (`high`); explicit fixed efforts must be route-supported. Unset reasoning budget leaves reasoning enabled. |
60
60
  | Output & floor (`PLURNK_PROVIDERS_OUTPUT_FLOOR`) | Shipped `25%` floor sets the input wall (`contextWindow - floor`); packets past the output budget flex output down to the remainder ({§provider-capacity-admission}). |
61
- | Concurrency & deadlines | Optional per-endpoint `MAX_CONCURRENCY` ({§provider-inference-admission}); one whole-call generation deadline across queueing and `Retry-After` retries. |
61
+ | Concurrency & deadlines | Per-endpoint FIFO `MAX_CONCURRENCY` gate ({§provider-inference-admission}), automatically bounded by probed `llama-server` `total_slots` and resized live across `.env` reloads; one whole-call generation deadline across queueing and `Retry-After` retries. |
62
62
  | Provider extensions | First-party `@plurnk/*` packages with `plurnk: { "kind": "provider", "name": "..." }` default-export `ProviderExtension` (`model(modelId)`, optional `attributions(context)`, {§extension-discovery}). |
63
63
 
64
64
  ## Live conformance & development
package/SPEC.md CHANGED
@@ -59,10 +59,10 @@ Every physical attempt settles a `ProviderRequestAccounting` record (`{ provider
59
59
  `ModelProvider.ts` delegates streaming, protocol framing, vendor request assembly, model compatibility flags (`getBuiltinModels`), header normalization (`headersToRecord`), interruptible backoff (`sleep`), token estimation (`estimateMessageTokens`), and context-overflow pattern matching (`getOverflowPatterns`) to `@earendil-works/pi-ai` across all supported APIs (`anthropic-messages`, `openai-completions`, `openai-responses`, `google-generative-ai`, `bedrock-converse-stream`):
60
60
 
61
61
  - Prompt cache affinity (`PLURNK_PROVIDERS_CACHE_AFFINITY=1`) projects each provider's documented session key (`sessionId` -> `prompt_cache_key` / `x-session-id` / `x-session-affinity`).
62
- - Explicit prompt-cache write policy (`PLURNK_PROVIDERS_CACHE_WRITE_POLICY`, default `conversation`: `conversation | stable-system | off`) governs explicit cache breakpoints (`cache_control: { type: "ephemeral" }` on Anthropic and OpenRouter Anthropic routes; `cachePoint: { type: "default" }` on Bedrock Claude prompt-caching routes), suppressed on stateless `callKind: "bare"` calls:
63
- - `off`: zero explicit cache breakpoints.
62
+ - Explicit prompt-cache write policy (`PLURNK_PROVIDERS_CACHE_WRITE_POLICY`, default `conversation`: `conversation | stable-system | off`) governs explicit cache breakpoints (`cache_control: { type: "ephemeral" }` on Anthropic, MiniMax Anthropic, and OpenRouter Anthropic routes; `cachePoint: { type: "default" }` on Bedrock Claude prompt-caching routes; `prompt_cache_options: { mode: "explicit" }` with `prompt_cache_breakpoint: { mode: "explicit" }` on OpenAI GPT-5.6+ routes, including OpenRouter `openai/*`), suppressed on stateless `callKind: "bare"` calls:
63
+ - `off` (and `callKind: "bare"`): zero explicit cache breakpoints; on OpenAI GPT-5.6+ routes `prompt_cache_options: { mode: "explicit" }` is still sent with zero breakpoints so implicit paid cache writes are disabled.
64
64
  - `stable-system`: one breakpoint on the final leading `system` instruction.
65
- - `conversation`: breakpoints on the final leading `system` instruction, the penultimate conversation message, and the caller-delimited durable prefix (`finalUserCachePrefix`) of the final `user` message before any volatile per-turn status tail (or the full final `user` message when no volatile tail is delimited), staying within three breakpoints per request.
65
+ - `conversation`: splits caller-delimited `userCachePrefixes` on historical `user` messages into separate content blocks so prior turns' cached durable log blocks retain their exact block boundaries. Ephemeral-breakpoint routes (`cache_control` / `cachePoint`) place breakpoints on the final leading `system` instruction, the penultimate conversation message, and the durable prefix of the final `user` message before any volatile per-turn status tail (or the full final `user` message when no volatile tail is delimited), staying within three breakpoints per request; OpenAI GPT-5.6+ explicit routes keep monotonic `prompt_cache_breakpoint` markers on the final leading `system` instruction and each `user` message's durable prefix across turns so prior segment boundaries remain stable while writing only new uncached prefixes.
66
66
  - Output token allowance (`responseMax ?? outputBudget`) is enforced both in flight during streaming (aborting a runaway stream once mid-stream reported output tokens or conservative token lower-bound `floor(estimateMessageTokens(partial) / 2)` exceeds the granted allowance) and on final reported `usage.outputTokens`.
67
67
 
68
68
  ## §provider-effort Reasoning effort and readable trace
@@ -84,7 +84,7 @@ Every physical attempt settles a `ProviderRequestAccounting` record (`{ provider
84
84
  | `PLURNK_PROVIDERS_EFFORT` / `EFFORT_FALLBACK` | `adaptive` / `high` | Portable reasoning effort selector and graded-route fallback |
85
85
  | `PLURNK_PROVIDERS_FETCH_TIMEOUT` / `OPERATION_TIMEOUT` | `30000` / `600000` ms | Per-attempt fetch/idle timeout and total operation wall-clock timeout |
86
86
  | `PLURNK_PROVIDERS_RETRY_ATTEMPTS` | `3` | Maximum automatic retry attempts for retryable transport failures |
87
- | `PLURNK_PROVIDERS_MAX_CONCURRENCY` | `-1` (unbounded or positive integer) | Per-endpoint FIFO concurrency gate ({§provider-inference-admission}) |
87
+ | `PLURNK_PROVIDERS_MAX_CONCURRENCY` | `-1` (unbounded when unprobed, or positive integer) | Per-endpoint FIFO concurrency gate, automatically bounded by probed physical endpoint slots ({§provider-inference-admission}) |
88
88
  | `PLURNK_PROVIDERS_COST_<CATEGORY>` | empty (non-negative USD per 1M tokens for `INPUT`, `OUTPUT`, `REASONING`, `CACHE_READ`, `CACHE_WRITE`) | Per-category operator rate overrides merged over catalog rates ({§provider-monetary-evidence}) |
89
89
  | `PLURNK_PROVIDERS_RAWBODY` | `0` (`0 \| 1`) | Retains redacted raw request/response forensic bodies in `forensicEvidence` |
90
90
 
@@ -97,7 +97,7 @@ Every physical attempt settles a `ProviderRequestAccounting` record (`{ provider
97
97
  | Route family | Fact authority |
98
98
  |---|---|
99
99
  | Cataloged cloud provider (`catalogProvider.ts`) | `resolveModel(provider, model)` from `@plurnk/plurnk-models`, capped by optional `PLURNK_PROVIDERS_CONTEXT_WINDOW`. Uncataloged models on a catalog provider require an explicit `PLURNK_PROVIDERS_CONTEXT_WINDOW`. Catalog credential names (`info.env`) or `PLURNK_PROVIDERS_PROVIDER_<ID>_API_KEY_ENV` (one exact variable name, never a comma list) supply authentication. |
100
- | Local / compatible endpoint (`compatibleProvider.ts`) | Probes `GET <baseUrl>/models` for `n_ctx` / `max_model_len` / `context_length`. When llama-server is detected (`meta.n_ctx` or `PLURNK_PROVIDERS_LLAMA_SERVER=1`), probes `/props` for `total_slots` and `eos_token` and measures prompts via `/v1/chat/completions/input_tokens`. |
100
+ | Local / compatible endpoint (`compatibleProvider.ts`) | Probes `GET <baseUrl>/models` for `n_ctx` / `max_model_len` / `context_length`. When llama-server is detected (`meta.n_ctx` or `PLURNK_PROVIDERS_LLAMA_SERVER=1`), probes `/props` for `total_slots` (bounding endpoint inference concurrency via {§provider-inference-admission}) and `eos_token` and measures prompts via `/v1/chat/completions/input_tokens`. |
101
101
  | Built-in Ollama rail (`ollama.ts`) | Probes `POST <origin>/api/show` for `*.context_length`, capped by optional `PLURNK_PROVIDERS_CONTEXT_WINDOW`, and dispatches chat completions to `<origin>/v1/chat/completions`. |
102
102
  | Operator probe scripts (`scripts/`) | `providers-ping.mjs` probes one cheapest declared route per keyed provider and writes redacted JSON evidence; `token-envelope-probe.mjs` probes explicit aliases across configured or physical output limits. |
103
103
 
@@ -132,8 +132,10 @@ All transport and response failures normalize into `ProviderError` carrying an R
132
132
 
133
133
  ## §provider-inference-admission Inference admission
134
134
 
135
- `InferenceAdmission.forEndpoint(endpoint, maxConcurrency)` coordinates in-flight generations across all `Provider` instances targeting the same canonical endpoint URL (`PLURNK_PROVIDERS_MAX_CONCURRENCY`, default `-1` unbounded, or a positive safe integer):
135
+ `InferenceAdmission.forEndpoint(endpoint, maxConcurrency, options?)` coordinates in-flight generations across all `Provider` instances targeting the same canonical endpoint URL (`PLURNK_PROVIDERS_MAX_CONCURRENCY`, default `-1` unbounded, or a positive safe integer):
136
136
 
137
+ - `effectiveMaxConcurrency(configuredLimit, physicalSlots)` reconciles operator policy with probed physical endpoint slot capacity (`GET /props` `total_slots` on llama-server routes, `null` when unprobed): `-1` adopts `physicalSlots` when probed (remaining `-1` when `null`), and a positive `configuredLimit` resolves to `min(configuredLimit, physicalSlots)` when probed.
137
138
  - Waiters acquire slots in strict FIFO order before opening a physical request (`observeRequest` does not fire while queued) and hold the slot until the response stream or error settles.
138
- - Aborted waiters are removed from the queue immediately without consuming capacity; retry backoff releases the slot before sleeping and re-acquires it for the next attempt.
139
- - Conflicting positive/unbounded `PLURNK_PROVIDERS_MAX_CONCURRENCY` declarations for the same canonical endpoint throw at construction.
139
+ - Aborted waiters are removed from the queue immediately in $O(1)$ without consuming capacity or retaining signal listeners; retry backoff releases the slot before sleeping and re-acquires it for the next attempt.
140
+ - Within one configuration generation, conflicting effective allowances across aliases targeting the same canonical endpoint throw at construction; aliases that resolve to the same effective allowance (such as `-1` alongside explicit `1` on a 1-slot llama-server) share the gate.
141
+ - Advancing the configuration generation (`InferenceAdmission.advanceGeneration`) on operator `.env` reload or observing a changed physical `total_slots` under unchanged operator policy resizes the existing endpoint semaphore in place: growing immediately wakes queued FIFO waiters up to the new allowance, and shrinking lets surplus in-flight leases finish naturally before admitting queued work.
@@ -1,7 +1,16 @@
1
+ export interface InferenceAdmissionOptions {
2
+ readonly configuredLimit?: number;
3
+ readonly generation?: number;
4
+ }
1
5
  export default class InferenceAdmission {
2
6
  #private;
3
- static forEndpoint(endpoint: string, limit: number): InferenceAdmission;
7
+ static get currentGeneration(): number;
8
+ static advanceGeneration(generation?: number): number;
9
+ static forEndpoint(endpoint: string, limit: number, options?: InferenceAdmissionOptions): InferenceAdmission;
4
10
  private constructor();
11
+ get limit(): number;
12
+ get activeCount(): number;
13
+ get waitingCount(): number;
5
14
  acquire(signal?: AbortSignal): Promise<() => void>;
6
15
  }
7
16
  //# sourceMappingURL=InferenceAdmission.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"InferenceAdmission.d.ts","sourceRoot":"","sources":["../src/InferenceAdmission.ts"],"names":[],"mappings":"AACA,MAAM,CAAC,OAAO,OAAO,kBAAkB;;IAMnC,MAAM,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,kBAAkB,CAetE;IAED,OAAO,eAEN;IAED,OAAO,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CA4BjD;CACJ"}
1
+ {"version":3,"file":"InferenceAdmission.d.ts","sourceRoot":"","sources":["../src/InferenceAdmission.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,yBAAyB;IACtC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAChC;AASD,MAAM,CAAC,OAAO,OAAO,kBAAkB;;IAUnC,MAAM,KAAK,iBAAiB,IAAI,MAAM,CAErC;IAID,MAAM,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,CAYpD;IAED,MAAM,CAAC,WAAW,CACd,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,yBAAyB,GACpC,kBAAkB,CAiCpB;IAED,OAAO,eAIN;IAED,IAAI,KAAK,IAAI,MAAM,CAElB;IAED,IAAI,WAAW,IAAI,MAAM,CAExB;IAED,IAAI,YAAY,IAAI,MAAM,CAEzB;IAeD,OAAO,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CA0BjD;CACJ"}
@@ -1,37 +1,104 @@
1
- // {§provider-inference-admission} One process-local allowance per resolved endpoint.
1
+ const assertValidLimit = (limit) => {
2
+ if (!Number.isSafeInteger(limit) || limit !== -1 && limit < 1) {
3
+ throw new TypeError("PLURNK_PROVIDERS_MAX_CONCURRENCY must be -1 or a positive safe integer");
4
+ }
5
+ };
6
+ // {§provider-inference-admission} One process-local FIFO allowance per canonical endpoint.
2
7
  export default class InferenceAdmission {
3
8
  static #endpoints = new Map();
9
+ static #currentGeneration = 0;
4
10
  #limit;
11
+ #configuredLimit;
12
+ #generation;
5
13
  #waiting = new Set();
6
14
  #active = 0;
7
- static forEndpoint(endpoint, limit) {
8
- if (!Number.isSafeInteger(limit) || limit !== -1 && limit < 1) {
9
- throw new TypeError("PLURNK_PROVIDERS_MAX_CONCURRENCY must be -1 or a positive safe integer");
15
+ static get currentGeneration() {
16
+ return this.#currentGeneration;
17
+ }
18
+ // Advance the configuration generation monotonically on operator environment reload so live
19
+ // endpoints resize in place while conflicting aliases within the same generation still fail fast.
20
+ static advanceGeneration(generation) {
21
+ if (generation === undefined) {
22
+ this.#currentGeneration += 1;
23
+ return this.#currentGeneration;
24
+ }
25
+ if (!Number.isSafeInteger(generation) || generation < 0) {
26
+ throw new TypeError("InferenceAdmission generation must be a non-negative safe integer");
27
+ }
28
+ if (generation > this.#currentGeneration) {
29
+ this.#currentGeneration = generation;
30
+ }
31
+ return this.#currentGeneration;
32
+ }
33
+ static forEndpoint(endpoint, limit, options) {
34
+ assertValidLimit(limit);
35
+ const configuredLimit = options?.configuredLimit ?? limit;
36
+ assertValidLimit(configuredLimit);
37
+ const generation = options?.generation ?? this.#currentGeneration;
38
+ if (!Number.isSafeInteger(generation) || generation < 0) {
39
+ throw new TypeError("InferenceAdmission generation must be a non-negative safe integer");
10
40
  }
11
41
  const key = new URL(endpoint).href.replace(/\/+$/, "");
12
42
  const existing = this.#endpoints.get(key);
13
43
  if (existing !== undefined) {
14
- if (existing.#limit !== limit) {
15
- throw new TypeError(`Provider aliases resolving to ${key} have conflicting PLURNK_PROVIDERS_MAX_CONCURRENCY values ${existing.#limit} and ${limit}; one endpoint has one allowance`);
44
+ if (generation > existing.#generation) {
45
+ existing.#generation = generation;
46
+ existing.#configuredLimit = configuredLimit;
47
+ existing.#resize(limit);
48
+ return existing;
49
+ }
50
+ if (existing.#limit === limit) {
51
+ if (configuredLimit !== -1)
52
+ existing.#configuredLimit = configuredLimit;
53
+ return existing;
54
+ }
55
+ // Same operator policy on a restarted endpoint whose probed physical slot count changed.
56
+ if (existing.#configuredLimit === configuredLimit) {
57
+ existing.#resize(limit);
58
+ return existing;
16
59
  }
17
- return existing;
60
+ throw new TypeError(`Provider aliases resolving to ${key} have conflicting PLURNK_PROVIDERS_MAX_CONCURRENCY values ${existing.#limit} and ${limit}; one endpoint has one allowance`);
18
61
  }
19
- const admission = new InferenceAdmission(limit);
62
+ const admission = new InferenceAdmission(limit, configuredLimit, generation);
20
63
  this.#endpoints.set(key, admission);
21
64
  return admission;
22
65
  }
23
- constructor(limit) {
66
+ constructor(limit, configuredLimit, generation) {
24
67
  this.#limit = limit;
68
+ this.#configuredLimit = configuredLimit;
69
+ this.#generation = generation;
70
+ }
71
+ get limit() {
72
+ return this.#limit;
73
+ }
74
+ get activeCount() {
75
+ return this.#active;
76
+ }
77
+ get waitingCount() {
78
+ return this.#waiting.size;
79
+ }
80
+ #resize(limit) {
81
+ this.#limit = limit;
82
+ this.#drainQueue();
83
+ }
84
+ #drainQueue() {
85
+ while (this.#waiting.size > 0 && (this.#limit === -1 || this.#active < this.#limit)) {
86
+ const next = this.#waiting.values().next().value;
87
+ this.#waiting.delete(next);
88
+ next();
89
+ }
25
90
  }
26
91
  acquire(signal) {
27
92
  signal?.throwIfAborted();
28
93
  return new Promise((resolve, reject) => {
94
+ let queued = false;
29
95
  const abort = () => {
30
96
  this.#waiting.delete(enter);
31
97
  reject(signal.reason);
32
98
  };
33
99
  const enter = () => {
34
- signal?.removeEventListener("abort", abort);
100
+ if (queued)
101
+ signal?.removeEventListener("abort", abort);
35
102
  this.#active += 1;
36
103
  let released = false;
37
104
  resolve(() => {
@@ -39,16 +106,13 @@ export default class InferenceAdmission {
39
106
  throw new Error("Inference admission lease was released twice");
40
107
  released = true;
41
108
  this.#active -= 1;
42
- const next = this.#waiting.values().next().value;
43
- if (next !== undefined) {
44
- this.#waiting.delete(next);
45
- next();
46
- }
109
+ this.#drainQueue();
47
110
  });
48
111
  };
49
112
  if (this.#limit === -1 || this.#active < this.#limit)
50
113
  enter();
51
114
  else {
115
+ queued = true;
52
116
  this.#waiting.add(enter);
53
117
  signal?.addEventListener("abort", abort, { once: true });
54
118
  }
@@ -1 +1 @@
1
- {"version":3,"file":"InferenceAdmission.js","sourceRoot":"","sources":["../src/InferenceAdmission.ts"],"names":[],"mappings":"AAAA,qFAAqF;AACrF,MAAM,CAAC,OAAO,OAAO,kBAAkB;IACnC,MAAM,CAAU,UAAU,GAAG,IAAI,GAAG,EAA8B,CAAC;IAC1D,MAAM,CAAS;IACf,QAAQ,GAAG,IAAI,GAAG,EAAc,CAAC;IAC1C,OAAO,GAAG,CAAC,CAAC;IAEZ,MAAM,CAAC,WAAW,CAAC,QAAgB,EAAE,KAAa;QAC9C,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,SAAS,CAAC,wEAAwE,CAAC,CAAC;QAClG,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1C,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YACzB,IAAI,QAAQ,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;gBAC5B,MAAM,IAAI,SAAS,CAAC,iCAAiC,GAAG,6DAA6D,QAAQ,CAAC,MAAM,QAAQ,KAAK,kCAAkC,CAAC,CAAC;YACzL,CAAC;YACD,OAAO,QAAQ,CAAC;QACpB,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACpC,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,YAAoB,KAAa;QAC7B,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACxB,CAAC;IAED,OAAO,CAAC,MAAoB;QACxB,MAAM,EAAE,cAAc,EAAE,CAAC;QACzB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,KAAK,GAAG,GAAS,EAAE;gBACrB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC5B,MAAM,CAAC,MAAO,CAAC,MAAM,CAAC,CAAC;YAC3B,CAAC,CAAC;YACF,MAAM,KAAK,GAAG,GAAS,EAAE;gBACrB,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBAC5C,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;gBAClB,IAAI,QAAQ,GAAG,KAAK,CAAC;gBACrB,OAAO,CAAC,GAAG,EAAE;oBACT,IAAI,QAAQ;wBAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;oBAC9E,QAAQ,GAAG,IAAI,CAAC;oBAChB,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;oBAClB,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;oBACjD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;wBACrB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;wBAC3B,IAAI,EAAE,CAAC;oBACX,CAAC;gBACL,CAAC,CAAC,CAAC;YACP,CAAC,CAAC;YACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM;gBAAE,KAAK,EAAE,CAAC;iBACzD,CAAC;gBACF,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACzB,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YAC7D,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;CACJ","sourcesContent":["// {§provider-inference-admission} One process-local allowance per resolved endpoint.\nexport default class InferenceAdmission {\n static readonly #endpoints = new Map<string, InferenceAdmission>();\n readonly #limit: number;\n readonly #waiting = new Set<() => void>();\n #active = 0;\n\n static forEndpoint(endpoint: string, limit: number): InferenceAdmission {\n if (!Number.isSafeInteger(limit) || limit !== -1 && limit < 1) {\n throw new TypeError(\"PLURNK_PROVIDERS_MAX_CONCURRENCY must be -1 or a positive safe integer\");\n }\n const key = new URL(endpoint).href.replace(/\\/+$/, \"\");\n const existing = this.#endpoints.get(key);\n if (existing !== undefined) {\n if (existing.#limit !== limit) {\n throw new TypeError(`Provider aliases resolving to ${key} have conflicting PLURNK_PROVIDERS_MAX_CONCURRENCY values ${existing.#limit} and ${limit}; one endpoint has one allowance`);\n }\n return existing;\n }\n const admission = new InferenceAdmission(limit);\n this.#endpoints.set(key, admission);\n return admission;\n }\n\n private constructor(limit: number) {\n this.#limit = limit;\n }\n\n acquire(signal?: AbortSignal): Promise<() => void> {\n signal?.throwIfAborted();\n return new Promise((resolve, reject) => {\n const abort = (): void => {\n this.#waiting.delete(enter);\n reject(signal!.reason);\n };\n const enter = (): void => {\n signal?.removeEventListener(\"abort\", abort);\n this.#active += 1;\n let released = false;\n resolve(() => {\n if (released) throw new Error(\"Inference admission lease was released twice\");\n released = true;\n this.#active -= 1;\n const next = this.#waiting.values().next().value;\n if (next !== undefined) {\n this.#waiting.delete(next);\n next();\n }\n });\n };\n if (this.#limit === -1 || this.#active < this.#limit) enter();\n else {\n this.#waiting.add(enter);\n signal?.addEventListener(\"abort\", abort, { once: true });\n }\n });\n }\n}\n"]}
1
+ {"version":3,"file":"InferenceAdmission.js","sourceRoot":"","sources":["../src/InferenceAdmission.ts"],"names":[],"mappings":"AAKA,MAAM,gBAAgB,GAAG,CAAC,KAAa,EAAQ,EAAE;IAC7C,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QAC5D,MAAM,IAAI,SAAS,CAAC,wEAAwE,CAAC,CAAC;IAClG,CAAC;AACL,CAAC,CAAC;AAEF,2FAA2F;AAC3F,MAAM,CAAC,OAAO,OAAO,kBAAkB;IACnC,MAAM,CAAU,UAAU,GAAG,IAAI,GAAG,EAA8B,CAAC;IACnE,MAAM,CAAC,kBAAkB,GAAG,CAAC,CAAC;IAE9B,MAAM,CAAS;IACf,gBAAgB,CAAS;IACzB,WAAW,CAAS;IACX,QAAQ,GAAG,IAAI,GAAG,EAAc,CAAC;IAC1C,OAAO,GAAG,CAAC,CAAC;IAEZ,MAAM,KAAK,iBAAiB;QACxB,OAAO,IAAI,CAAC,kBAAkB,CAAC;IACnC,CAAC;IAED,4FAA4F;IAC5F,kGAAkG;IAClG,MAAM,CAAC,iBAAiB,CAAC,UAAmB;QACxC,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC,kBAAkB,IAAI,CAAC,CAAC;YAC7B,OAAO,IAAI,CAAC,kBAAkB,CAAC;QACnC,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;YACtD,MAAM,IAAI,SAAS,CAAC,mEAAmE,CAAC,CAAC;QAC7F,CAAC;QACD,IAAI,UAAU,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACvC,IAAI,CAAC,kBAAkB,GAAG,UAAU,CAAC;QACzC,CAAC;QACD,OAAO,IAAI,CAAC,kBAAkB,CAAC;IACnC,CAAC;IAED,MAAM,CAAC,WAAW,CACd,QAAgB,EAChB,KAAa,EACb,OAAmC;QAEnC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QACxB,MAAM,eAAe,GAAG,OAAO,EAAE,eAAe,IAAI,KAAK,CAAC;QAC1D,gBAAgB,CAAC,eAAe,CAAC,CAAC;QAClC,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,IAAI,CAAC,kBAAkB,CAAC;QAClE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;YACtD,MAAM,IAAI,SAAS,CAAC,mEAAmE,CAAC,CAAC;QAC7F,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1C,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YACzB,IAAI,UAAU,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;gBACpC,QAAQ,CAAC,WAAW,GAAG,UAAU,CAAC;gBAClC,QAAQ,CAAC,gBAAgB,GAAG,eAAe,CAAC;gBAC5C,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBACxB,OAAO,QAAQ,CAAC;YACpB,CAAC;YACD,IAAI,QAAQ,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;gBAC5B,IAAI,eAAe,KAAK,CAAC,CAAC;oBAAE,QAAQ,CAAC,gBAAgB,GAAG,eAAe,CAAC;gBACxE,OAAO,QAAQ,CAAC;YACpB,CAAC;YACD,yFAAyF;YACzF,IAAI,QAAQ,CAAC,gBAAgB,KAAK,eAAe,EAAE,CAAC;gBAChD,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBACxB,OAAO,QAAQ,CAAC;YACpB,CAAC;YACD,MAAM,IAAI,SAAS,CACf,iCAAiC,GAAG,6DAA6D,QAAQ,CAAC,MAAM,QAAQ,KAAK,kCAAkC,CAClK,CAAC;QACN,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,KAAK,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;QAC7E,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACpC,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,YAAoB,KAAa,EAAE,eAAuB,EAAE,UAAkB;QAC1E,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;QACxC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAClC,CAAC;IAED,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAED,IAAI,WAAW;QACX,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,IAAI,YAAY;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;IAC9B,CAAC;IAED,OAAO,CAAC,KAAa;QACjB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,WAAW,EAAE,CAAC;IACvB,CAAC;IAED,WAAW;QACP,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YAClF,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAM,CAAC;YAClD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC3B,IAAI,EAAE,CAAC;QACX,CAAC;IACL,CAAC;IAED,OAAO,CAAC,MAAoB;QACxB,MAAM,EAAE,cAAc,EAAE,CAAC;QACzB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,MAAM,GAAG,KAAK,CAAC;YACnB,MAAM,KAAK,GAAG,GAAS,EAAE;gBACrB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC5B,MAAM,CAAC,MAAO,CAAC,MAAM,CAAC,CAAC;YAC3B,CAAC,CAAC;YACF,MAAM,KAAK,GAAG,GAAS,EAAE;gBACrB,IAAI,MAAM;oBAAE,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACxD,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;gBAClB,IAAI,QAAQ,GAAG,KAAK,CAAC;gBACrB,OAAO,CAAC,GAAG,EAAE;oBACT,IAAI,QAAQ;wBAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;oBAC9E,QAAQ,GAAG,IAAI,CAAC;oBAChB,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;oBAClB,IAAI,CAAC,WAAW,EAAE,CAAC;gBACvB,CAAC,CAAC,CAAC;YACP,CAAC,CAAC;YACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM;gBAAE,KAAK,EAAE,CAAC;iBACzD,CAAC;gBACF,MAAM,GAAG,IAAI,CAAC;gBACd,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACzB,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YAC7D,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;CACJ","sourcesContent":["export interface InferenceAdmissionOptions {\n readonly configuredLimit?: number;\n readonly generation?: number;\n}\n\nconst assertValidLimit = (limit: number): void => {\n if (!Number.isSafeInteger(limit) || limit !== -1 && limit < 1) {\n throw new TypeError(\"PLURNK_PROVIDERS_MAX_CONCURRENCY must be -1 or a positive safe integer\");\n }\n};\n\n// {§provider-inference-admission} One process-local FIFO allowance per canonical endpoint.\nexport default class InferenceAdmission {\n static readonly #endpoints = new Map<string, InferenceAdmission>();\n static #currentGeneration = 0;\n\n #limit: number;\n #configuredLimit: number;\n #generation: number;\n readonly #waiting = new Set<() => void>();\n #active = 0;\n\n static get currentGeneration(): number {\n return this.#currentGeneration;\n }\n\n // Advance the configuration generation monotonically on operator environment reload so live\n // endpoints resize in place while conflicting aliases within the same generation still fail fast.\n static advanceGeneration(generation?: number): number {\n if (generation === undefined) {\n this.#currentGeneration += 1;\n return this.#currentGeneration;\n }\n if (!Number.isSafeInteger(generation) || generation < 0) {\n throw new TypeError(\"InferenceAdmission generation must be a non-negative safe integer\");\n }\n if (generation > this.#currentGeneration) {\n this.#currentGeneration = generation;\n }\n return this.#currentGeneration;\n }\n\n static forEndpoint(\n endpoint: string,\n limit: number,\n options?: InferenceAdmissionOptions,\n ): InferenceAdmission {\n assertValidLimit(limit);\n const configuredLimit = options?.configuredLimit ?? limit;\n assertValidLimit(configuredLimit);\n const generation = options?.generation ?? this.#currentGeneration;\n if (!Number.isSafeInteger(generation) || generation < 0) {\n throw new TypeError(\"InferenceAdmission generation must be a non-negative safe integer\");\n }\n const key = new URL(endpoint).href.replace(/\\/+$/, \"\");\n const existing = this.#endpoints.get(key);\n if (existing !== undefined) {\n if (generation > existing.#generation) {\n existing.#generation = generation;\n existing.#configuredLimit = configuredLimit;\n existing.#resize(limit);\n return existing;\n }\n if (existing.#limit === limit) {\n if (configuredLimit !== -1) existing.#configuredLimit = configuredLimit;\n return existing;\n }\n // Same operator policy on a restarted endpoint whose probed physical slot count changed.\n if (existing.#configuredLimit === configuredLimit) {\n existing.#resize(limit);\n return existing;\n }\n throw new TypeError(\n `Provider aliases resolving to ${key} have conflicting PLURNK_PROVIDERS_MAX_CONCURRENCY values ${existing.#limit} and ${limit}; one endpoint has one allowance`,\n );\n }\n const admission = new InferenceAdmission(limit, configuredLimit, generation);\n this.#endpoints.set(key, admission);\n return admission;\n }\n\n private constructor(limit: number, configuredLimit: number, generation: number) {\n this.#limit = limit;\n this.#configuredLimit = configuredLimit;\n this.#generation = generation;\n }\n\n get limit(): number {\n return this.#limit;\n }\n\n get activeCount(): number {\n return this.#active;\n }\n\n get waitingCount(): number {\n return this.#waiting.size;\n }\n\n #resize(limit: number): void {\n this.#limit = limit;\n this.#drainQueue();\n }\n\n #drainQueue(): void {\n while (this.#waiting.size > 0 && (this.#limit === -1 || this.#active < this.#limit)) {\n const next = this.#waiting.values().next().value!;\n this.#waiting.delete(next);\n next();\n }\n }\n\n acquire(signal?: AbortSignal): Promise<() => void> {\n signal?.throwIfAborted();\n return new Promise((resolve, reject) => {\n let queued = false;\n const abort = (): void => {\n this.#waiting.delete(enter);\n reject(signal!.reason);\n };\n const enter = (): void => {\n if (queued) signal?.removeEventListener(\"abort\", abort);\n this.#active += 1;\n let released = false;\n resolve(() => {\n if (released) throw new Error(\"Inference admission lease was released twice\");\n released = true;\n this.#active -= 1;\n this.#drainQueue();\n });\n };\n if (this.#limit === -1 || this.#active < this.#limit) enter();\n else {\n queued = true;\n this.#waiting.add(enter);\n signal?.addEventListener(\"abort\", abort, { once: true });\n }\n });\n }\n}\n"]}
@@ -11,6 +11,7 @@ export type CacheAffinity = {
11
11
  readonly name: string;
12
12
  };
13
13
  export type CacheBreakpoints = Exclude<CacheWritePolicy, "off">;
14
+ export type PromptCacheStyle = "ephemeral" | "openai-explicit";
14
15
  export type ModelProviderConfig = {
15
16
  inferenceAdmission?: InferenceAdmission;
16
17
  model: string;
@@ -42,6 +43,7 @@ export type ModelProviderConfig = {
42
43
  source?: string;
43
44
  cacheAffinity?: CacheAffinity;
44
45
  cacheBreakpoints?: CacheBreakpoints;
46
+ promptCacheStyle?: PromptCacheStyle;
45
47
  serviceTier?: string;
46
48
  requestControls?: NativeRequestControls;
47
49
  apiKeyRejectedMessage?: string;
@@ -96,6 +98,6 @@ export default class ModelProvider implements Provider {
96
98
  get requiresOutputBudget(): boolean | undefined;
97
99
  countPromptTokens(messages: readonly ChatMessage[], signal?: AbortSignal): Promise<PromptTokenMeasurement>;
98
100
  assessRequestCapacity(messages: readonly ChatMessage[], maxOutputTokens?: number, signal?: AbortSignal): Promise<ProviderRequestCapacity>;
99
- generate({ messages, sessionId, signal, maxOutputTokens, sampling, observeRequest, observeReasoning, callKind, finalUserCachePrefix }: ProviderGenerateArgs): Promise<ProviderResponse>;
101
+ generate({ messages, sessionId, signal, maxOutputTokens, sampling, observeRequest, observeReasoning, callKind, userCachePrefixes }: ProviderGenerateArgs): Promise<ProviderResponse>;
100
102
  }
101
103
  //# sourceMappingURL=ModelProvider.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"ModelProvider.d.ts","sourceRoot":"","sources":["../src/ModelProvider.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EACR,WAAW,EACX,MAAM,EACN,aAAa,EACb,sBAAsB,EACtB,QAAQ,EAER,sBAAsB,EACtB,oBAAoB,EACpB,aAAa,EAEb,uBAAuB,EAEvB,gBAAgB,EAChB,aAAa,EAChB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAE7D,OAAO,KAAK,EAAE,gBAAgB,EAAE,aAAa,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAC;AAMxF,OAAO,EAAiB,KAAK,oBAAoB,EAAE,KAAK,2BAA2B,EAAE,MAAM,qBAAqB,CAAC;AAOjH,OAAO,KAAK,kBAAkB,MAAM,yBAAyB,CAAC;AAM9D,MAAM,MAAM,aAAa,GAAG,OAAO,UAAU,CAAC,KAAK,CAAC;AAGpD,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,OAAO,GAAG,UAAU,CAAC;AAE3D,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,OAAO,CAAC;AAE3E,MAAM,MAAM,aAAa,GAAG;IAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAC1F,MAAM,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;AAEhE,MAAM,MAAM,mBAAmB,GAAG;IAC9B,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IACxC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,2BAA2B,KAAK,oBAAoB,CAAC;IAC9E,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,eAAe,CAAC,EAAE,WAAW,CAAC,aAAa,CAAC,CAAC;IAC7C,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,gBAAgB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAIrC,cAAc,CAAC,EAAE,YAAY,GAAG,kBAAkB,CAAC;IACnD,sBAAsB,CAAC,EAAE,OAAO,CAAC;IAKjC,yBAAyB,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;IACpD,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,gBAAgB,CAAC,EAAE,YAAY,GAAG,uBAAuB,CAAC;IAC1D,oBAAoB,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,GAAG,IAAI,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjG,sBAAsB,CAAC,EAAE,sBAAsB,CAAC;IAChD,iBAAiB,CAAC,EAAE,CAAC,QAAQ,EAAE,SAAS,WAAW,EAAE,EAAE,MAAM,CAAC,EAAE,WAAW,KAAK,sBAAsB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;IACzI,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,GAAG,SAAS,KAAK,YAAY,CAAC;IAClE,aAAa,CAAC,EAAE,sBAAsB,CAAC;IACvC,MAAM,CAAC,EAAE,MAAM,CAAC;IAGhB,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IAIpC,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,eAAe,CAAC,EAAE,qBAAqB,CAAC;IACxC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAI1B,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,eAAe,CAAC,EAAE,MAAM,CAAC;IAKzB,WAAW,CAAC,EAAE,MAAM,CAAC;IAIrB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAG/B,MAAM,EAAE,aAAa,CAAC;IAGtB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAE7B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAK1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IAKrB,aAAa,EAAE,MAAM,CAAC;IAItB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAQ1B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,OAAO,CAAC,EAAE,OAAO,CAAC;CACrB,CAAC;AAmCF,MAAM,WAAW,qBAAqB;IAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;CAC9B;AA+BD,MAAM,CAAC,OAAO,OAAO,aAAc,YAAW,QAAQ;;IAmDlD,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,2BAA2B,KAAK,oBAAoB,CAAC;IAMvF,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAE/C,YAAY,MAAM,EAAE,mBAAmB,EA4JtC;IAED,IAAI,aAAa,IAAI,MAAM,GAAG,IAAI,CAAgC;IAClE,IAAI,eAAe,IAAI,WAAW,CAAC,aAAa,CAAC,CAAkC;IACnF,IAAI,cAAc,IAAI,MAAM,GAAG,IAAI,CAAiC;IACpE,IAAI,eAAe,IAAI,MAAM,GAAG,IAAI,CAAkC;IACtE,IAAI,YAAY,IAAI,MAAM,GAAG,IAAI,CAA+B;IAChE,IAAI,WAAW,IAAI,MAAM,GAAG,IAAI,CAA8B;IAC9D,IAAI,eAAe,IAAI,MAAM,GAAG,IAAI,CAAkC;IACtE,IAAI,gBAAgB,IAAI,SAAS,MAAM,EAAE,CAAmC;IAC5E,IAAI,aAAa,IAAI,MAAM,GAAG,IAAI,CAMjC;IAED,IAAI,SAAS,IAAI,MAAM,GAAG,IAAI,CAM7B;IACD,IAAI,KAAK,IAAI,MAAM,CAAwB;IAC3C,IAAI,WAAW,IAAI,MAAM,GAAG,IAAI,CAAuD;IACvF,IAAI,IAAI,IAAI,MAAM,GAAG,SAAS,CAAwC;IAEtE,IAAI,WAAW,IAAI,MAAM,GAAG,SAAS,CAA8B;IAEnE,IAAI,oBAAoB,IAAI,OAAO,GAAG,SAAS,CAAuC;IAEhF,iBAAiB,CACnB,QAAQ,EAAE,SAAS,WAAW,EAAE,EAChC,MAAM,CAAC,EAAE,WAAW,GACrB,OAAO,CAAC,sBAAsB,CAAC,CAsDjC;IAEK,qBAAqB,CACvB,QAAQ,EAAE,SAAS,WAAW,EAAE,EAChC,eAAe,CAAC,EAAE,MAAM,EACxB,MAAM,CAAC,EAAE,WAAW,GACrB,OAAO,CAAC,uBAAuB,CAAC,CAoBlC;IA4BK,QAAQ,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,cAAc,EAAE,gBAAgB,EAAE,QAAQ,EAAE,oBAAoB,EAAE,EAAE,oBAAoB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA8a5L;CAEJ"}
1
+ {"version":3,"file":"ModelProvider.d.ts","sourceRoot":"","sources":["../src/ModelProvider.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EACR,WAAW,EACX,MAAM,EACN,aAAa,EACb,sBAAsB,EACtB,QAAQ,EAER,sBAAsB,EACtB,oBAAoB,EACpB,aAAa,EAEb,uBAAuB,EAEvB,gBAAgB,EAChB,aAAa,EAChB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAE7D,OAAO,KAAK,EAAE,gBAAgB,EAAE,aAAa,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAC;AAMxF,OAAO,EAAiB,KAAK,oBAAoB,EAAE,KAAK,2BAA2B,EAAE,MAAM,qBAAqB,CAAC;AAOjH,OAAO,KAAK,kBAAkB,MAAM,yBAAyB,CAAC;AAM9D,MAAM,MAAM,aAAa,GAAG,OAAO,UAAU,CAAC,KAAK,CAAC;AAGpD,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,OAAO,GAAG,UAAU,CAAC;AAE3D,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,OAAO,CAAC;AAE3E,MAAM,MAAM,aAAa,GAAG;IAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAC1F,MAAM,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;AAChE,MAAM,MAAM,gBAAgB,GAAG,WAAW,GAAG,iBAAiB,CAAC;AAE/D,MAAM,MAAM,mBAAmB,GAAG;IAC9B,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IACxC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,2BAA2B,KAAK,oBAAoB,CAAC;IAC9E,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,eAAe,CAAC,EAAE,WAAW,CAAC,aAAa,CAAC,CAAC;IAC7C,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,gBAAgB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAIrC,cAAc,CAAC,EAAE,YAAY,GAAG,kBAAkB,CAAC;IACnD,sBAAsB,CAAC,EAAE,OAAO,CAAC;IAKjC,yBAAyB,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;IACpD,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,gBAAgB,CAAC,EAAE,YAAY,GAAG,uBAAuB,CAAC;IAC1D,oBAAoB,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,GAAG,IAAI,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjG,sBAAsB,CAAC,EAAE,sBAAsB,CAAC;IAChD,iBAAiB,CAAC,EAAE,CAAC,QAAQ,EAAE,SAAS,WAAW,EAAE,EAAE,MAAM,CAAC,EAAE,WAAW,KAAK,sBAAsB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;IACzI,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,GAAG,SAAS,KAAK,YAAY,CAAC;IAClE,aAAa,CAAC,EAAE,sBAAsB,CAAC;IACvC,MAAM,CAAC,EAAE,MAAM,CAAC;IAGhB,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IAIpC,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,eAAe,CAAC,EAAE,qBAAqB,CAAC;IACxC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAI1B,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,eAAe,CAAC,EAAE,MAAM,CAAC;IAKzB,WAAW,CAAC,EAAE,MAAM,CAAC;IAIrB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAG/B,MAAM,EAAE,aAAa,CAAC;IAGtB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAE7B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAK1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IAKrB,aAAa,EAAE,MAAM,CAAC;IAItB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAQ1B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,OAAO,CAAC,EAAE,OAAO,CAAC;CACrB,CAAC;AAmCF,MAAM,WAAW,qBAAqB;IAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;CAC9B;AA+BD,MAAM,CAAC,OAAO,OAAO,aAAc,YAAW,QAAQ;;IAoDlD,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,2BAA2B,KAAK,oBAAoB,CAAC;IAMvF,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAE/C,YAAY,MAAM,EAAE,mBAAmB,EA6JtC;IAED,IAAI,aAAa,IAAI,MAAM,GAAG,IAAI,CAAgC;IAClE,IAAI,eAAe,IAAI,WAAW,CAAC,aAAa,CAAC,CAAkC;IACnF,IAAI,cAAc,IAAI,MAAM,GAAG,IAAI,CAAiC;IACpE,IAAI,eAAe,IAAI,MAAM,GAAG,IAAI,CAAkC;IACtE,IAAI,YAAY,IAAI,MAAM,GAAG,IAAI,CAA+B;IAChE,IAAI,WAAW,IAAI,MAAM,GAAG,IAAI,CAA8B;IAC9D,IAAI,eAAe,IAAI,MAAM,GAAG,IAAI,CAAkC;IACtE,IAAI,gBAAgB,IAAI,SAAS,MAAM,EAAE,CAAmC;IAC5E,IAAI,aAAa,IAAI,MAAM,GAAG,IAAI,CAMjC;IAED,IAAI,SAAS,IAAI,MAAM,GAAG,IAAI,CAM7B;IACD,IAAI,KAAK,IAAI,MAAM,CAAwB;IAC3C,IAAI,WAAW,IAAI,MAAM,GAAG,IAAI,CAAuD;IACvF,IAAI,IAAI,IAAI,MAAM,GAAG,SAAS,CAAwC;IAEtE,IAAI,WAAW,IAAI,MAAM,GAAG,SAAS,CAA8B;IAEnE,IAAI,oBAAoB,IAAI,OAAO,GAAG,SAAS,CAAuC;IAEhF,iBAAiB,CACnB,QAAQ,EAAE,SAAS,WAAW,EAAE,EAChC,MAAM,CAAC,EAAE,WAAW,GACrB,OAAO,CAAC,sBAAsB,CAAC,CAsDjC;IAEK,qBAAqB,CACvB,QAAQ,EAAE,SAAS,WAAW,EAAE,EAChC,eAAe,CAAC,EAAE,MAAM,EACxB,MAAM,CAAC,EAAE,WAAW,GACrB,OAAO,CAAC,uBAAuB,CAAC,CAoBlC;IA4BK,QAAQ,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,cAAc,EAAE,gBAAgB,EAAE,QAAQ,EAAE,iBAAiB,EAAE,EAAE,oBAAoB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA+azL;CAEJ"}
@@ -18,7 +18,7 @@ import { assessRequestCapacity, effectiveInputCapacity, effectiveInputWall, effe
18
18
  import ModelRequestBody from "./ModelRequestBody.js";
19
19
  import LeadingReasoning from "./LeadingReasoning.js";
20
20
  import RequestCapture from "./RequestCapture.js";
21
- import { executePiTransportStream } from "./pi-transport.js";
21
+ import { executePiTransportStream, formatCacheAffinityValue } from "./pi-transport.js";
22
22
  import { executeWithRetry, raceAgainstDeadline } from "./retry-deadline.js";
23
23
  import { RETRYABLE_MALFORMED_FINISH_REASONS, readFailureEvidence } from "./stream-forensics.js";
24
24
  class ProviderRequestObserverError extends Error {
@@ -116,6 +116,7 @@ export default class ModelProvider {
116
116
  #source;
117
117
  #cacheAffinity;
118
118
  #cacheBreakpoints;
119
+ #promptCacheStyle;
119
120
  #serviceTier;
120
121
  #resolvedControls;
121
122
  #supportsSlotPinning;
@@ -213,7 +214,8 @@ export default class ModelProvider {
213
214
  this.#source = config.source ?? "provider";
214
215
  this.#cacheAffinity = config.cacheAffinity;
215
216
  this.#cacheBreakpoints = config.cacheBreakpoints;
216
- if (this.#providerModel === undefined && this.#cacheBreakpoints !== undefined) {
217
+ this.#promptCacheStyle = config.promptCacheStyle;
218
+ if (this.#providerModel === undefined && (this.#cacheBreakpoints !== undefined || this.#promptCacheStyle !== undefined)) {
217
219
  throw new Error(`${this.#source}: prompt cache breakpoints require a model transport`);
218
220
  }
219
221
  if (this.#providerModel === undefined && this.#additiveReasoningProvider !== undefined) {
@@ -400,7 +402,7 @@ export default class ModelProvider {
400
402
  cost: resolveProviderCost(direct, this.#estimateCost(knownUsage)),
401
403
  });
402
404
  }
403
- async generate({ messages, sessionId, signal, maxOutputTokens, sampling, observeRequest, observeReasoning, callKind, finalUserCachePrefix }) {
405
+ async generate({ messages, sessionId, signal, maxOutputTokens, sampling, observeRequest, observeReasoning, callKind, userCachePrefixes }) {
404
406
  // {§provider-interface} The session identity is required.
405
407
  if (sessionId === undefined || sessionId.length === 0)
406
408
  throw new Error("generate: sessionId is required — the session's stable, opaque identity");
@@ -439,7 +441,7 @@ export default class ModelProvider {
439
441
  ...(this.#topLogprobs !== null ? { logprobs: true, top_logprobs: this.#topLogprobs } : {}),
440
442
  ...this.#requestBody.slotBody(sessionId),
441
443
  ...(this.#cacheAffinity?.target === "body"
442
- ? { [this.#cacheAffinity.name]: sessionId }
444
+ ? { [this.#cacheAffinity.name]: formatCacheAffinityValue(this.#cacheAffinity, sessionId) }
443
445
  : {}),
444
446
  };
445
447
  for (const field of ["tools", "tool_choice", "functions", "function_call", "parallel_tool_calls"]) {
@@ -575,7 +577,8 @@ export default class ModelProvider {
575
577
  nativeAdaptiveThinking: this.#nativeAdaptiveThinking,
576
578
  reasoningRequestBody: this.#reasoningRequestBody,
577
579
  cacheBreakpoints: callKind === "bare" ? undefined : this.#cacheBreakpoints,
578
- finalUserCachePrefix,
580
+ promptCacheStyle: this.#promptCacheStyle,
581
+ userCachePrefixes,
579
582
  cacheAffinity: this.#cacheAffinity,
580
583
  resolvedControls: this.#resolvedControls,
581
584
  rawBody: this.#rawBody,