@plurnk/plurnk-providers 6.0.3 → 6.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/.env.defaults +4 -5
  2. package/README.md +31 -213
  3. package/SPEC.md +11 -7
  4. package/dist/InferenceAdmission.js.map +1 -1
  5. package/dist/LeadingReasoning.d.ts.map +1 -1
  6. package/dist/LeadingReasoning.js +12 -1
  7. package/dist/LeadingReasoning.js.map +1 -1
  8. package/dist/Mock.d.ts +5 -1
  9. package/dist/Mock.d.ts.map +1 -1
  10. package/dist/Mock.js +7 -1
  11. package/dist/Mock.js.map +1 -1
  12. package/dist/ModelProvider.d.ts +6 -3
  13. package/dist/ModelProvider.d.ts.map +1 -1
  14. package/dist/ModelProvider.js +24 -24
  15. package/dist/ModelProvider.js.map +1 -1
  16. package/dist/ModelRequestBody.js.map +1 -1
  17. package/dist/Pool.d.ts +2 -0
  18. package/dist/Pool.d.ts.map +1 -1
  19. package/dist/Pool.js +8 -0
  20. package/dist/Pool.js.map +1 -1
  21. package/dist/ProviderRegistry.d.ts.map +1 -1
  22. package/dist/ProviderRegistry.js +5 -10
  23. package/dist/ProviderRegistry.js.map +1 -1
  24. package/dist/RequestCapture.js.map +1 -1
  25. package/dist/accounting.js.map +1 -1
  26. package/dist/accountingPublic.js.map +1 -1
  27. package/dist/capacity.js.map +1 -1
  28. package/dist/catalogProvider.d.ts +2 -2
  29. package/dist/catalogProvider.d.ts.map +1 -1
  30. package/dist/catalogProvider.js +4 -4
  31. package/dist/catalogProvider.js.map +1 -1
  32. package/dist/compatibleProvider.js.map +1 -1
  33. package/dist/cost.js.map +1 -1
  34. package/dist/defaults.js.map +1 -1
  35. package/dist/discover.d.ts +0 -2
  36. package/dist/discover.d.ts.map +1 -1
  37. package/dist/discover.js +2 -8
  38. package/dist/discover.js.map +1 -1
  39. package/dist/env.d.ts +1 -1
  40. package/dist/env.d.ts.map +1 -1
  41. package/dist/env.js +2 -2
  42. package/dist/env.js.map +1 -1
  43. package/dist/errors.d.ts.map +1 -1
  44. package/dist/errors.js +4 -1
  45. package/dist/errors.js.map +1 -1
  46. package/dist/index.d.ts +1 -1
  47. package/dist/index.d.ts.map +1 -1
  48. package/dist/index.js.map +1 -1
  49. package/dist/modelAdapters.d.ts +1 -1
  50. package/dist/modelAdapters.d.ts.map +1 -1
  51. package/dist/modelAdapters.js +20 -4
  52. package/dist/modelAdapters.js.map +1 -1
  53. package/dist/notices.js.map +1 -1
  54. package/dist/ollama.js.map +1 -1
  55. package/dist/openai.js.map +1 -1
  56. package/dist/pi-transport.d.ts +6 -3
  57. package/dist/pi-transport.d.ts.map +1 -1
  58. package/dist/pi-transport.js +304 -49
  59. package/dist/pi-transport.js.map +1 -1
  60. package/dist/promptTokens.js.map +1 -1
  61. package/dist/provider-env.js.map +1 -1
  62. package/dist/providerError.js.map +1 -1
  63. package/dist/reasoning-effort.js.map +1 -1
  64. package/dist/retry-deadline.d.ts.map +1 -1
  65. package/dist/retry-deadline.js +10 -14
  66. package/dist/retry-deadline.js.map +1 -1
  67. package/dist/stream-forensics.d.ts.map +1 -1
  68. package/dist/stream-forensics.js +173 -25
  69. package/dist/stream-forensics.js.map +1 -1
  70. package/dist/types.d.ts +10 -1
  71. package/dist/types.d.ts.map +1 -1
  72. package/dist/types.js.map +1 -1
  73. package/dist/usage.js.map +1 -1
  74. package/dist/warnings.js.map +1 -1
  75. package/docs/models.md +9 -15
  76. package/package.json +6 -6
@@ -6,16 +6,37 @@ import { stream as streamBedrockConverseStream } from "@earendil-works/pi-ai/api
6
6
  import { stream as streamGoogleGenerativeAi } from "@earendil-works/pi-ai/api/google-generative-ai";
7
7
  import { stream as streamOpenAICompletions } from "@earendil-works/pi-ai/api/openai-completions";
8
8
  import { stream as streamOpenAIResponses } from "@earendil-works/pi-ai/api/openai-responses";
9
+ import { getBuiltinModels, getBuiltinProviders, } from "@earendil-works/pi-ai/providers/all";
10
+ import { estimateMessageTokens } from "@earendil-works/pi-ai/utils/estimate";
11
+ import { headersToRecord } from "@earendil-works/pi-ai/utils/headers";
9
12
  import { isRecord } from "@plurnk/plurnk-contracts";
10
13
  import { outboundFetch, syncAmbientHttp } from "@plurnk/plurnk-meta";
11
14
  import { APICallError } from "./errors.js";
12
15
  import { nativeFixedEffort } from "./reasoning-effort.js";
13
16
  import { abortFailure, attachFailureEvidence, collectChunkForensics, extractFromAssistantMessage, serializeError, toUsageFromPiMessage, } from "./stream-forensics.js";
14
- const headersToRecord = (headers) => {
15
- const out = {};
16
- for (const [key, value] of headers.entries())
17
- out[key] = value;
18
- return out;
17
+ const PI_BUILTIN_PROVIDER_ALIASES = {
18
+ "alibaba": "qwen-token-plan",
19
+ "alibaba-cn": "qwen-token-plan-cn",
20
+ "fireworks-ai": "fireworks",
21
+ "togetherai": "together",
22
+ "zai-coding-plan": "zai",
23
+ };
24
+ const lookupPiBuiltinCompat = (providerId, modelId) => {
25
+ const targetProvider = PI_BUILTIN_PROVIDER_ALIASES[providerId] ?? providerId;
26
+ const builtinProvider = getBuiltinProviders().find((candidate) => candidate === targetProvider);
27
+ if (builtinProvider === undefined)
28
+ return {};
29
+ const builtin = getBuiltinModels(builtinProvider).find((candidate) => candidate.id === modelId);
30
+ const rawCompat = builtin?.compat;
31
+ const compat = isRecord(rawCompat) ? rawCompat : undefined;
32
+ return {
33
+ ...(typeof compat?.requiresReasoningContentOnAssistantMessages === "boolean"
34
+ ? { requiresReasoningContentOnAssistantMessages: compat.requiresReasoningContentOnAssistantMessages }
35
+ : {}),
36
+ ...(typeof compat?.zaiToolStream === "boolean"
37
+ ? { zaiToolStream: compat.zaiToolStream }
38
+ : {}),
39
+ };
19
40
  };
20
41
  const toPiContext = (messages) => {
21
42
  const systemParts = [];
@@ -68,57 +89,157 @@ const toPiContext = (messages) => {
68
89
  messages: contextMessages,
69
90
  });
70
91
  };
71
- const mapOpenAICompatibleMessages = (messages, systemCacheBreakpoint) => {
92
+ const hasWireContent = (message) => typeof message.content === "string"
93
+ ? message.content.trim().length > 0
94
+ : message.content.some((part) => part.type !== "text" || part.text.trim().length > 0);
95
+ const countWireParts = (parts, limit) => {
96
+ let count = 0;
97
+ const end = Math.min(limit, parts.length);
98
+ for (let i = 0; i < end; i += 1) {
99
+ const part = parts[i];
100
+ if (part.type !== "text" || part.text.trim().length > 0)
101
+ count += 1;
102
+ }
103
+ return count;
104
+ };
105
+ const toOpenAIPart = (part) => {
106
+ if (part.type === "text")
107
+ return { type: "text", text: part.text };
108
+ const base64 = Buffer.from(part.data).toString("base64");
109
+ if (part.mediaType.startsWith("audio/")) {
110
+ return {
111
+ type: "input_audio",
112
+ input_audio: {
113
+ data: base64,
114
+ format: part.mediaType.slice("audio/".length),
115
+ },
116
+ };
117
+ }
118
+ return {
119
+ type: "image_url",
120
+ image_url: { url: `data:${part.mediaType};base64,${base64}` },
121
+ };
122
+ };
123
+ const withEphemeralCacheControl = (parts, atIndex) => {
124
+ if (atIndex >= 0 && atIndex < parts.length) {
125
+ parts[atIndex] = { ...parts[atIndex], cache_control: { type: "ephemeral" } };
126
+ }
127
+ return parts;
128
+ };
129
+ const mapOpenAICompatibleMessages = (messages, cacheBreakpoints, finalUserCachePrefix) => {
72
130
  let lastLeadingSystemIndex = -1;
73
- if (systemCacheBreakpoint) {
74
- for (let i = 0; i < messages.length; i += 1) {
75
- if (messages[i].role !== "system")
76
- break;
131
+ let firstNonSystemIndex = messages.length;
132
+ for (let i = 0; i < messages.length; i += 1) {
133
+ if (messages[i].role !== "system") {
134
+ firstNonSystemIndex = i;
135
+ break;
136
+ }
137
+ if (cacheBreakpoints !== undefined && hasWireContent(messages[i])) {
77
138
  lastLeadingSystemIndex = i;
78
139
  }
79
140
  }
141
+ let penultimateNonSystemIndex = -1;
142
+ let finalUserIndex = -1;
143
+ if (cacheBreakpoints === "conversation") {
144
+ const nonSystemIndices = [];
145
+ for (let i = firstNonSystemIndex; i < messages.length; i += 1) {
146
+ if (hasWireContent(messages[i]))
147
+ nonSystemIndices.push(i);
148
+ }
149
+ if (nonSystemIndices.length >= 2) {
150
+ penultimateNonSystemIndex = nonSystemIndices[nonSystemIndices.length - 2];
151
+ }
152
+ const lastIdx = nonSystemIndices.at(-1);
153
+ if (lastIdx !== undefined && messages[lastIdx].role === "user") {
154
+ finalUserIndex = lastIdx;
155
+ }
156
+ }
80
157
  return messages.map((message, index) => {
81
- if (index === lastLeadingSystemIndex && typeof message.content === "string") {
158
+ if (index === lastLeadingSystemIndex || index === penultimateNonSystemIndex) {
159
+ const parts = typeof message.content === "string"
160
+ ? [{ type: "text", text: message.content }]
161
+ : message.content.map(toOpenAIPart);
82
162
  return {
83
163
  role: message.role,
84
- content: [{
85
- type: "text",
86
- text: message.content,
87
- cache_control: { type: "ephemeral" },
88
- }],
164
+ content: withEphemeralCacheControl(parts, parts.length - 1),
89
165
  };
90
166
  }
91
- if (typeof message.content === "string") {
92
- return { role: message.role, content: message.content };
93
- }
94
- return {
95
- role: message.role,
96
- content: message.content.map((part) => {
97
- if (part.type === "text")
98
- return { type: "text", text: part.text };
99
- const base64 = Buffer.from(part.data).toString("base64");
100
- if (part.mediaType.startsWith("audio/")) {
167
+ if (index === finalUserIndex) {
168
+ if (finalUserCachePrefix === undefined) {
169
+ const parts = typeof message.content === "string"
170
+ ? [{ type: "text", text: message.content }]
171
+ : message.content.map(toOpenAIPart);
172
+ return {
173
+ role: message.role,
174
+ content: withEphemeralCacheControl(parts, parts.length - 1),
175
+ };
176
+ }
177
+ if (finalUserCachePrefix.kind === "chars" && typeof message.content === "string") {
178
+ if (finalUserCachePrefix.length <= 0) {
179
+ return { role: message.role, content: message.content };
180
+ }
181
+ if (finalUserCachePrefix.length >= message.content.length) {
101
182
  return {
102
- type: "input_audio",
103
- input_audio: {
104
- data: base64,
105
- format: part.mediaType.slice("audio/".length),
106
- },
183
+ role: message.role,
184
+ content: [{ type: "text", text: message.content, cache_control: { type: "ephemeral" } }],
107
185
  };
108
186
  }
187
+ const durable = message.content.slice(0, finalUserCachePrefix.length);
188
+ const volatile = message.content.slice(finalUserCachePrefix.length);
189
+ if (durable.trim().length === 0) {
190
+ return { role: message.role, content: message.content };
191
+ }
109
192
  return {
110
- type: "image_url",
111
- image_url: { url: `data:${part.mediaType};base64,${base64}` },
193
+ role: message.role,
194
+ content: volatile.trim().length > 0
195
+ ? [
196
+ { type: "text", text: durable, cache_control: { type: "ephemeral" } },
197
+ { type: "text", text: volatile },
198
+ ]
199
+ : [{ type: "text", text: durable, cache_control: { type: "ephemeral" } }],
112
200
  };
113
- }),
201
+ }
202
+ if (finalUserCachePrefix.kind === "parts" && Array.isArray(message.content)) {
203
+ const parts = message.content.map(toOpenAIPart);
204
+ const targetIndex = Math.min(finalUserCachePrefix.count, parts.length) - 1;
205
+ return {
206
+ role: message.role,
207
+ content: finalUserCachePrefix.count > 0
208
+ ? withEphemeralCacheControl(parts, targetIndex)
209
+ : parts,
210
+ };
211
+ }
212
+ }
213
+ if (typeof message.content === "string") {
214
+ return { role: message.role, content: message.content };
215
+ }
216
+ return {
217
+ role: message.role,
218
+ content: message.content.map(toOpenAIPart),
114
219
  };
115
220
  });
116
221
  };
117
- const applyAnthropicPayloadAdjustments = (params, messages, systemCacheBreakpoint, serviceTier) => {
222
+ const markAnthropicMessageEnd = (wireMsg) => {
223
+ if (!isRecord(wireMsg))
224
+ return;
225
+ if (typeof wireMsg.content === "string") {
226
+ if (wireMsg.content.trim().length > 0) {
227
+ wireMsg.content = [{ type: "text", text: wireMsg.content, cache_control: { type: "ephemeral" } }];
228
+ }
229
+ return;
230
+ }
231
+ if (Array.isArray(wireMsg.content) && wireMsg.content.length > 0) {
232
+ const last = wireMsg.content.at(-1);
233
+ if (isRecord(last)) {
234
+ last.cache_control = { type: "ephemeral" };
235
+ }
236
+ }
237
+ };
238
+ const applyAnthropicPayloadAdjustments = (params, messages, cacheBreakpoints, finalUserCachePrefix, serviceTier) => {
118
239
  if (serviceTier !== undefined) {
119
240
  params.service_tier = serviceTier;
120
241
  }
121
- if (systemCacheBreakpoint && Array.isArray(params.system) && params.system.length > 0) {
242
+ if (cacheBreakpoints !== undefined && Array.isArray(params.system) && params.system.length > 0) {
122
243
  const last = params.system.at(-1);
123
244
  if (isRecord(last)) {
124
245
  last.cache_control = { type: "ephemeral" };
@@ -127,14 +248,15 @@ const applyAnthropicPayloadAdjustments = (params, messages, systemCacheBreakpoin
127
248
  const wireMessages = Array.isArray(params.messages) ? params.messages : undefined;
128
249
  if (wireMessages === undefined)
129
250
  return;
130
- const nonSystem = messages.filter((m) => m.role !== "system");
131
- for (let i = 0; i < Math.min(wireMessages.length, nonSystem.length); i += 1) {
132
- const sourceMsg = nonSystem[i];
251
+ const effectiveNonSystem = messages.filter((m) => m.role !== "system" && hasWireContent(m));
252
+ for (let i = 0; i < Math.min(wireMessages.length, effectiveNonSystem.length); i += 1) {
253
+ const sourceMsg = effectiveNonSystem[i];
133
254
  const wireMsg = wireMessages[i];
134
255
  if (!Array.isArray(sourceMsg.content) || !isRecord(wireMsg) || !Array.isArray(wireMsg.content))
135
256
  continue;
136
- for (let j = 0; j < Math.min(sourceMsg.content.length, wireMsg.content.length); j += 1) {
137
- const sourcePart = sourceMsg.content[j];
257
+ const nonBlankSourceParts = sourceMsg.content.filter((p) => p.type !== "text" || p.text.trim().length > 0);
258
+ for (let j = 0; j < Math.min(nonBlankSourceParts.length, wireMsg.content.length); j += 1) {
259
+ const sourcePart = nonBlankSourceParts[j];
138
260
  const wireBlock = wireMsg.content[j];
139
261
  if (sourcePart.type === "file"
140
262
  && sourcePart.mediaType === "application/pdf"
@@ -144,19 +266,121 @@ const applyAnthropicPayloadAdjustments = (params, messages, systemCacheBreakpoin
144
266
  }
145
267
  }
146
268
  }
269
+ if (cacheBreakpoints !== "conversation" || wireMessages.length === 0)
270
+ return;
271
+ if (wireMessages.length >= 2) {
272
+ markAnthropicMessageEnd(wireMessages[wireMessages.length - 2]);
273
+ }
274
+ const lastWire = wireMessages.at(-1);
275
+ const lastSource = effectiveNonSystem.at(-1);
276
+ if (lastSource?.role !== "user" || !isRecord(lastWire) || lastWire.role !== "user")
277
+ return;
278
+ if (finalUserCachePrefix === undefined) {
279
+ markAnthropicMessageEnd(lastWire);
280
+ return;
281
+ }
282
+ if (finalUserCachePrefix.kind === "chars" && typeof lastSource.content === "string") {
283
+ if (finalUserCachePrefix.length <= 0)
284
+ return;
285
+ if (finalUserCachePrefix.length >= lastSource.content.length) {
286
+ markAnthropicMessageEnd(lastWire);
287
+ return;
288
+ }
289
+ const durable = lastSource.content.slice(0, finalUserCachePrefix.length);
290
+ const volatile = lastSource.content.slice(finalUserCachePrefix.length);
291
+ if (durable.trim().length === 0)
292
+ return;
293
+ lastWire.content = volatile.trim().length > 0
294
+ ? [
295
+ { type: "text", text: durable, cache_control: { type: "ephemeral" } },
296
+ { type: "text", text: volatile },
297
+ ]
298
+ : [{ type: "text", text: durable, cache_control: { type: "ephemeral" } }];
299
+ return;
300
+ }
301
+ if (finalUserCachePrefix.kind === "parts" && Array.isArray(lastSource.content) && Array.isArray(lastWire.content)) {
302
+ const k = countWireParts(lastSource.content, finalUserCachePrefix.count);
303
+ if (k > 0 && k <= lastWire.content.length) {
304
+ const target = lastWire.content[k - 1];
305
+ if (isRecord(target))
306
+ target.cache_control = { type: "ephemeral" };
307
+ }
308
+ }
309
+ };
310
+ const appendBedrockCachePoint = (wireMsg, sourceMsg) => {
311
+ if (sourceMsg === undefined || !hasWireContent(sourceMsg))
312
+ return;
313
+ if (!isRecord(wireMsg) || !Array.isArray(wireMsg.content) || wireMsg.content.length === 0)
314
+ return;
315
+ wireMsg.content = [...wireMsg.content, { cachePoint: { type: "default" } }];
316
+ };
317
+ const applyBedrockPayloadAdjustments = (params, messages, cacheBreakpoints, finalUserCachePrefix) => {
318
+ if (cacheBreakpoints === undefined)
319
+ return;
320
+ if (Array.isArray(params.system) && params.system.length > 0) {
321
+ params.system = [...params.system, { cachePoint: { type: "default" } }];
322
+ }
323
+ if (cacheBreakpoints !== "conversation")
324
+ return;
325
+ const wireMessages = Array.isArray(params.messages) ? params.messages : undefined;
326
+ if (wireMessages === undefined || wireMessages.length === 0)
327
+ return;
328
+ const bedrockNonSystem = messages.filter((m) => m.role === "user" || (m.role === "assistant" && hasWireContent(m)));
329
+ if (wireMessages.length >= 2) {
330
+ appendBedrockCachePoint(wireMessages[wireMessages.length - 2], bedrockNonSystem[bedrockNonSystem.length - 2]);
331
+ }
332
+ const lastWire = wireMessages.at(-1);
333
+ const lastSource = bedrockNonSystem.at(-1);
334
+ if (lastSource?.role !== "user" || !hasWireContent(lastSource) || !isRecord(lastWire) || lastWire.role !== "user" || !Array.isArray(lastWire.content)) {
335
+ return;
336
+ }
337
+ if (finalUserCachePrefix === undefined) {
338
+ appendBedrockCachePoint(lastWire, lastSource);
339
+ return;
340
+ }
341
+ if (finalUserCachePrefix.kind === "chars" && typeof lastSource.content === "string") {
342
+ if (finalUserCachePrefix.length <= 0)
343
+ return;
344
+ if (finalUserCachePrefix.length >= lastSource.content.length) {
345
+ appendBedrockCachePoint(lastWire, lastSource);
346
+ return;
347
+ }
348
+ const durable = lastSource.content.slice(0, finalUserCachePrefix.length);
349
+ const volatile = lastSource.content.slice(finalUserCachePrefix.length);
350
+ if (durable.trim().length === 0)
351
+ return;
352
+ lastWire.content = volatile.trim().length > 0
353
+ ? [{ text: durable }, { cachePoint: { type: "default" } }, { text: volatile }]
354
+ : [{ text: durable }, { cachePoint: { type: "default" } }];
355
+ return;
356
+ }
357
+ if (finalUserCachePrefix.kind === "parts" && Array.isArray(lastSource.content)) {
358
+ const k = countWireParts(lastSource.content, finalUserCachePrefix.count);
359
+ if (k > 0 && k <= lastWire.content.length) {
360
+ lastWire.content = [
361
+ ...lastWire.content.slice(0, k),
362
+ { cachePoint: { type: "default" } },
363
+ ...lastWire.content.slice(k),
364
+ ];
365
+ }
366
+ }
147
367
  };
148
368
  export const executePiTransportStream = async (opts) => {
149
- const { model, source, url, providerModel, fetch: configuredFetch, requestHeaders, messages, sessionId, operationSignal, requestSampling, body, effectiveMaxOutputTokens, nativeReasoningBudget, capacity, requestBody, effort, adaptiveEffort, nativeAdaptiveThinking, reasoningRequestBody, systemCacheBreakpoint, cacheAffinity, resolvedControls, rawBody, capture, reasoningStepDown = 0, observers, } = opts;
369
+ const { model, source, url, providerModel, fetch: configuredFetch, requestHeaders, messages, sessionId, operationSignal, requestSampling, body, outputTokenField = "max_tokens", effectiveMaxOutputTokens, nativeReasoningBudget, capacity, requestBody, effort, adaptiveEffort, nativeAdaptiveThinking, reasoningRequestBody, cacheBreakpoints, finalUserCachePrefix, cacheAffinity, resolvedControls, rawBody, capture, reasoningStepDown = 0, observers, } = opts;
370
+ const providerId = source.replace(/^provider:/, "");
150
371
  const effectiveModel = providerModel ?? {
151
372
  id: model,
152
373
  api: "openai-completions",
153
- provider: source,
374
+ provider: providerId,
154
375
  baseUrl: url.replace(/\/chat\/completions\/?$/, ""),
155
376
  compat: {
377
+ ...lookupPiBuiltinCompat(providerId, model),
156
378
  supportsStore: false,
157
379
  supportsDeveloperRole: false,
158
380
  supportsUsageInStreaming: true,
159
381
  supportsFinishReason: true,
382
+ maxTokensField: outputTokenField,
383
+ thinkingFormat: "openai",
160
384
  },
161
385
  };
162
386
  const buildPiModel = (api) => ({
@@ -240,8 +464,12 @@ export const executePiTransportStream = async (opts) => {
240
464
  : effort.mode === "adaptive"
241
465
  ? adaptiveEffort
242
466
  : nativeFixedEffort(effort.mode);
467
+ const streamAbort = new AbortController();
468
+ const streamSignal = operationSignal === undefined
469
+ ? streamAbort.signal
470
+ : AbortSignal.any([operationSignal, streamAbort.signal]);
243
471
  const commonOptions = {
244
- signal: operationSignal,
472
+ signal: streamSignal,
245
473
  headers: requestHeaders,
246
474
  ...(temperature === undefined ? {} : { temperature }),
247
475
  ...(maxTokens === undefined ? {} : { maxTokens }),
@@ -255,9 +483,9 @@ export const executePiTransportStream = async (opts) => {
255
483
  : payload;
256
484
  const params = isRecord(transformed) ? { ...transformed } : {};
257
485
  if (effectiveModel.api === "openai-completions") {
258
- if (systemCacheBreakpoint
486
+ if (cacheBreakpoints !== undefined
259
487
  || messages.some((m) => Array.isArray(m.content) && m.content.some((p) => p.type === "file" && !p.mediaType.startsWith("image/")))) {
260
- params.messages = mapOpenAICompatibleMessages(messages, systemCacheBreakpoint);
488
+ params.messages = mapOpenAICompatibleMessages(messages, cacheBreakpoints, finalUserCachePrefix);
261
489
  }
262
490
  else if (messages[0]?.role === "system" && Array.isArray(params.messages)) {
263
491
  const first = params.messages[0];
@@ -283,7 +511,7 @@ export const executePiTransportStream = async (opts) => {
283
511
  params.top_k = topK;
284
512
  if (stopSequences !== undefined)
285
513
  params.stop_sequences = stopSequences;
286
- applyAnthropicPayloadAdjustments(params, messages, systemCacheBreakpoint, resolvedControls.serviceTier);
514
+ applyAnthropicPayloadAdjustments(params, messages, cacheBreakpoints, finalUserCachePrefix, resolvedControls.serviceTier);
287
515
  }
288
516
  else if (effectiveModel.api === "google-generative-ai") {
289
517
  const config = isRecord(params.config) ? { ...params.config } : {};
@@ -329,6 +557,7 @@ export const executePiTransportStream = async (opts) => {
329
557
  if (resolvedControls.serviceTier !== undefined) {
330
558
  params.serviceTier = { type: resolvedControls.serviceTier };
331
559
  }
560
+ applyBedrockPayloadAdjustments(params, messages, cacheBreakpoints, finalUserCachePrefix);
332
561
  try {
333
562
  capture?.record({
334
563
  method: "POST",
@@ -350,7 +579,10 @@ export const executePiTransportStream = async (opts) => {
350
579
  ? undefined
351
580
  : reasoning;
352
581
  const samplingParams = providerModel === undefined
353
- ? Object.fromEntries(Object.entries(body).filter(([, value]) => value !== undefined))
582
+ ? Object.fromEntries(Object.entries(body).filter(([key, value]) => value !== undefined
583
+ && key !== "temperature"
584
+ && key !== "max_tokens"
585
+ && key !== "max_completion_tokens"))
354
586
  : {
355
587
  ...(topP === undefined ? {} : { top_p: topP }),
356
588
  ...(topK !== undefined && effectiveModel.includeTopK === true ? { top_k: topK } : {}),
@@ -470,6 +702,7 @@ export const executePiTransportStream = async (opts) => {
470
702
  : (reasoning === "xhigh" ? "max" : reasoning);
471
703
  return streamBedrockConverseStream(buildPiModel("bedrock-converse-stream"), context, {
472
704
  ...commonOptions,
705
+ cacheRetention: "none",
473
706
  ...(bedrockReasoning === undefined ? {} : { reasoning: bedrockReasoning }),
474
707
  ...(nativeReasoningBudget !== null && bedrockReasoning !== undefined
475
708
  ? {
@@ -486,22 +719,41 @@ export const executePiTransportStream = async (opts) => {
486
719
  }
487
720
  }
488
721
  };
722
+ const grantedOutput = capacity.responseMax ?? capacity.outputBudget;
489
723
  let outputObserved = false;
724
+ let exceededOutputTokens;
490
725
  let finalMessage;
491
726
  let lastErrorMessage;
492
727
  let streamError;
728
+ const checkRunawayOutput = (partial) => {
729
+ if (grantedOutput === null)
730
+ return false;
731
+ const midStreamOutputTokens = Math.max(partial.usage.output, Math.floor(estimateMessageTokens(partial) / 2));
732
+ if (midStreamOutputTokens <= grantedOutput)
733
+ return false;
734
+ exceededOutputTokens = midStreamOutputTokens;
735
+ finalMessage = partial;
736
+ streamAbort.abort();
737
+ return true;
738
+ };
493
739
  const stream = startStream();
494
740
  for await (const event of stream) {
495
741
  if (event.type === "text_delta" && event.delta.length > 0) {
496
742
  outputObserved = true;
497
743
  observers.observeText?.(event.delta);
744
+ if (checkRunawayOutput(event.partial))
745
+ break;
498
746
  }
499
747
  else if (event.type === "thinking_delta" && event.delta.length > 0) {
500
748
  outputObserved = true;
501
749
  observers.observeReasoning?.(event.delta);
750
+ if (checkRunawayOutput(event.partial))
751
+ break;
502
752
  }
503
753
  else if (event.type === "toolcall_start" || event.type === "toolcall_delta" || event.type === "toolcall_end") {
504
754
  outputObserved = true;
755
+ if (checkRunawayOutput(event.partial))
756
+ break;
505
757
  }
506
758
  else if (event.type === "done") {
507
759
  finalMessage = event.message;
@@ -646,9 +898,12 @@ export const executePiTransportStream = async (opts) => {
646
898
  content: messageInfo.content,
647
899
  reasoning: messageInfo.reasoning,
648
900
  reasoningProjected: messageInfo.reasoningProjected,
649
- finishReason: messageInfo.finishReason,
901
+ finishReason: exceededOutputTokens !== undefined && messageInfo.finishReason === null
902
+ ? "length"
903
+ : messageInfo.finishReason,
650
904
  ...(messageInfo.rawFinishReason === undefined ? {} : { rawFinishReason: messageInfo.rawFinishReason }),
651
905
  usage,
906
+ ...(exceededOutputTokens === undefined ? {} : { exceededOutputTokens }),
652
907
  chargeEvidence,
653
908
  logprobs: forensics.logprobs,
654
909
  metadata: forensics.metadata,