@plurnk/plurnk-providers 6.0.3 → 6.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.defaults +4 -5
- package/README.md +31 -213
- package/SPEC.md +11 -7
- package/dist/InferenceAdmission.js.map +1 -1
- package/dist/LeadingReasoning.d.ts.map +1 -1
- package/dist/LeadingReasoning.js +12 -1
- package/dist/LeadingReasoning.js.map +1 -1
- package/dist/Mock.d.ts +5 -1
- package/dist/Mock.d.ts.map +1 -1
- package/dist/Mock.js +7 -1
- package/dist/Mock.js.map +1 -1
- package/dist/ModelProvider.d.ts +6 -3
- package/dist/ModelProvider.d.ts.map +1 -1
- package/dist/ModelProvider.js +24 -24
- package/dist/ModelProvider.js.map +1 -1
- package/dist/ModelRequestBody.js.map +1 -1
- package/dist/Pool.d.ts +2 -0
- package/dist/Pool.d.ts.map +1 -1
- package/dist/Pool.js +8 -0
- package/dist/Pool.js.map +1 -1
- package/dist/ProviderRegistry.d.ts.map +1 -1
- package/dist/ProviderRegistry.js +5 -10
- package/dist/ProviderRegistry.js.map +1 -1
- package/dist/RequestCapture.js.map +1 -1
- package/dist/accounting.js.map +1 -1
- package/dist/accountingPublic.js.map +1 -1
- package/dist/capacity.js.map +1 -1
- package/dist/catalogProvider.d.ts +2 -2
- package/dist/catalogProvider.d.ts.map +1 -1
- package/dist/catalogProvider.js +4 -4
- package/dist/catalogProvider.js.map +1 -1
- package/dist/compatibleProvider.js.map +1 -1
- package/dist/cost.js.map +1 -1
- package/dist/defaults.js.map +1 -1
- package/dist/discover.d.ts +0 -2
- package/dist/discover.d.ts.map +1 -1
- package/dist/discover.js +2 -8
- package/dist/discover.js.map +1 -1
- package/dist/env.d.ts +1 -1
- package/dist/env.d.ts.map +1 -1
- package/dist/env.js +2 -2
- package/dist/env.js.map +1 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +4 -1
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/modelAdapters.d.ts +1 -1
- package/dist/modelAdapters.d.ts.map +1 -1
- package/dist/modelAdapters.js +20 -4
- package/dist/modelAdapters.js.map +1 -1
- package/dist/notices.js.map +1 -1
- package/dist/ollama.js.map +1 -1
- package/dist/openai.js.map +1 -1
- package/dist/pi-transport.d.ts +6 -3
- package/dist/pi-transport.d.ts.map +1 -1
- package/dist/pi-transport.js +304 -49
- package/dist/pi-transport.js.map +1 -1
- package/dist/promptTokens.js.map +1 -1
- package/dist/provider-env.js.map +1 -1
- package/dist/providerError.js.map +1 -1
- package/dist/reasoning-effort.js.map +1 -1
- package/dist/retry-deadline.d.ts.map +1 -1
- package/dist/retry-deadline.js +10 -14
- package/dist/retry-deadline.js.map +1 -1
- package/dist/stream-forensics.d.ts.map +1 -1
- package/dist/stream-forensics.js +173 -25
- package/dist/stream-forensics.js.map +1 -1
- package/dist/types.d.ts +10 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/dist/usage.js.map +1 -1
- package/dist/warnings.js.map +1 -1
- package/docs/models.md +9 -15
- package/package.json +6 -6
package/dist/pi-transport.js
CHANGED
|
@@ -6,16 +6,37 @@ import { stream as streamBedrockConverseStream } from "@earendil-works/pi-ai/api
|
|
|
6
6
|
import { stream as streamGoogleGenerativeAi } from "@earendil-works/pi-ai/api/google-generative-ai";
|
|
7
7
|
import { stream as streamOpenAICompletions } from "@earendil-works/pi-ai/api/openai-completions";
|
|
8
8
|
import { stream as streamOpenAIResponses } from "@earendil-works/pi-ai/api/openai-responses";
|
|
9
|
+
import { getBuiltinModels, getBuiltinProviders, } from "@earendil-works/pi-ai/providers/all";
|
|
10
|
+
import { estimateMessageTokens } from "@earendil-works/pi-ai/utils/estimate";
|
|
11
|
+
import { headersToRecord } from "@earendil-works/pi-ai/utils/headers";
|
|
9
12
|
import { isRecord } from "@plurnk/plurnk-contracts";
|
|
10
13
|
import { outboundFetch, syncAmbientHttp } from "@plurnk/plurnk-meta";
|
|
11
14
|
import { APICallError } from "./errors.js";
|
|
12
15
|
import { nativeFixedEffort } from "./reasoning-effort.js";
|
|
13
16
|
import { abortFailure, attachFailureEvidence, collectChunkForensics, extractFromAssistantMessage, serializeError, toUsageFromPiMessage, } from "./stream-forensics.js";
|
|
14
|
-
const
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
17
|
+
const PI_BUILTIN_PROVIDER_ALIASES = {
|
|
18
|
+
"alibaba": "qwen-token-plan",
|
|
19
|
+
"alibaba-cn": "qwen-token-plan-cn",
|
|
20
|
+
"fireworks-ai": "fireworks",
|
|
21
|
+
"togetherai": "together",
|
|
22
|
+
"zai-coding-plan": "zai",
|
|
23
|
+
};
|
|
24
|
+
const lookupPiBuiltinCompat = (providerId, modelId) => {
|
|
25
|
+
const targetProvider = PI_BUILTIN_PROVIDER_ALIASES[providerId] ?? providerId;
|
|
26
|
+
const builtinProvider = getBuiltinProviders().find((candidate) => candidate === targetProvider);
|
|
27
|
+
if (builtinProvider === undefined)
|
|
28
|
+
return {};
|
|
29
|
+
const builtin = getBuiltinModels(builtinProvider).find((candidate) => candidate.id === modelId);
|
|
30
|
+
const rawCompat = builtin?.compat;
|
|
31
|
+
const compat = isRecord(rawCompat) ? rawCompat : undefined;
|
|
32
|
+
return {
|
|
33
|
+
...(typeof compat?.requiresReasoningContentOnAssistantMessages === "boolean"
|
|
34
|
+
? { requiresReasoningContentOnAssistantMessages: compat.requiresReasoningContentOnAssistantMessages }
|
|
35
|
+
: {}),
|
|
36
|
+
...(typeof compat?.zaiToolStream === "boolean"
|
|
37
|
+
? { zaiToolStream: compat.zaiToolStream }
|
|
38
|
+
: {}),
|
|
39
|
+
};
|
|
19
40
|
};
|
|
20
41
|
const toPiContext = (messages) => {
|
|
21
42
|
const systemParts = [];
|
|
@@ -68,57 +89,157 @@ const toPiContext = (messages) => {
|
|
|
68
89
|
messages: contextMessages,
|
|
69
90
|
});
|
|
70
91
|
};
|
|
71
|
-
const
|
|
92
|
+
const hasWireContent = (message) => typeof message.content === "string"
|
|
93
|
+
? message.content.trim().length > 0
|
|
94
|
+
: message.content.some((part) => part.type !== "text" || part.text.trim().length > 0);
|
|
95
|
+
const countWireParts = (parts, limit) => {
|
|
96
|
+
let count = 0;
|
|
97
|
+
const end = Math.min(limit, parts.length);
|
|
98
|
+
for (let i = 0; i < end; i += 1) {
|
|
99
|
+
const part = parts[i];
|
|
100
|
+
if (part.type !== "text" || part.text.trim().length > 0)
|
|
101
|
+
count += 1;
|
|
102
|
+
}
|
|
103
|
+
return count;
|
|
104
|
+
};
|
|
105
|
+
const toOpenAIPart = (part) => {
|
|
106
|
+
if (part.type === "text")
|
|
107
|
+
return { type: "text", text: part.text };
|
|
108
|
+
const base64 = Buffer.from(part.data).toString("base64");
|
|
109
|
+
if (part.mediaType.startsWith("audio/")) {
|
|
110
|
+
return {
|
|
111
|
+
type: "input_audio",
|
|
112
|
+
input_audio: {
|
|
113
|
+
data: base64,
|
|
114
|
+
format: part.mediaType.slice("audio/".length),
|
|
115
|
+
},
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
return {
|
|
119
|
+
type: "image_url",
|
|
120
|
+
image_url: { url: `data:${part.mediaType};base64,${base64}` },
|
|
121
|
+
};
|
|
122
|
+
};
|
|
123
|
+
const withEphemeralCacheControl = (parts, atIndex) => {
|
|
124
|
+
if (atIndex >= 0 && atIndex < parts.length) {
|
|
125
|
+
parts[atIndex] = { ...parts[atIndex], cache_control: { type: "ephemeral" } };
|
|
126
|
+
}
|
|
127
|
+
return parts;
|
|
128
|
+
};
|
|
129
|
+
const mapOpenAICompatibleMessages = (messages, cacheBreakpoints, finalUserCachePrefix) => {
|
|
72
130
|
let lastLeadingSystemIndex = -1;
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
131
|
+
let firstNonSystemIndex = messages.length;
|
|
132
|
+
for (let i = 0; i < messages.length; i += 1) {
|
|
133
|
+
if (messages[i].role !== "system") {
|
|
134
|
+
firstNonSystemIndex = i;
|
|
135
|
+
break;
|
|
136
|
+
}
|
|
137
|
+
if (cacheBreakpoints !== undefined && hasWireContent(messages[i])) {
|
|
77
138
|
lastLeadingSystemIndex = i;
|
|
78
139
|
}
|
|
79
140
|
}
|
|
141
|
+
let penultimateNonSystemIndex = -1;
|
|
142
|
+
let finalUserIndex = -1;
|
|
143
|
+
if (cacheBreakpoints === "conversation") {
|
|
144
|
+
const nonSystemIndices = [];
|
|
145
|
+
for (let i = firstNonSystemIndex; i < messages.length; i += 1) {
|
|
146
|
+
if (hasWireContent(messages[i]))
|
|
147
|
+
nonSystemIndices.push(i);
|
|
148
|
+
}
|
|
149
|
+
if (nonSystemIndices.length >= 2) {
|
|
150
|
+
penultimateNonSystemIndex = nonSystemIndices[nonSystemIndices.length - 2];
|
|
151
|
+
}
|
|
152
|
+
const lastIdx = nonSystemIndices.at(-1);
|
|
153
|
+
if (lastIdx !== undefined && messages[lastIdx].role === "user") {
|
|
154
|
+
finalUserIndex = lastIdx;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
80
157
|
return messages.map((message, index) => {
|
|
81
|
-
if (index === lastLeadingSystemIndex
|
|
158
|
+
if (index === lastLeadingSystemIndex || index === penultimateNonSystemIndex) {
|
|
159
|
+
const parts = typeof message.content === "string"
|
|
160
|
+
? [{ type: "text", text: message.content }]
|
|
161
|
+
: message.content.map(toOpenAIPart);
|
|
82
162
|
return {
|
|
83
163
|
role: message.role,
|
|
84
|
-
content:
|
|
85
|
-
type: "text",
|
|
86
|
-
text: message.content,
|
|
87
|
-
cache_control: { type: "ephemeral" },
|
|
88
|
-
}],
|
|
164
|
+
content: withEphemeralCacheControl(parts, parts.length - 1),
|
|
89
165
|
};
|
|
90
166
|
}
|
|
91
|
-
if (
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
167
|
+
if (index === finalUserIndex) {
|
|
168
|
+
if (finalUserCachePrefix === undefined) {
|
|
169
|
+
const parts = typeof message.content === "string"
|
|
170
|
+
? [{ type: "text", text: message.content }]
|
|
171
|
+
: message.content.map(toOpenAIPart);
|
|
172
|
+
return {
|
|
173
|
+
role: message.role,
|
|
174
|
+
content: withEphemeralCacheControl(parts, parts.length - 1),
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
if (finalUserCachePrefix.kind === "chars" && typeof message.content === "string") {
|
|
178
|
+
if (finalUserCachePrefix.length <= 0) {
|
|
179
|
+
return { role: message.role, content: message.content };
|
|
180
|
+
}
|
|
181
|
+
if (finalUserCachePrefix.length >= message.content.length) {
|
|
101
182
|
return {
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
data: base64,
|
|
105
|
-
format: part.mediaType.slice("audio/".length),
|
|
106
|
-
},
|
|
183
|
+
role: message.role,
|
|
184
|
+
content: [{ type: "text", text: message.content, cache_control: { type: "ephemeral" } }],
|
|
107
185
|
};
|
|
108
186
|
}
|
|
187
|
+
const durable = message.content.slice(0, finalUserCachePrefix.length);
|
|
188
|
+
const volatile = message.content.slice(finalUserCachePrefix.length);
|
|
189
|
+
if (durable.trim().length === 0) {
|
|
190
|
+
return { role: message.role, content: message.content };
|
|
191
|
+
}
|
|
109
192
|
return {
|
|
110
|
-
|
|
111
|
-
|
|
193
|
+
role: message.role,
|
|
194
|
+
content: volatile.trim().length > 0
|
|
195
|
+
? [
|
|
196
|
+
{ type: "text", text: durable, cache_control: { type: "ephemeral" } },
|
|
197
|
+
{ type: "text", text: volatile },
|
|
198
|
+
]
|
|
199
|
+
: [{ type: "text", text: durable, cache_control: { type: "ephemeral" } }],
|
|
112
200
|
};
|
|
113
|
-
}
|
|
201
|
+
}
|
|
202
|
+
if (finalUserCachePrefix.kind === "parts" && Array.isArray(message.content)) {
|
|
203
|
+
const parts = message.content.map(toOpenAIPart);
|
|
204
|
+
const targetIndex = Math.min(finalUserCachePrefix.count, parts.length) - 1;
|
|
205
|
+
return {
|
|
206
|
+
role: message.role,
|
|
207
|
+
content: finalUserCachePrefix.count > 0
|
|
208
|
+
? withEphemeralCacheControl(parts, targetIndex)
|
|
209
|
+
: parts,
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
if (typeof message.content === "string") {
|
|
214
|
+
return { role: message.role, content: message.content };
|
|
215
|
+
}
|
|
216
|
+
return {
|
|
217
|
+
role: message.role,
|
|
218
|
+
content: message.content.map(toOpenAIPart),
|
|
114
219
|
};
|
|
115
220
|
});
|
|
116
221
|
};
|
|
117
|
-
const
|
|
222
|
+
const markAnthropicMessageEnd = (wireMsg) => {
|
|
223
|
+
if (!isRecord(wireMsg))
|
|
224
|
+
return;
|
|
225
|
+
if (typeof wireMsg.content === "string") {
|
|
226
|
+
if (wireMsg.content.trim().length > 0) {
|
|
227
|
+
wireMsg.content = [{ type: "text", text: wireMsg.content, cache_control: { type: "ephemeral" } }];
|
|
228
|
+
}
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
if (Array.isArray(wireMsg.content) && wireMsg.content.length > 0) {
|
|
232
|
+
const last = wireMsg.content.at(-1);
|
|
233
|
+
if (isRecord(last)) {
|
|
234
|
+
last.cache_control = { type: "ephemeral" };
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
};
|
|
238
|
+
const applyAnthropicPayloadAdjustments = (params, messages, cacheBreakpoints, finalUserCachePrefix, serviceTier) => {
|
|
118
239
|
if (serviceTier !== undefined) {
|
|
119
240
|
params.service_tier = serviceTier;
|
|
120
241
|
}
|
|
121
|
-
if (
|
|
242
|
+
if (cacheBreakpoints !== undefined && Array.isArray(params.system) && params.system.length > 0) {
|
|
122
243
|
const last = params.system.at(-1);
|
|
123
244
|
if (isRecord(last)) {
|
|
124
245
|
last.cache_control = { type: "ephemeral" };
|
|
@@ -127,14 +248,15 @@ const applyAnthropicPayloadAdjustments = (params, messages, systemCacheBreakpoin
|
|
|
127
248
|
const wireMessages = Array.isArray(params.messages) ? params.messages : undefined;
|
|
128
249
|
if (wireMessages === undefined)
|
|
129
250
|
return;
|
|
130
|
-
const
|
|
131
|
-
for (let i = 0; i < Math.min(wireMessages.length,
|
|
132
|
-
const sourceMsg =
|
|
251
|
+
const effectiveNonSystem = messages.filter((m) => m.role !== "system" && hasWireContent(m));
|
|
252
|
+
for (let i = 0; i < Math.min(wireMessages.length, effectiveNonSystem.length); i += 1) {
|
|
253
|
+
const sourceMsg = effectiveNonSystem[i];
|
|
133
254
|
const wireMsg = wireMessages[i];
|
|
134
255
|
if (!Array.isArray(sourceMsg.content) || !isRecord(wireMsg) || !Array.isArray(wireMsg.content))
|
|
135
256
|
continue;
|
|
136
|
-
|
|
137
|
-
|
|
257
|
+
const nonBlankSourceParts = sourceMsg.content.filter((p) => p.type !== "text" || p.text.trim().length > 0);
|
|
258
|
+
for (let j = 0; j < Math.min(nonBlankSourceParts.length, wireMsg.content.length); j += 1) {
|
|
259
|
+
const sourcePart = nonBlankSourceParts[j];
|
|
138
260
|
const wireBlock = wireMsg.content[j];
|
|
139
261
|
if (sourcePart.type === "file"
|
|
140
262
|
&& sourcePart.mediaType === "application/pdf"
|
|
@@ -144,19 +266,121 @@ const applyAnthropicPayloadAdjustments = (params, messages, systemCacheBreakpoin
|
|
|
144
266
|
}
|
|
145
267
|
}
|
|
146
268
|
}
|
|
269
|
+
if (cacheBreakpoints !== "conversation" || wireMessages.length === 0)
|
|
270
|
+
return;
|
|
271
|
+
if (wireMessages.length >= 2) {
|
|
272
|
+
markAnthropicMessageEnd(wireMessages[wireMessages.length - 2]);
|
|
273
|
+
}
|
|
274
|
+
const lastWire = wireMessages.at(-1);
|
|
275
|
+
const lastSource = effectiveNonSystem.at(-1);
|
|
276
|
+
if (lastSource?.role !== "user" || !isRecord(lastWire) || lastWire.role !== "user")
|
|
277
|
+
return;
|
|
278
|
+
if (finalUserCachePrefix === undefined) {
|
|
279
|
+
markAnthropicMessageEnd(lastWire);
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
282
|
+
if (finalUserCachePrefix.kind === "chars" && typeof lastSource.content === "string") {
|
|
283
|
+
if (finalUserCachePrefix.length <= 0)
|
|
284
|
+
return;
|
|
285
|
+
if (finalUserCachePrefix.length >= lastSource.content.length) {
|
|
286
|
+
markAnthropicMessageEnd(lastWire);
|
|
287
|
+
return;
|
|
288
|
+
}
|
|
289
|
+
const durable = lastSource.content.slice(0, finalUserCachePrefix.length);
|
|
290
|
+
const volatile = lastSource.content.slice(finalUserCachePrefix.length);
|
|
291
|
+
if (durable.trim().length === 0)
|
|
292
|
+
return;
|
|
293
|
+
lastWire.content = volatile.trim().length > 0
|
|
294
|
+
? [
|
|
295
|
+
{ type: "text", text: durable, cache_control: { type: "ephemeral" } },
|
|
296
|
+
{ type: "text", text: volatile },
|
|
297
|
+
]
|
|
298
|
+
: [{ type: "text", text: durable, cache_control: { type: "ephemeral" } }];
|
|
299
|
+
return;
|
|
300
|
+
}
|
|
301
|
+
if (finalUserCachePrefix.kind === "parts" && Array.isArray(lastSource.content) && Array.isArray(lastWire.content)) {
|
|
302
|
+
const k = countWireParts(lastSource.content, finalUserCachePrefix.count);
|
|
303
|
+
if (k > 0 && k <= lastWire.content.length) {
|
|
304
|
+
const target = lastWire.content[k - 1];
|
|
305
|
+
if (isRecord(target))
|
|
306
|
+
target.cache_control = { type: "ephemeral" };
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
};
|
|
310
|
+
const appendBedrockCachePoint = (wireMsg, sourceMsg) => {
|
|
311
|
+
if (sourceMsg === undefined || !hasWireContent(sourceMsg))
|
|
312
|
+
return;
|
|
313
|
+
if (!isRecord(wireMsg) || !Array.isArray(wireMsg.content) || wireMsg.content.length === 0)
|
|
314
|
+
return;
|
|
315
|
+
wireMsg.content = [...wireMsg.content, { cachePoint: { type: "default" } }];
|
|
316
|
+
};
|
|
317
|
+
const applyBedrockPayloadAdjustments = (params, messages, cacheBreakpoints, finalUserCachePrefix) => {
|
|
318
|
+
if (cacheBreakpoints === undefined)
|
|
319
|
+
return;
|
|
320
|
+
if (Array.isArray(params.system) && params.system.length > 0) {
|
|
321
|
+
params.system = [...params.system, { cachePoint: { type: "default" } }];
|
|
322
|
+
}
|
|
323
|
+
if (cacheBreakpoints !== "conversation")
|
|
324
|
+
return;
|
|
325
|
+
const wireMessages = Array.isArray(params.messages) ? params.messages : undefined;
|
|
326
|
+
if (wireMessages === undefined || wireMessages.length === 0)
|
|
327
|
+
return;
|
|
328
|
+
const bedrockNonSystem = messages.filter((m) => m.role === "user" || (m.role === "assistant" && hasWireContent(m)));
|
|
329
|
+
if (wireMessages.length >= 2) {
|
|
330
|
+
appendBedrockCachePoint(wireMessages[wireMessages.length - 2], bedrockNonSystem[bedrockNonSystem.length - 2]);
|
|
331
|
+
}
|
|
332
|
+
const lastWire = wireMessages.at(-1);
|
|
333
|
+
const lastSource = bedrockNonSystem.at(-1);
|
|
334
|
+
if (lastSource?.role !== "user" || !hasWireContent(lastSource) || !isRecord(lastWire) || lastWire.role !== "user" || !Array.isArray(lastWire.content)) {
|
|
335
|
+
return;
|
|
336
|
+
}
|
|
337
|
+
if (finalUserCachePrefix === undefined) {
|
|
338
|
+
appendBedrockCachePoint(lastWire, lastSource);
|
|
339
|
+
return;
|
|
340
|
+
}
|
|
341
|
+
if (finalUserCachePrefix.kind === "chars" && typeof lastSource.content === "string") {
|
|
342
|
+
if (finalUserCachePrefix.length <= 0)
|
|
343
|
+
return;
|
|
344
|
+
if (finalUserCachePrefix.length >= lastSource.content.length) {
|
|
345
|
+
appendBedrockCachePoint(lastWire, lastSource);
|
|
346
|
+
return;
|
|
347
|
+
}
|
|
348
|
+
const durable = lastSource.content.slice(0, finalUserCachePrefix.length);
|
|
349
|
+
const volatile = lastSource.content.slice(finalUserCachePrefix.length);
|
|
350
|
+
if (durable.trim().length === 0)
|
|
351
|
+
return;
|
|
352
|
+
lastWire.content = volatile.trim().length > 0
|
|
353
|
+
? [{ text: durable }, { cachePoint: { type: "default" } }, { text: volatile }]
|
|
354
|
+
: [{ text: durable }, { cachePoint: { type: "default" } }];
|
|
355
|
+
return;
|
|
356
|
+
}
|
|
357
|
+
if (finalUserCachePrefix.kind === "parts" && Array.isArray(lastSource.content)) {
|
|
358
|
+
const k = countWireParts(lastSource.content, finalUserCachePrefix.count);
|
|
359
|
+
if (k > 0 && k <= lastWire.content.length) {
|
|
360
|
+
lastWire.content = [
|
|
361
|
+
...lastWire.content.slice(0, k),
|
|
362
|
+
{ cachePoint: { type: "default" } },
|
|
363
|
+
...lastWire.content.slice(k),
|
|
364
|
+
];
|
|
365
|
+
}
|
|
366
|
+
}
|
|
147
367
|
};
|
|
148
368
|
export const executePiTransportStream = async (opts) => {
|
|
149
|
-
const { model, source, url, providerModel, fetch: configuredFetch, requestHeaders, messages, sessionId, operationSignal, requestSampling, body, effectiveMaxOutputTokens, nativeReasoningBudget, capacity, requestBody, effort, adaptiveEffort, nativeAdaptiveThinking, reasoningRequestBody,
|
|
369
|
+
const { model, source, url, providerModel, fetch: configuredFetch, requestHeaders, messages, sessionId, operationSignal, requestSampling, body, outputTokenField = "max_tokens", effectiveMaxOutputTokens, nativeReasoningBudget, capacity, requestBody, effort, adaptiveEffort, nativeAdaptiveThinking, reasoningRequestBody, cacheBreakpoints, finalUserCachePrefix, cacheAffinity, resolvedControls, rawBody, capture, reasoningStepDown = 0, observers, } = opts;
|
|
370
|
+
const providerId = source.replace(/^provider:/, "");
|
|
150
371
|
const effectiveModel = providerModel ?? {
|
|
151
372
|
id: model,
|
|
152
373
|
api: "openai-completions",
|
|
153
|
-
provider:
|
|
374
|
+
provider: providerId,
|
|
154
375
|
baseUrl: url.replace(/\/chat\/completions\/?$/, ""),
|
|
155
376
|
compat: {
|
|
377
|
+
...lookupPiBuiltinCompat(providerId, model),
|
|
156
378
|
supportsStore: false,
|
|
157
379
|
supportsDeveloperRole: false,
|
|
158
380
|
supportsUsageInStreaming: true,
|
|
159
381
|
supportsFinishReason: true,
|
|
382
|
+
maxTokensField: outputTokenField,
|
|
383
|
+
thinkingFormat: "openai",
|
|
160
384
|
},
|
|
161
385
|
};
|
|
162
386
|
const buildPiModel = (api) => ({
|
|
@@ -240,8 +464,12 @@ export const executePiTransportStream = async (opts) => {
|
|
|
240
464
|
: effort.mode === "adaptive"
|
|
241
465
|
? adaptiveEffort
|
|
242
466
|
: nativeFixedEffort(effort.mode);
|
|
467
|
+
const streamAbort = new AbortController();
|
|
468
|
+
const streamSignal = operationSignal === undefined
|
|
469
|
+
? streamAbort.signal
|
|
470
|
+
: AbortSignal.any([operationSignal, streamAbort.signal]);
|
|
243
471
|
const commonOptions = {
|
|
244
|
-
signal:
|
|
472
|
+
signal: streamSignal,
|
|
245
473
|
headers: requestHeaders,
|
|
246
474
|
...(temperature === undefined ? {} : { temperature }),
|
|
247
475
|
...(maxTokens === undefined ? {} : { maxTokens }),
|
|
@@ -255,9 +483,9 @@ export const executePiTransportStream = async (opts) => {
|
|
|
255
483
|
: payload;
|
|
256
484
|
const params = isRecord(transformed) ? { ...transformed } : {};
|
|
257
485
|
if (effectiveModel.api === "openai-completions") {
|
|
258
|
-
if (
|
|
486
|
+
if (cacheBreakpoints !== undefined
|
|
259
487
|
|| messages.some((m) => Array.isArray(m.content) && m.content.some((p) => p.type === "file" && !p.mediaType.startsWith("image/")))) {
|
|
260
|
-
params.messages = mapOpenAICompatibleMessages(messages,
|
|
488
|
+
params.messages = mapOpenAICompatibleMessages(messages, cacheBreakpoints, finalUserCachePrefix);
|
|
261
489
|
}
|
|
262
490
|
else if (messages[0]?.role === "system" && Array.isArray(params.messages)) {
|
|
263
491
|
const first = params.messages[0];
|
|
@@ -283,7 +511,7 @@ export const executePiTransportStream = async (opts) => {
|
|
|
283
511
|
params.top_k = topK;
|
|
284
512
|
if (stopSequences !== undefined)
|
|
285
513
|
params.stop_sequences = stopSequences;
|
|
286
|
-
applyAnthropicPayloadAdjustments(params, messages,
|
|
514
|
+
applyAnthropicPayloadAdjustments(params, messages, cacheBreakpoints, finalUserCachePrefix, resolvedControls.serviceTier);
|
|
287
515
|
}
|
|
288
516
|
else if (effectiveModel.api === "google-generative-ai") {
|
|
289
517
|
const config = isRecord(params.config) ? { ...params.config } : {};
|
|
@@ -329,6 +557,7 @@ export const executePiTransportStream = async (opts) => {
|
|
|
329
557
|
if (resolvedControls.serviceTier !== undefined) {
|
|
330
558
|
params.serviceTier = { type: resolvedControls.serviceTier };
|
|
331
559
|
}
|
|
560
|
+
applyBedrockPayloadAdjustments(params, messages, cacheBreakpoints, finalUserCachePrefix);
|
|
332
561
|
try {
|
|
333
562
|
capture?.record({
|
|
334
563
|
method: "POST",
|
|
@@ -350,7 +579,10 @@ export const executePiTransportStream = async (opts) => {
|
|
|
350
579
|
? undefined
|
|
351
580
|
: reasoning;
|
|
352
581
|
const samplingParams = providerModel === undefined
|
|
353
|
-
? Object.fromEntries(Object.entries(body).filter(([, value]) => value !== undefined
|
|
582
|
+
? Object.fromEntries(Object.entries(body).filter(([key, value]) => value !== undefined
|
|
583
|
+
&& key !== "temperature"
|
|
584
|
+
&& key !== "max_tokens"
|
|
585
|
+
&& key !== "max_completion_tokens"))
|
|
354
586
|
: {
|
|
355
587
|
...(topP === undefined ? {} : { top_p: topP }),
|
|
356
588
|
...(topK !== undefined && effectiveModel.includeTopK === true ? { top_k: topK } : {}),
|
|
@@ -470,6 +702,7 @@ export const executePiTransportStream = async (opts) => {
|
|
|
470
702
|
: (reasoning === "xhigh" ? "max" : reasoning);
|
|
471
703
|
return streamBedrockConverseStream(buildPiModel("bedrock-converse-stream"), context, {
|
|
472
704
|
...commonOptions,
|
|
705
|
+
cacheRetention: "none",
|
|
473
706
|
...(bedrockReasoning === undefined ? {} : { reasoning: bedrockReasoning }),
|
|
474
707
|
...(nativeReasoningBudget !== null && bedrockReasoning !== undefined
|
|
475
708
|
? {
|
|
@@ -486,22 +719,41 @@ export const executePiTransportStream = async (opts) => {
|
|
|
486
719
|
}
|
|
487
720
|
}
|
|
488
721
|
};
|
|
722
|
+
const grantedOutput = capacity.responseMax ?? capacity.outputBudget;
|
|
489
723
|
let outputObserved = false;
|
|
724
|
+
let exceededOutputTokens;
|
|
490
725
|
let finalMessage;
|
|
491
726
|
let lastErrorMessage;
|
|
492
727
|
let streamError;
|
|
728
|
+
const checkRunawayOutput = (partial) => {
|
|
729
|
+
if (grantedOutput === null)
|
|
730
|
+
return false;
|
|
731
|
+
const midStreamOutputTokens = Math.max(partial.usage.output, Math.floor(estimateMessageTokens(partial) / 2));
|
|
732
|
+
if (midStreamOutputTokens <= grantedOutput)
|
|
733
|
+
return false;
|
|
734
|
+
exceededOutputTokens = midStreamOutputTokens;
|
|
735
|
+
finalMessage = partial;
|
|
736
|
+
streamAbort.abort();
|
|
737
|
+
return true;
|
|
738
|
+
};
|
|
493
739
|
const stream = startStream();
|
|
494
740
|
for await (const event of stream) {
|
|
495
741
|
if (event.type === "text_delta" && event.delta.length > 0) {
|
|
496
742
|
outputObserved = true;
|
|
497
743
|
observers.observeText?.(event.delta);
|
|
744
|
+
if (checkRunawayOutput(event.partial))
|
|
745
|
+
break;
|
|
498
746
|
}
|
|
499
747
|
else if (event.type === "thinking_delta" && event.delta.length > 0) {
|
|
500
748
|
outputObserved = true;
|
|
501
749
|
observers.observeReasoning?.(event.delta);
|
|
750
|
+
if (checkRunawayOutput(event.partial))
|
|
751
|
+
break;
|
|
502
752
|
}
|
|
503
753
|
else if (event.type === "toolcall_start" || event.type === "toolcall_delta" || event.type === "toolcall_end") {
|
|
504
754
|
outputObserved = true;
|
|
755
|
+
if (checkRunawayOutput(event.partial))
|
|
756
|
+
break;
|
|
505
757
|
}
|
|
506
758
|
else if (event.type === "done") {
|
|
507
759
|
finalMessage = event.message;
|
|
@@ -646,9 +898,12 @@ export const executePiTransportStream = async (opts) => {
|
|
|
646
898
|
content: messageInfo.content,
|
|
647
899
|
reasoning: messageInfo.reasoning,
|
|
648
900
|
reasoningProjected: messageInfo.reasoningProjected,
|
|
649
|
-
finishReason: messageInfo.finishReason
|
|
901
|
+
finishReason: exceededOutputTokens !== undefined && messageInfo.finishReason === null
|
|
902
|
+
? "length"
|
|
903
|
+
: messageInfo.finishReason,
|
|
650
904
|
...(messageInfo.rawFinishReason === undefined ? {} : { rawFinishReason: messageInfo.rawFinishReason }),
|
|
651
905
|
usage,
|
|
906
|
+
...(exceededOutputTokens === undefined ? {} : { exceededOutputTokens }),
|
|
652
907
|
chargeEvidence,
|
|
653
908
|
logprobs: forensics.logprobs,
|
|
654
909
|
metadata: forensics.metadata,
|