@plurnk/plurnk-providers 6.1.0 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.defaults +7 -7
- package/README.md +31 -213
- package/SPEC.md +15 -10
- package/dist/InferenceAdmission.d.ts +10 -1
- package/dist/InferenceAdmission.d.ts.map +1 -1
- package/dist/InferenceAdmission.js +79 -15
- package/dist/InferenceAdmission.js.map +1 -1
- package/dist/LeadingReasoning.js.map +1 -1
- package/dist/Mock.d.ts +5 -1
- package/dist/Mock.d.ts.map +1 -1
- package/dist/Mock.js +7 -1
- package/dist/Mock.js.map +1 -1
- package/dist/ModelProvider.d.ts +8 -3
- package/dist/ModelProvider.d.ts.map +1 -1
- package/dist/ModelProvider.js +14 -8
- package/dist/ModelProvider.js.map +1 -1
- package/dist/ModelRequestBody.js.map +1 -1
- package/dist/Pool.d.ts +2 -0
- package/dist/Pool.d.ts.map +1 -1
- package/dist/Pool.js +8 -0
- package/dist/Pool.js.map +1 -1
- package/dist/ProviderRegistry.d.ts.map +1 -1
- package/dist/ProviderRegistry.js +5 -10
- package/dist/ProviderRegistry.js.map +1 -1
- package/dist/RequestCapture.js.map +1 -1
- package/dist/accounting.js.map +1 -1
- package/dist/accountingPublic.js.map +1 -1
- package/dist/capacity.js.map +1 -1
- package/dist/catalogProvider.d.ts +3 -3
- package/dist/catalogProvider.d.ts.map +1 -1
- package/dist/catalogProvider.js +5 -4
- package/dist/catalogProvider.js.map +1 -1
- package/dist/compatibleProvider.d.ts.map +1 -1
- package/dist/compatibleProvider.js +6 -3
- package/dist/compatibleProvider.js.map +1 -1
- package/dist/cost.js.map +1 -1
- package/dist/defaults.js.map +1 -1
- package/dist/discover.d.ts +0 -2
- package/dist/discover.d.ts.map +1 -1
- package/dist/discover.js +2 -8
- package/dist/discover.js.map +1 -1
- package/dist/env.d.ts +4 -2
- package/dist/env.d.ts.map +1 -1
- package/dist/env.js +26 -4
- package/dist/env.js.map +1 -1
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/modelAdapters.d.ts +2 -2
- package/dist/modelAdapters.d.ts.map +1 -1
- package/dist/modelAdapters.js +68 -4
- package/dist/modelAdapters.js.map +1 -1
- package/dist/notices.js.map +1 -1
- package/dist/ollama.js.map +1 -1
- package/dist/openai.js.map +1 -1
- package/dist/pi-transport.d.ts +6 -3
- package/dist/pi-transport.d.ts.map +1 -1
- package/dist/pi-transport.js +436 -38
- package/dist/pi-transport.js.map +1 -1
- package/dist/promptTokens.js.map +1 -1
- package/dist/provider-env.js.map +1 -1
- package/dist/providerError.js.map +1 -1
- package/dist/reasoning-effort.js.map +1 -1
- package/dist/retry-deadline.js.map +1 -1
- package/dist/stream-forensics.d.ts.map +1 -1
- package/dist/stream-forensics.js +70 -0
- package/dist/stream-forensics.js.map +1 -1
- package/dist/types.d.ts +10 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/dist/usage.js.map +1 -1
- package/dist/warnings.js.map +1 -1
- package/docs/models.md +7 -17
- package/package.json +5 -5
package/dist/pi-transport.js
CHANGED
|
@@ -89,57 +89,289 @@ const toPiContext = (messages) => {
|
|
|
89
89
|
messages: contextMessages,
|
|
90
90
|
});
|
|
91
91
|
};
|
|
92
|
-
const
|
|
92
|
+
const hasWireContent = (message) => typeof message.content === "string"
|
|
93
|
+
? message.content.trim().length > 0
|
|
94
|
+
: message.content.some((part) => part.type !== "text" || part.text.trim().length > 0);
|
|
95
|
+
const countWireParts = (parts, limit) => {
|
|
96
|
+
let count = 0;
|
|
97
|
+
const end = Math.min(limit, parts.length);
|
|
98
|
+
for (let i = 0; i < end; i += 1) {
|
|
99
|
+
const part = parts[i];
|
|
100
|
+
if (part.type !== "text" || part.text.trim().length > 0)
|
|
101
|
+
count += 1;
|
|
102
|
+
}
|
|
103
|
+
return count;
|
|
104
|
+
};
|
|
105
|
+
const toOpenAIPart = (part) => {
|
|
106
|
+
if (part.type === "text")
|
|
107
|
+
return { type: "text", text: part.text };
|
|
108
|
+
const base64 = Buffer.from(part.data).toString("base64");
|
|
109
|
+
if (part.mediaType.startsWith("audio/")) {
|
|
110
|
+
return {
|
|
111
|
+
type: "input_audio",
|
|
112
|
+
input_audio: {
|
|
113
|
+
data: base64,
|
|
114
|
+
format: part.mediaType.slice("audio/".length),
|
|
115
|
+
},
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
return {
|
|
119
|
+
type: "image_url",
|
|
120
|
+
image_url: { url: `data:${part.mediaType};base64,${base64}` },
|
|
121
|
+
};
|
|
122
|
+
};
|
|
123
|
+
export const formatCacheAffinityValue = (affinity, sessionId) => {
|
|
124
|
+
if (affinity.name !== "prompt_cache_key")
|
|
125
|
+
return sessionId;
|
|
126
|
+
const chars = Array.from(sessionId);
|
|
127
|
+
return chars.length <= 64 ? sessionId : chars.slice(0, 64).join("");
|
|
128
|
+
};
|
|
129
|
+
const firstNonSystemMessageIndex = (messages) => {
|
|
130
|
+
for (let i = 0; i < messages.length; i += 1) {
|
|
131
|
+
if (messages[i].role !== "system")
|
|
132
|
+
return i;
|
|
133
|
+
}
|
|
134
|
+
return messages.length;
|
|
135
|
+
};
|
|
136
|
+
const splitHistoricalUserChars = (text, prefix) => {
|
|
137
|
+
if (prefix?.kind !== "chars" || prefix.length <= 0 || prefix.length >= text.length)
|
|
138
|
+
return null;
|
|
139
|
+
const durable = text.slice(0, prefix.length);
|
|
140
|
+
const tail = text.slice(prefix.length);
|
|
141
|
+
return durable.trim().length > 0 && tail.trim().length > 0 ? [durable, tail] : null;
|
|
142
|
+
};
|
|
143
|
+
const withEphemeralCacheControl = (parts, atIndex) => {
|
|
144
|
+
if (atIndex >= 0 && atIndex < parts.length) {
|
|
145
|
+
parts[atIndex] = { ...parts[atIndex], cache_control: { type: "ephemeral" } };
|
|
146
|
+
}
|
|
147
|
+
return parts;
|
|
148
|
+
};
|
|
149
|
+
const withExplicitCacheBreakpoint = (parts, atIndex) => {
|
|
150
|
+
if (atIndex >= 0 && atIndex < parts.length) {
|
|
151
|
+
parts[atIndex] = { ...parts[atIndex], prompt_cache_breakpoint: { mode: "explicit" } };
|
|
152
|
+
}
|
|
153
|
+
return parts;
|
|
154
|
+
};
|
|
155
|
+
const mapOpenAIExplicitCacheMessages = (messages, cacheBreakpoints, userCachePrefixes) => {
|
|
93
156
|
let lastLeadingSystemIndex = -1;
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
break;
|
|
157
|
+
const firstNonSystemIndex = firstNonSystemMessageIndex(messages);
|
|
158
|
+
for (let i = 0; i < firstNonSystemIndex; i += 1) {
|
|
159
|
+
if (cacheBreakpoints !== undefined && hasWireContent(messages[i])) {
|
|
98
160
|
lastLeadingSystemIndex = i;
|
|
99
161
|
}
|
|
100
162
|
}
|
|
163
|
+
const unscopedConversation = cacheBreakpoints === "conversation" && userCachePrefixes === undefined;
|
|
101
164
|
return messages.map((message, index) => {
|
|
102
|
-
if (index === lastLeadingSystemIndex
|
|
165
|
+
if (index === lastLeadingSystemIndex) {
|
|
166
|
+
const parts = typeof message.content === "string"
|
|
167
|
+
? [{ type: "text", text: message.content }]
|
|
168
|
+
: message.content.map(toOpenAIPart);
|
|
103
169
|
return {
|
|
104
170
|
role: message.role,
|
|
105
|
-
content:
|
|
106
|
-
type: "text",
|
|
107
|
-
text: message.content,
|
|
108
|
-
cache_control: { type: "ephemeral" },
|
|
109
|
-
}],
|
|
171
|
+
content: withExplicitCacheBreakpoint(parts, parts.length - 1),
|
|
110
172
|
};
|
|
111
173
|
}
|
|
174
|
+
if (cacheBreakpoints === "conversation" && index >= firstNonSystemIndex && message.role === "user") {
|
|
175
|
+
const prefix = userCachePrefixes?.[index];
|
|
176
|
+
if (prefix === undefined) {
|
|
177
|
+
if (unscopedConversation && hasWireContent(message)) {
|
|
178
|
+
const parts = typeof message.content === "string"
|
|
179
|
+
? [{ type: "text", text: message.content }]
|
|
180
|
+
: message.content.map(toOpenAIPart);
|
|
181
|
+
return {
|
|
182
|
+
role: message.role,
|
|
183
|
+
content: withExplicitCacheBreakpoint(parts, parts.length - 1),
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
else if (prefix.kind === "chars" && typeof message.content === "string") {
|
|
188
|
+
if (prefix.length <= 0) {
|
|
189
|
+
return { role: message.role, content: message.content };
|
|
190
|
+
}
|
|
191
|
+
if (prefix.length >= message.content.length) {
|
|
192
|
+
return message.content.trim().length > 0
|
|
193
|
+
? {
|
|
194
|
+
role: message.role,
|
|
195
|
+
content: [{
|
|
196
|
+
type: "text",
|
|
197
|
+
text: message.content,
|
|
198
|
+
prompt_cache_breakpoint: { mode: "explicit" },
|
|
199
|
+
}],
|
|
200
|
+
}
|
|
201
|
+
: { role: message.role, content: message.content };
|
|
202
|
+
}
|
|
203
|
+
const durable = message.content.slice(0, prefix.length);
|
|
204
|
+
const tail = message.content.slice(prefix.length);
|
|
205
|
+
if (durable.trim().length === 0) {
|
|
206
|
+
return { role: message.role, content: message.content };
|
|
207
|
+
}
|
|
208
|
+
return {
|
|
209
|
+
role: message.role,
|
|
210
|
+
content: tail.trim().length > 0
|
|
211
|
+
? [
|
|
212
|
+
{ type: "text", text: durable, prompt_cache_breakpoint: { mode: "explicit" } },
|
|
213
|
+
{ type: "text", text: tail },
|
|
214
|
+
]
|
|
215
|
+
: [{ type: "text", text: durable, prompt_cache_breakpoint: { mode: "explicit" } }],
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
else if (prefix.kind === "parts" && Array.isArray(message.content)) {
|
|
219
|
+
const parts = message.content.map(toOpenAIPart);
|
|
220
|
+
const targetIndex = Math.min(prefix.count, parts.length) - 1;
|
|
221
|
+
return {
|
|
222
|
+
role: message.role,
|
|
223
|
+
content: prefix.count > 0
|
|
224
|
+
? withExplicitCacheBreakpoint(parts, targetIndex)
|
|
225
|
+
: parts,
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
}
|
|
112
229
|
if (typeof message.content === "string") {
|
|
113
230
|
return { role: message.role, content: message.content };
|
|
114
231
|
}
|
|
115
232
|
return {
|
|
116
233
|
role: message.role,
|
|
117
|
-
content: message.content.map(
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
234
|
+
content: message.content.map(toOpenAIPart),
|
|
235
|
+
};
|
|
236
|
+
});
|
|
237
|
+
};
|
|
238
|
+
const mapOpenAICompatibleMessages = (messages, cacheBreakpoints, userCachePrefixes) => {
|
|
239
|
+
let lastLeadingSystemIndex = -1;
|
|
240
|
+
const firstNonSystemIndex = firstNonSystemMessageIndex(messages);
|
|
241
|
+
for (let i = 0; i < firstNonSystemIndex; i += 1) {
|
|
242
|
+
if (cacheBreakpoints !== undefined && hasWireContent(messages[i])) {
|
|
243
|
+
lastLeadingSystemIndex = i;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
let penultimateNonSystemIndex = -1;
|
|
247
|
+
let finalUserIndex = -1;
|
|
248
|
+
if (cacheBreakpoints === "conversation") {
|
|
249
|
+
const nonSystemIndices = [];
|
|
250
|
+
for (let i = firstNonSystemIndex; i < messages.length; i += 1) {
|
|
251
|
+
if (hasWireContent(messages[i]))
|
|
252
|
+
nonSystemIndices.push(i);
|
|
253
|
+
}
|
|
254
|
+
if (nonSystemIndices.length >= 2) {
|
|
255
|
+
penultimateNonSystemIndex = nonSystemIndices[nonSystemIndices.length - 2];
|
|
256
|
+
}
|
|
257
|
+
const lastIdx = nonSystemIndices.at(-1);
|
|
258
|
+
if (lastIdx !== undefined && messages[lastIdx].role === "user") {
|
|
259
|
+
finalUserIndex = lastIdx;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
return messages.map((message, index) => {
|
|
263
|
+
if (index === lastLeadingSystemIndex) {
|
|
264
|
+
const parts = typeof message.content === "string"
|
|
265
|
+
? [{ type: "text", text: message.content }]
|
|
266
|
+
: message.content.map(toOpenAIPart);
|
|
267
|
+
return {
|
|
268
|
+
role: message.role,
|
|
269
|
+
content: withEphemeralCacheControl(parts, parts.length - 1),
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
if (index === finalUserIndex) {
|
|
273
|
+
const finalPrefix = userCachePrefixes?.[index];
|
|
274
|
+
if (finalPrefix === undefined) {
|
|
275
|
+
const parts = typeof message.content === "string"
|
|
276
|
+
? [{ type: "text", text: message.content }]
|
|
277
|
+
: message.content.map(toOpenAIPart);
|
|
278
|
+
return {
|
|
279
|
+
role: message.role,
|
|
280
|
+
content: withEphemeralCacheControl(parts, parts.length - 1),
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
if (finalPrefix.kind === "chars" && typeof message.content === "string") {
|
|
284
|
+
if (finalPrefix.length <= 0) {
|
|
285
|
+
return { role: message.role, content: message.content };
|
|
286
|
+
}
|
|
287
|
+
if (finalPrefix.length >= message.content.length) {
|
|
122
288
|
return {
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
data: base64,
|
|
126
|
-
format: part.mediaType.slice("audio/".length),
|
|
127
|
-
},
|
|
289
|
+
role: message.role,
|
|
290
|
+
content: [{ type: "text", text: message.content, cache_control: { type: "ephemeral" } }],
|
|
128
291
|
};
|
|
129
292
|
}
|
|
293
|
+
const durable = message.content.slice(0, finalPrefix.length);
|
|
294
|
+
const volatile = message.content.slice(finalPrefix.length);
|
|
295
|
+
if (durable.trim().length === 0) {
|
|
296
|
+
return { role: message.role, content: message.content };
|
|
297
|
+
}
|
|
130
298
|
return {
|
|
131
|
-
|
|
132
|
-
|
|
299
|
+
role: message.role,
|
|
300
|
+
content: volatile.trim().length > 0
|
|
301
|
+
? [
|
|
302
|
+
{ type: "text", text: durable, cache_control: { type: "ephemeral" } },
|
|
303
|
+
{ type: "text", text: volatile },
|
|
304
|
+
]
|
|
305
|
+
: [{ type: "text", text: durable, cache_control: { type: "ephemeral" } }],
|
|
133
306
|
};
|
|
134
|
-
}
|
|
307
|
+
}
|
|
308
|
+
if (finalPrefix.kind === "parts" && Array.isArray(message.content)) {
|
|
309
|
+
const parts = message.content.map(toOpenAIPart);
|
|
310
|
+
const targetIndex = Math.min(finalPrefix.count, parts.length) - 1;
|
|
311
|
+
return {
|
|
312
|
+
role: message.role,
|
|
313
|
+
content: finalPrefix.count > 0
|
|
314
|
+
? withEphemeralCacheControl(parts, targetIndex)
|
|
315
|
+
: parts,
|
|
316
|
+
};
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
const historicalSplit = cacheBreakpoints === "conversation"
|
|
320
|
+
&& index >= firstNonSystemIndex
|
|
321
|
+
&& message.role === "user"
|
|
322
|
+
&& typeof message.content === "string"
|
|
323
|
+
? splitHistoricalUserChars(message.content, userCachePrefixes?.[index])
|
|
324
|
+
: null;
|
|
325
|
+
if (index === penultimateNonSystemIndex) {
|
|
326
|
+
const parts = historicalSplit !== null
|
|
327
|
+
? [{ type: "text", text: historicalSplit[0] }, { type: "text", text: historicalSplit[1] }]
|
|
328
|
+
: typeof message.content === "string"
|
|
329
|
+
? [{ type: "text", text: message.content }]
|
|
330
|
+
: message.content.map(toOpenAIPart);
|
|
331
|
+
return {
|
|
332
|
+
role: message.role,
|
|
333
|
+
content: withEphemeralCacheControl(parts, parts.length - 1),
|
|
334
|
+
};
|
|
335
|
+
}
|
|
336
|
+
if (historicalSplit !== null) {
|
|
337
|
+
return {
|
|
338
|
+
role: message.role,
|
|
339
|
+
content: [
|
|
340
|
+
{ type: "text", text: historicalSplit[0] },
|
|
341
|
+
{ type: "text", text: historicalSplit[1] },
|
|
342
|
+
],
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
if (typeof message.content === "string") {
|
|
346
|
+
return { role: message.role, content: message.content };
|
|
347
|
+
}
|
|
348
|
+
return {
|
|
349
|
+
role: message.role,
|
|
350
|
+
content: message.content.map(toOpenAIPart),
|
|
135
351
|
};
|
|
136
352
|
});
|
|
137
353
|
};
|
|
138
|
-
const
|
|
354
|
+
const markAnthropicMessageEnd = (wireMsg) => {
|
|
355
|
+
if (!isRecord(wireMsg))
|
|
356
|
+
return;
|
|
357
|
+
if (typeof wireMsg.content === "string") {
|
|
358
|
+
if (wireMsg.content.trim().length > 0) {
|
|
359
|
+
wireMsg.content = [{ type: "text", text: wireMsg.content, cache_control: { type: "ephemeral" } }];
|
|
360
|
+
}
|
|
361
|
+
return;
|
|
362
|
+
}
|
|
363
|
+
if (Array.isArray(wireMsg.content) && wireMsg.content.length > 0) {
|
|
364
|
+
const last = wireMsg.content.at(-1);
|
|
365
|
+
if (isRecord(last)) {
|
|
366
|
+
last.cache_control = { type: "ephemeral" };
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
};
|
|
370
|
+
const applyAnthropicPayloadAdjustments = (params, messages, cacheBreakpoints, userCachePrefixes, serviceTier) => {
|
|
139
371
|
if (serviceTier !== undefined) {
|
|
140
372
|
params.service_tier = serviceTier;
|
|
141
373
|
}
|
|
142
|
-
if (
|
|
374
|
+
if (cacheBreakpoints !== undefined && Array.isArray(params.system) && params.system.length > 0) {
|
|
143
375
|
const last = params.system.at(-1);
|
|
144
376
|
if (isRecord(last)) {
|
|
145
377
|
last.cache_control = { type: "ephemeral" };
|
|
@@ -148,14 +380,20 @@ const applyAnthropicPayloadAdjustments = (params, messages, systemCacheBreakpoin
|
|
|
148
380
|
const wireMessages = Array.isArray(params.messages) ? params.messages : undefined;
|
|
149
381
|
if (wireMessages === undefined)
|
|
150
382
|
return;
|
|
151
|
-
const
|
|
152
|
-
|
|
153
|
-
|
|
383
|
+
const firstNonSystem = firstNonSystemMessageIndex(messages);
|
|
384
|
+
const effectiveSourceIndices = [];
|
|
385
|
+
for (let i = firstNonSystem; i < messages.length; i += 1) {
|
|
386
|
+
if (hasWireContent(messages[i]))
|
|
387
|
+
effectiveSourceIndices.push(i);
|
|
388
|
+
}
|
|
389
|
+
for (let i = 0; i < Math.min(wireMessages.length, effectiveSourceIndices.length); i += 1) {
|
|
390
|
+
const sourceMsg = messages[effectiveSourceIndices[i]];
|
|
154
391
|
const wireMsg = wireMessages[i];
|
|
155
392
|
if (!Array.isArray(sourceMsg.content) || !isRecord(wireMsg) || !Array.isArray(wireMsg.content))
|
|
156
393
|
continue;
|
|
157
|
-
|
|
158
|
-
|
|
394
|
+
const nonBlankSourceParts = sourceMsg.content.filter((p) => p.type !== "text" || p.text.trim().length > 0);
|
|
395
|
+
for (let j = 0; j < Math.min(nonBlankSourceParts.length, wireMsg.content.length); j += 1) {
|
|
396
|
+
const sourcePart = nonBlankSourceParts[j];
|
|
159
397
|
const wireBlock = wireMsg.content[j];
|
|
160
398
|
if (sourcePart.type === "file"
|
|
161
399
|
&& sourcePart.mediaType === "application/pdf"
|
|
@@ -165,9 +403,153 @@ const applyAnthropicPayloadAdjustments = (params, messages, systemCacheBreakpoin
|
|
|
165
403
|
}
|
|
166
404
|
}
|
|
167
405
|
}
|
|
406
|
+
if (cacheBreakpoints !== "conversation" || wireMessages.length === 0)
|
|
407
|
+
return;
|
|
408
|
+
for (let w = 0; w < Math.min(wireMessages.length - 1, effectiveSourceIndices.length); w += 1) {
|
|
409
|
+
const sourceIdx = effectiveSourceIndices[w];
|
|
410
|
+
const sourceMsg = messages[sourceIdx];
|
|
411
|
+
const wireMsg = wireMessages[w];
|
|
412
|
+
if (sourceMsg.role !== "assistant" && typeof sourceMsg.content === "string" && isRecord(wireMsg)) {
|
|
413
|
+
const text = typeof wireMsg.content === "string" ? wireMsg.content : sourceMsg.content.toWellFormed();
|
|
414
|
+
const split = splitHistoricalUserChars(text, userCachePrefixes?.[sourceIdx]);
|
|
415
|
+
if (split !== null) {
|
|
416
|
+
wireMsg.content = [
|
|
417
|
+
{ type: "text", text: split[0] },
|
|
418
|
+
{ type: "text", text: split[1] },
|
|
419
|
+
];
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
if (wireMessages.length >= 2) {
|
|
424
|
+
markAnthropicMessageEnd(wireMessages[wireMessages.length - 2]);
|
|
425
|
+
}
|
|
426
|
+
const lastWire = wireMessages.at(-1);
|
|
427
|
+
const lastSourceIdx = effectiveSourceIndices.at(-1);
|
|
428
|
+
const lastSource = lastSourceIdx === undefined ? undefined : messages[lastSourceIdx];
|
|
429
|
+
if (lastSource === undefined || lastSource.role === "assistant" || !isRecord(lastWire) || lastWire.role !== "user")
|
|
430
|
+
return;
|
|
431
|
+
const finalPrefix = lastSourceIdx === undefined ? undefined : userCachePrefixes?.[lastSourceIdx];
|
|
432
|
+
if (finalPrefix === undefined) {
|
|
433
|
+
markAnthropicMessageEnd(lastWire);
|
|
434
|
+
return;
|
|
435
|
+
}
|
|
436
|
+
if (finalPrefix.kind === "chars" && typeof lastSource.content === "string") {
|
|
437
|
+
if (finalPrefix.length <= 0)
|
|
438
|
+
return;
|
|
439
|
+
const text = typeof lastWire.content === "string" ? lastWire.content : lastSource.content.toWellFormed();
|
|
440
|
+
if (finalPrefix.length >= text.length) {
|
|
441
|
+
markAnthropicMessageEnd(lastWire);
|
|
442
|
+
return;
|
|
443
|
+
}
|
|
444
|
+
const durable = text.slice(0, finalPrefix.length);
|
|
445
|
+
const volatile = text.slice(finalPrefix.length);
|
|
446
|
+
if (durable.trim().length === 0)
|
|
447
|
+
return;
|
|
448
|
+
lastWire.content = volatile.trim().length > 0
|
|
449
|
+
? [
|
|
450
|
+
{ type: "text", text: durable, cache_control: { type: "ephemeral" } },
|
|
451
|
+
{ type: "text", text: volatile },
|
|
452
|
+
]
|
|
453
|
+
: [{ type: "text", text: durable, cache_control: { type: "ephemeral" } }];
|
|
454
|
+
return;
|
|
455
|
+
}
|
|
456
|
+
if (finalPrefix.kind === "parts" && Array.isArray(lastSource.content) && Array.isArray(lastWire.content)) {
|
|
457
|
+
const k = countWireParts(lastSource.content, finalPrefix.count);
|
|
458
|
+
if (k > 0 && k <= lastWire.content.length) {
|
|
459
|
+
const target = lastWire.content[k - 1];
|
|
460
|
+
if (isRecord(target))
|
|
461
|
+
target.cache_control = { type: "ephemeral" };
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
};
|
|
465
|
+
const appendBedrockCachePoint = (wireMsg, sourceMsg) => {
|
|
466
|
+
if (sourceMsg === undefined || !hasWireContent(sourceMsg))
|
|
467
|
+
return;
|
|
468
|
+
if (!isRecord(wireMsg) || !Array.isArray(wireMsg.content) || wireMsg.content.length === 0)
|
|
469
|
+
return;
|
|
470
|
+
wireMsg.content = [...wireMsg.content, { cachePoint: { type: "default" } }];
|
|
471
|
+
};
|
|
472
|
+
const applyBedrockPayloadAdjustments = (params, messages, cacheBreakpoints, userCachePrefixes) => {
|
|
473
|
+
if (cacheBreakpoints === undefined)
|
|
474
|
+
return;
|
|
475
|
+
if (Array.isArray(params.system) && params.system.length > 0) {
|
|
476
|
+
params.system = [...params.system, { cachePoint: { type: "default" } }];
|
|
477
|
+
}
|
|
478
|
+
if (cacheBreakpoints !== "conversation")
|
|
479
|
+
return;
|
|
480
|
+
const wireMessages = Array.isArray(params.messages) ? params.messages : undefined;
|
|
481
|
+
if (wireMessages === undefined || wireMessages.length === 0)
|
|
482
|
+
return;
|
|
483
|
+
const firstNonSystem = firstNonSystemMessageIndex(messages);
|
|
484
|
+
const bedrockSourceIndices = [];
|
|
485
|
+
for (let i = firstNonSystem; i < messages.length; i += 1) {
|
|
486
|
+
const m = messages[i];
|
|
487
|
+
if (m.role !== "assistant" || hasWireContent(m))
|
|
488
|
+
bedrockSourceIndices.push(i);
|
|
489
|
+
}
|
|
490
|
+
for (let w = 0; w < Math.min(wireMessages.length - 1, bedrockSourceIndices.length); w += 1) {
|
|
491
|
+
const sourceIdx = bedrockSourceIndices[w];
|
|
492
|
+
const sourceMsg = messages[sourceIdx];
|
|
493
|
+
const wireMsg = wireMessages[w];
|
|
494
|
+
if (sourceMsg.role !== "assistant" && typeof sourceMsg.content === "string" && isRecord(wireMsg) && Array.isArray(wireMsg.content)) {
|
|
495
|
+
const firstBlock = wireMsg.content[0];
|
|
496
|
+
const text = wireMsg.content.length === 1 && isRecord(firstBlock) && typeof firstBlock.text === "string"
|
|
497
|
+
? firstBlock.text
|
|
498
|
+
: sourceMsg.content.toWellFormed();
|
|
499
|
+
const split = splitHistoricalUserChars(text, userCachePrefixes?.[sourceIdx]);
|
|
500
|
+
if (split !== null) {
|
|
501
|
+
wireMsg.content = [{ text: split[0] }, { text: split[1] }];
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
if (wireMessages.length >= 2) {
|
|
506
|
+
const penultSourceIdx = bedrockSourceIndices[wireMessages.length - 2];
|
|
507
|
+
appendBedrockCachePoint(wireMessages[wireMessages.length - 2], penultSourceIdx === undefined ? undefined : messages[penultSourceIdx]);
|
|
508
|
+
}
|
|
509
|
+
const lastWire = wireMessages.at(-1);
|
|
510
|
+
const lastSourceIdx = bedrockSourceIndices.at(-1);
|
|
511
|
+
const lastSource = lastSourceIdx === undefined ? undefined : messages[lastSourceIdx];
|
|
512
|
+
if (lastSource === undefined || lastSource.role === "assistant" || !hasWireContent(lastSource) || !isRecord(lastWire) || lastWire.role !== "user" || !Array.isArray(lastWire.content)) {
|
|
513
|
+
return;
|
|
514
|
+
}
|
|
515
|
+
const finalPrefix = lastSourceIdx === undefined ? undefined : userCachePrefixes?.[lastSourceIdx];
|
|
516
|
+
if (finalPrefix === undefined) {
|
|
517
|
+
appendBedrockCachePoint(lastWire, lastSource);
|
|
518
|
+
return;
|
|
519
|
+
}
|
|
520
|
+
if (finalPrefix.kind === "chars" && typeof lastSource.content === "string") {
|
|
521
|
+
if (finalPrefix.length <= 0)
|
|
522
|
+
return;
|
|
523
|
+
const firstBlock = lastWire.content[0];
|
|
524
|
+
const text = lastWire.content.length === 1 && isRecord(firstBlock) && typeof firstBlock.text === "string"
|
|
525
|
+
? firstBlock.text
|
|
526
|
+
: lastSource.content.toWellFormed();
|
|
527
|
+
if (finalPrefix.length >= text.length) {
|
|
528
|
+
appendBedrockCachePoint(lastWire, lastSource);
|
|
529
|
+
return;
|
|
530
|
+
}
|
|
531
|
+
const durable = text.slice(0, finalPrefix.length);
|
|
532
|
+
const volatile = text.slice(finalPrefix.length);
|
|
533
|
+
if (durable.trim().length === 0)
|
|
534
|
+
return;
|
|
535
|
+
lastWire.content = volatile.trim().length > 0
|
|
536
|
+
? [{ text: durable }, { cachePoint: { type: "default" } }, { text: volatile }]
|
|
537
|
+
: [{ text: durable }, { cachePoint: { type: "default" } }];
|
|
538
|
+
return;
|
|
539
|
+
}
|
|
540
|
+
if (finalPrefix.kind === "parts" && Array.isArray(lastSource.content)) {
|
|
541
|
+
const k = countWireParts(lastSource.content, finalPrefix.count);
|
|
542
|
+
if (k > 0 && k <= lastWire.content.length) {
|
|
543
|
+
lastWire.content = [
|
|
544
|
+
...lastWire.content.slice(0, k),
|
|
545
|
+
{ cachePoint: { type: "default" } },
|
|
546
|
+
...lastWire.content.slice(k),
|
|
547
|
+
];
|
|
548
|
+
}
|
|
549
|
+
}
|
|
168
550
|
};
|
|
169
551
|
export const executePiTransportStream = async (opts) => {
|
|
170
|
-
const { model, source, url, providerModel, fetch: configuredFetch, requestHeaders, messages, sessionId, operationSignal, requestSampling, body, outputTokenField = "max_tokens", effectiveMaxOutputTokens, nativeReasoningBudget, capacity, requestBody, effort, adaptiveEffort, nativeAdaptiveThinking, reasoningRequestBody,
|
|
552
|
+
const { model, source, url, providerModel, fetch: configuredFetch, requestHeaders, messages, sessionId, operationSignal, requestSampling, body, outputTokenField = "max_tokens", effectiveMaxOutputTokens, nativeReasoningBudget, capacity, requestBody, effort, adaptiveEffort, nativeAdaptiveThinking, reasoningRequestBody, cacheBreakpoints, promptCacheStyle, userCachePrefixes, cacheAffinity, resolvedControls, rawBody, capture, reasoningStepDown = 0, observers, } = opts;
|
|
171
553
|
const providerId = source.replace(/^provider:/, "");
|
|
172
554
|
const effectiveModel = providerModel ?? {
|
|
173
555
|
id: model,
|
|
@@ -284,9 +666,21 @@ export const executePiTransportStream = async (opts) => {
|
|
|
284
666
|
: payload;
|
|
285
667
|
const params = isRecord(transformed) ? { ...transformed } : {};
|
|
286
668
|
if (effectiveModel.api === "openai-completions") {
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
params.
|
|
669
|
+
const hasNonImageFile = messages.some((m) => Array.isArray(m.content) && m.content.some((p) => p.type === "file" && !p.mediaType.startsWith("image/")));
|
|
670
|
+
if (promptCacheStyle === "openai-explicit") {
|
|
671
|
+
params.prompt_cache_options = { mode: "explicit" };
|
|
672
|
+
if (cacheBreakpoints !== undefined || hasNonImageFile) {
|
|
673
|
+
params.messages = mapOpenAIExplicitCacheMessages(messages, cacheBreakpoints, userCachePrefixes);
|
|
674
|
+
}
|
|
675
|
+
else if (messages[0]?.role === "system" && Array.isArray(params.messages)) {
|
|
676
|
+
const first = params.messages[0];
|
|
677
|
+
if (!isRecord(first) || (first.role !== "system" && first.role !== "developer")) {
|
|
678
|
+
params.messages.unshift({ role: "system", content: "" });
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
else if (cacheBreakpoints !== undefined || hasNonImageFile) {
|
|
683
|
+
params.messages = mapOpenAICompatibleMessages(messages, cacheBreakpoints, userCachePrefixes);
|
|
290
684
|
}
|
|
291
685
|
else if (messages[0]?.role === "system" && Array.isArray(params.messages)) {
|
|
292
686
|
const first = params.messages[0];
|
|
@@ -312,7 +706,7 @@ export const executePiTransportStream = async (opts) => {
|
|
|
312
706
|
params.top_k = topK;
|
|
313
707
|
if (stopSequences !== undefined)
|
|
314
708
|
params.stop_sequences = stopSequences;
|
|
315
|
-
applyAnthropicPayloadAdjustments(params, messages,
|
|
709
|
+
applyAnthropicPayloadAdjustments(params, messages, cacheBreakpoints, userCachePrefixes, resolvedControls.serviceTier);
|
|
316
710
|
}
|
|
317
711
|
else if (effectiveModel.api === "google-generative-ai") {
|
|
318
712
|
const config = isRecord(params.config) ? { ...params.config } : {};
|
|
@@ -358,6 +752,7 @@ export const executePiTransportStream = async (opts) => {
|
|
|
358
752
|
if (resolvedControls.serviceTier !== undefined) {
|
|
359
753
|
params.serviceTier = { type: resolvedControls.serviceTier };
|
|
360
754
|
}
|
|
755
|
+
applyBedrockPayloadAdjustments(params, messages, cacheBreakpoints, userCachePrefixes);
|
|
361
756
|
try {
|
|
362
757
|
capture?.record({
|
|
363
758
|
method: "POST",
|
|
@@ -391,7 +786,7 @@ export const executePiTransportStream = async (opts) => {
|
|
|
391
786
|
...(stopSequences === undefined ? {} : { stop: stopSequences }),
|
|
392
787
|
...(seed === undefined ? {} : { seed }),
|
|
393
788
|
...(reasoningRequestBody === undefined ? {} : reasoningRequestBody(effort.mode, capacity.reasoningBudget)),
|
|
394
|
-
...(cacheAffinity?.target === "body" ? { [cacheAffinity.name]: sessionId } : {}),
|
|
789
|
+
...(cacheAffinity?.target === "body" ? { [cacheAffinity.name]: formatCacheAffinityValue(cacheAffinity, sessionId) } : {}),
|
|
395
790
|
...(resolvedControls.topLogprobs === undefined ? {} : { logprobs: true, top_logprobs: resolvedControls.topLogprobs }),
|
|
396
791
|
...(resolvedControls.serviceTier === undefined ? {} : { service_tier: resolvedControls.serviceTier }),
|
|
397
792
|
};
|
|
@@ -402,6 +797,7 @@ export const executePiTransportStream = async (opts) => {
|
|
|
402
797
|
...commonOptions,
|
|
403
798
|
fetch: customFetch,
|
|
404
799
|
apiKey: effectiveModel.apiKey ?? "plurnk-unset",
|
|
800
|
+
cacheRetention: "none",
|
|
405
801
|
...(reasoningEffort === undefined ? {} : { reasoningEffort }),
|
|
406
802
|
...(Object.keys(samplingParams).length === 0 ? {} : { samplingParams }),
|
|
407
803
|
onPayload,
|
|
@@ -411,7 +807,7 @@ export const executePiTransportStream = async (opts) => {
|
|
|
411
807
|
const reasoningEffort = reasoning === "none" || reasoning === "provider-default" ? undefined : reasoning;
|
|
412
808
|
const samplingParams = {
|
|
413
809
|
...(topP === undefined ? {} : { top_p: topP }),
|
|
414
|
-
...(cacheAffinity?.target === "body" ? { [cacheAffinity.name]: sessionId } : {}),
|
|
810
|
+
...(cacheAffinity?.target === "body" ? { [cacheAffinity.name]: formatCacheAffinityValue(cacheAffinity, sessionId) } : {}),
|
|
415
811
|
...(resolvedControls.topLogprobs === undefined ? {} : { top_logprobs: resolvedControls.topLogprobs }),
|
|
416
812
|
...(resolvedControls.serviceTier === undefined ? {} : { service_tier: resolvedControls.serviceTier }),
|
|
417
813
|
};
|
|
@@ -419,6 +815,7 @@ export const executePiTransportStream = async (opts) => {
|
|
|
419
815
|
...commonOptions,
|
|
420
816
|
fetch: customFetch,
|
|
421
817
|
apiKey: effectiveModel.apiKey ?? "plurnk-unset",
|
|
818
|
+
cacheRetention: "none",
|
|
422
819
|
...(reasoningEffort === undefined ? {} : { reasoningEffort }),
|
|
423
820
|
...(Object.keys(samplingParams).length === 0 ? {} : { samplingParams }),
|
|
424
821
|
onPayload,
|
|
@@ -502,6 +899,7 @@ export const executePiTransportStream = async (opts) => {
|
|
|
502
899
|
: (reasoning === "xhigh" ? "max" : reasoning);
|
|
503
900
|
return streamBedrockConverseStream(buildPiModel("bedrock-converse-stream"), context, {
|
|
504
901
|
...commonOptions,
|
|
902
|
+
cacheRetention: "none",
|
|
505
903
|
...(bedrockReasoning === undefined ? {} : { reasoning: bedrockReasoning }),
|
|
506
904
|
...(nativeReasoningBudget !== null && bedrockReasoning !== undefined
|
|
507
905
|
? {
|