@plurnk/plurnk-providers 6.1.1 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.defaults +5 -4
- package/README.md +1 -1
- package/SPEC.md +10 -8
- package/dist/InferenceAdmission.d.ts +10 -1
- package/dist/InferenceAdmission.d.ts.map +1 -1
- package/dist/InferenceAdmission.js +79 -15
- package/dist/InferenceAdmission.js.map +1 -1
- package/dist/ModelProvider.d.ts +3 -1
- package/dist/ModelProvider.d.ts.map +1 -1
- package/dist/ModelProvider.js +8 -5
- package/dist/ModelProvider.js.map +1 -1
- package/dist/catalogProvider.d.ts +2 -2
- package/dist/catalogProvider.d.ts.map +1 -1
- package/dist/catalogProvider.js +2 -1
- package/dist/catalogProvider.js.map +1 -1
- package/dist/compatibleProvider.d.ts.map +1 -1
- package/dist/compatibleProvider.js +6 -3
- package/dist/compatibleProvider.js.map +1 -1
- package/dist/env.d.ts +3 -1
- package/dist/env.d.ts.map +1 -1
- package/dist/env.js +24 -2
- package/dist/env.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/modelAdapters.d.ts +2 -2
- package/dist/modelAdapters.d.ts.map +1 -1
- package/dist/modelAdapters.js +53 -4
- package/dist/modelAdapters.js.map +1 -1
- package/dist/pi-transport.d.ts +5 -3
- package/dist/pi-transport.d.ts.map +1 -1
- package/dist/pi-transport.js +248 -51
- package/dist/pi-transport.js.map +1 -1
- package/dist/types.d.ts +2 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +3 -3
package/dist/pi-transport.js
CHANGED
|
@@ -120,20 +120,125 @@ const toOpenAIPart = (part) => {
|
|
|
120
120
|
image_url: { url: `data:${part.mediaType};base64,${base64}` },
|
|
121
121
|
};
|
|
122
122
|
};
|
|
123
|
+
export const formatCacheAffinityValue = (affinity, sessionId) => {
|
|
124
|
+
if (affinity.name !== "prompt_cache_key")
|
|
125
|
+
return sessionId;
|
|
126
|
+
const chars = Array.from(sessionId);
|
|
127
|
+
return chars.length <= 64 ? sessionId : chars.slice(0, 64).join("");
|
|
128
|
+
};
|
|
129
|
+
const firstNonSystemMessageIndex = (messages) => {
|
|
130
|
+
for (let i = 0; i < messages.length; i += 1) {
|
|
131
|
+
if (messages[i].role !== "system")
|
|
132
|
+
return i;
|
|
133
|
+
}
|
|
134
|
+
return messages.length;
|
|
135
|
+
};
|
|
136
|
+
const splitHistoricalUserChars = (text, prefix) => {
|
|
137
|
+
if (prefix?.kind !== "chars" || prefix.length <= 0 || prefix.length >= text.length)
|
|
138
|
+
return null;
|
|
139
|
+
const durable = text.slice(0, prefix.length);
|
|
140
|
+
const tail = text.slice(prefix.length);
|
|
141
|
+
return durable.trim().length > 0 && tail.trim().length > 0 ? [durable, tail] : null;
|
|
142
|
+
};
|
|
123
143
|
const withEphemeralCacheControl = (parts, atIndex) => {
|
|
124
144
|
if (atIndex >= 0 && atIndex < parts.length) {
|
|
125
145
|
parts[atIndex] = { ...parts[atIndex], cache_control: { type: "ephemeral" } };
|
|
126
146
|
}
|
|
127
147
|
return parts;
|
|
128
148
|
};
|
|
129
|
-
const
|
|
149
|
+
const withExplicitCacheBreakpoint = (parts, atIndex) => {
|
|
150
|
+
if (atIndex >= 0 && atIndex < parts.length) {
|
|
151
|
+
parts[atIndex] = { ...parts[atIndex], prompt_cache_breakpoint: { mode: "explicit" } };
|
|
152
|
+
}
|
|
153
|
+
return parts;
|
|
154
|
+
};
|
|
155
|
+
const mapOpenAIExplicitCacheMessages = (messages, cacheBreakpoints, userCachePrefixes) => {
|
|
130
156
|
let lastLeadingSystemIndex = -1;
|
|
131
|
-
|
|
132
|
-
for (let i = 0; i <
|
|
133
|
-
if (messages[i]
|
|
134
|
-
|
|
135
|
-
|
|
157
|
+
const firstNonSystemIndex = firstNonSystemMessageIndex(messages);
|
|
158
|
+
for (let i = 0; i < firstNonSystemIndex; i += 1) {
|
|
159
|
+
if (cacheBreakpoints !== undefined && hasWireContent(messages[i])) {
|
|
160
|
+
lastLeadingSystemIndex = i;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
const unscopedConversation = cacheBreakpoints === "conversation" && userCachePrefixes === undefined;
|
|
164
|
+
return messages.map((message, index) => {
|
|
165
|
+
if (index === lastLeadingSystemIndex) {
|
|
166
|
+
const parts = typeof message.content === "string"
|
|
167
|
+
? [{ type: "text", text: message.content }]
|
|
168
|
+
: message.content.map(toOpenAIPart);
|
|
169
|
+
return {
|
|
170
|
+
role: message.role,
|
|
171
|
+
content: withExplicitCacheBreakpoint(parts, parts.length - 1),
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
if (cacheBreakpoints === "conversation" && index >= firstNonSystemIndex && message.role === "user") {
|
|
175
|
+
const prefix = userCachePrefixes?.[index];
|
|
176
|
+
if (prefix === undefined) {
|
|
177
|
+
if (unscopedConversation && hasWireContent(message)) {
|
|
178
|
+
const parts = typeof message.content === "string"
|
|
179
|
+
? [{ type: "text", text: message.content }]
|
|
180
|
+
: message.content.map(toOpenAIPart);
|
|
181
|
+
return {
|
|
182
|
+
role: message.role,
|
|
183
|
+
content: withExplicitCacheBreakpoint(parts, parts.length - 1),
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
else if (prefix.kind === "chars" && typeof message.content === "string") {
|
|
188
|
+
if (prefix.length <= 0) {
|
|
189
|
+
return { role: message.role, content: message.content };
|
|
190
|
+
}
|
|
191
|
+
if (prefix.length >= message.content.length) {
|
|
192
|
+
return message.content.trim().length > 0
|
|
193
|
+
? {
|
|
194
|
+
role: message.role,
|
|
195
|
+
content: [{
|
|
196
|
+
type: "text",
|
|
197
|
+
text: message.content,
|
|
198
|
+
prompt_cache_breakpoint: { mode: "explicit" },
|
|
199
|
+
}],
|
|
200
|
+
}
|
|
201
|
+
: { role: message.role, content: message.content };
|
|
202
|
+
}
|
|
203
|
+
const durable = message.content.slice(0, prefix.length);
|
|
204
|
+
const tail = message.content.slice(prefix.length);
|
|
205
|
+
if (durable.trim().length === 0) {
|
|
206
|
+
return { role: message.role, content: message.content };
|
|
207
|
+
}
|
|
208
|
+
return {
|
|
209
|
+
role: message.role,
|
|
210
|
+
content: tail.trim().length > 0
|
|
211
|
+
? [
|
|
212
|
+
{ type: "text", text: durable, prompt_cache_breakpoint: { mode: "explicit" } },
|
|
213
|
+
{ type: "text", text: tail },
|
|
214
|
+
]
|
|
215
|
+
: [{ type: "text", text: durable, prompt_cache_breakpoint: { mode: "explicit" } }],
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
else if (prefix.kind === "parts" && Array.isArray(message.content)) {
|
|
219
|
+
const parts = message.content.map(toOpenAIPart);
|
|
220
|
+
const targetIndex = Math.min(prefix.count, parts.length) - 1;
|
|
221
|
+
return {
|
|
222
|
+
role: message.role,
|
|
223
|
+
content: prefix.count > 0
|
|
224
|
+
? withExplicitCacheBreakpoint(parts, targetIndex)
|
|
225
|
+
: parts,
|
|
226
|
+
};
|
|
227
|
+
}
|
|
136
228
|
}
|
|
229
|
+
if (typeof message.content === "string") {
|
|
230
|
+
return { role: message.role, content: message.content };
|
|
231
|
+
}
|
|
232
|
+
return {
|
|
233
|
+
role: message.role,
|
|
234
|
+
content: message.content.map(toOpenAIPart),
|
|
235
|
+
};
|
|
236
|
+
});
|
|
237
|
+
};
|
|
238
|
+
const mapOpenAICompatibleMessages = (messages, cacheBreakpoints, userCachePrefixes) => {
|
|
239
|
+
let lastLeadingSystemIndex = -1;
|
|
240
|
+
const firstNonSystemIndex = firstNonSystemMessageIndex(messages);
|
|
241
|
+
for (let i = 0; i < firstNonSystemIndex; i += 1) {
|
|
137
242
|
if (cacheBreakpoints !== undefined && hasWireContent(messages[i])) {
|
|
138
243
|
lastLeadingSystemIndex = i;
|
|
139
244
|
}
|
|
@@ -155,7 +260,7 @@ const mapOpenAICompatibleMessages = (messages, cacheBreakpoints, finalUserCacheP
|
|
|
155
260
|
}
|
|
156
261
|
}
|
|
157
262
|
return messages.map((message, index) => {
|
|
158
|
-
if (index === lastLeadingSystemIndex
|
|
263
|
+
if (index === lastLeadingSystemIndex) {
|
|
159
264
|
const parts = typeof message.content === "string"
|
|
160
265
|
? [{ type: "text", text: message.content }]
|
|
161
266
|
: message.content.map(toOpenAIPart);
|
|
@@ -165,7 +270,8 @@ const mapOpenAICompatibleMessages = (messages, cacheBreakpoints, finalUserCacheP
|
|
|
165
270
|
};
|
|
166
271
|
}
|
|
167
272
|
if (index === finalUserIndex) {
|
|
168
|
-
|
|
273
|
+
const finalPrefix = userCachePrefixes?.[index];
|
|
274
|
+
if (finalPrefix === undefined) {
|
|
169
275
|
const parts = typeof message.content === "string"
|
|
170
276
|
? [{ type: "text", text: message.content }]
|
|
171
277
|
: message.content.map(toOpenAIPart);
|
|
@@ -174,18 +280,18 @@ const mapOpenAICompatibleMessages = (messages, cacheBreakpoints, finalUserCacheP
|
|
|
174
280
|
content: withEphemeralCacheControl(parts, parts.length - 1),
|
|
175
281
|
};
|
|
176
282
|
}
|
|
177
|
-
if (
|
|
178
|
-
if (
|
|
283
|
+
if (finalPrefix.kind === "chars" && typeof message.content === "string") {
|
|
284
|
+
if (finalPrefix.length <= 0) {
|
|
179
285
|
return { role: message.role, content: message.content };
|
|
180
286
|
}
|
|
181
|
-
if (
|
|
287
|
+
if (finalPrefix.length >= message.content.length) {
|
|
182
288
|
return {
|
|
183
289
|
role: message.role,
|
|
184
290
|
content: [{ type: "text", text: message.content, cache_control: { type: "ephemeral" } }],
|
|
185
291
|
};
|
|
186
292
|
}
|
|
187
|
-
const durable = message.content.slice(0,
|
|
188
|
-
const volatile = message.content.slice(
|
|
293
|
+
const durable = message.content.slice(0, finalPrefix.length);
|
|
294
|
+
const volatile = message.content.slice(finalPrefix.length);
|
|
189
295
|
if (durable.trim().length === 0) {
|
|
190
296
|
return { role: message.role, content: message.content };
|
|
191
297
|
}
|
|
@@ -199,17 +305,43 @@ const mapOpenAICompatibleMessages = (messages, cacheBreakpoints, finalUserCacheP
|
|
|
199
305
|
: [{ type: "text", text: durable, cache_control: { type: "ephemeral" } }],
|
|
200
306
|
};
|
|
201
307
|
}
|
|
202
|
-
if (
|
|
308
|
+
if (finalPrefix.kind === "parts" && Array.isArray(message.content)) {
|
|
203
309
|
const parts = message.content.map(toOpenAIPart);
|
|
204
|
-
const targetIndex = Math.min(
|
|
310
|
+
const targetIndex = Math.min(finalPrefix.count, parts.length) - 1;
|
|
205
311
|
return {
|
|
206
312
|
role: message.role,
|
|
207
|
-
content:
|
|
313
|
+
content: finalPrefix.count > 0
|
|
208
314
|
? withEphemeralCacheControl(parts, targetIndex)
|
|
209
315
|
: parts,
|
|
210
316
|
};
|
|
211
317
|
}
|
|
212
318
|
}
|
|
319
|
+
const historicalSplit = cacheBreakpoints === "conversation"
|
|
320
|
+
&& index >= firstNonSystemIndex
|
|
321
|
+
&& message.role === "user"
|
|
322
|
+
&& typeof message.content === "string"
|
|
323
|
+
? splitHistoricalUserChars(message.content, userCachePrefixes?.[index])
|
|
324
|
+
: null;
|
|
325
|
+
if (index === penultimateNonSystemIndex) {
|
|
326
|
+
const parts = historicalSplit !== null
|
|
327
|
+
? [{ type: "text", text: historicalSplit[0] }, { type: "text", text: historicalSplit[1] }]
|
|
328
|
+
: typeof message.content === "string"
|
|
329
|
+
? [{ type: "text", text: message.content }]
|
|
330
|
+
: message.content.map(toOpenAIPart);
|
|
331
|
+
return {
|
|
332
|
+
role: message.role,
|
|
333
|
+
content: withEphemeralCacheControl(parts, parts.length - 1),
|
|
334
|
+
};
|
|
335
|
+
}
|
|
336
|
+
if (historicalSplit !== null) {
|
|
337
|
+
return {
|
|
338
|
+
role: message.role,
|
|
339
|
+
content: [
|
|
340
|
+
{ type: "text", text: historicalSplit[0] },
|
|
341
|
+
{ type: "text", text: historicalSplit[1] },
|
|
342
|
+
],
|
|
343
|
+
};
|
|
344
|
+
}
|
|
213
345
|
if (typeof message.content === "string") {
|
|
214
346
|
return { role: message.role, content: message.content };
|
|
215
347
|
}
|
|
@@ -235,7 +367,7 @@ const markAnthropicMessageEnd = (wireMsg) => {
|
|
|
235
367
|
}
|
|
236
368
|
}
|
|
237
369
|
};
|
|
238
|
-
const applyAnthropicPayloadAdjustments = (params, messages, cacheBreakpoints,
|
|
370
|
+
const applyAnthropicPayloadAdjustments = (params, messages, cacheBreakpoints, userCachePrefixes, serviceTier) => {
|
|
239
371
|
if (serviceTier !== undefined) {
|
|
240
372
|
params.service_tier = serviceTier;
|
|
241
373
|
}
|
|
@@ -248,9 +380,14 @@ const applyAnthropicPayloadAdjustments = (params, messages, cacheBreakpoints, fi
|
|
|
248
380
|
const wireMessages = Array.isArray(params.messages) ? params.messages : undefined;
|
|
249
381
|
if (wireMessages === undefined)
|
|
250
382
|
return;
|
|
251
|
-
const
|
|
252
|
-
|
|
253
|
-
|
|
383
|
+
const firstNonSystem = firstNonSystemMessageIndex(messages);
|
|
384
|
+
const effectiveSourceIndices = [];
|
|
385
|
+
for (let i = firstNonSystem; i < messages.length; i += 1) {
|
|
386
|
+
if (hasWireContent(messages[i]))
|
|
387
|
+
effectiveSourceIndices.push(i);
|
|
388
|
+
}
|
|
389
|
+
for (let i = 0; i < Math.min(wireMessages.length, effectiveSourceIndices.length); i += 1) {
|
|
390
|
+
const sourceMsg = messages[effectiveSourceIndices[i]];
|
|
254
391
|
const wireMsg = wireMessages[i];
|
|
255
392
|
if (!Array.isArray(sourceMsg.content) || !isRecord(wireMsg) || !Array.isArray(wireMsg.content))
|
|
256
393
|
continue;
|
|
@@ -268,26 +405,44 @@ const applyAnthropicPayloadAdjustments = (params, messages, cacheBreakpoints, fi
|
|
|
268
405
|
}
|
|
269
406
|
if (cacheBreakpoints !== "conversation" || wireMessages.length === 0)
|
|
270
407
|
return;
|
|
408
|
+
for (let w = 0; w < Math.min(wireMessages.length - 1, effectiveSourceIndices.length); w += 1) {
|
|
409
|
+
const sourceIdx = effectiveSourceIndices[w];
|
|
410
|
+
const sourceMsg = messages[sourceIdx];
|
|
411
|
+
const wireMsg = wireMessages[w];
|
|
412
|
+
if (sourceMsg.role !== "assistant" && typeof sourceMsg.content === "string" && isRecord(wireMsg)) {
|
|
413
|
+
const text = typeof wireMsg.content === "string" ? wireMsg.content : sourceMsg.content.toWellFormed();
|
|
414
|
+
const split = splitHistoricalUserChars(text, userCachePrefixes?.[sourceIdx]);
|
|
415
|
+
if (split !== null) {
|
|
416
|
+
wireMsg.content = [
|
|
417
|
+
{ type: "text", text: split[0] },
|
|
418
|
+
{ type: "text", text: split[1] },
|
|
419
|
+
];
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
}
|
|
271
423
|
if (wireMessages.length >= 2) {
|
|
272
424
|
markAnthropicMessageEnd(wireMessages[wireMessages.length - 2]);
|
|
273
425
|
}
|
|
274
426
|
const lastWire = wireMessages.at(-1);
|
|
275
|
-
const
|
|
276
|
-
|
|
427
|
+
const lastSourceIdx = effectiveSourceIndices.at(-1);
|
|
428
|
+
const lastSource = lastSourceIdx === undefined ? undefined : messages[lastSourceIdx];
|
|
429
|
+
if (lastSource === undefined || lastSource.role === "assistant" || !isRecord(lastWire) || lastWire.role !== "user")
|
|
277
430
|
return;
|
|
278
|
-
|
|
431
|
+
const finalPrefix = lastSourceIdx === undefined ? undefined : userCachePrefixes?.[lastSourceIdx];
|
|
432
|
+
if (finalPrefix === undefined) {
|
|
279
433
|
markAnthropicMessageEnd(lastWire);
|
|
280
434
|
return;
|
|
281
435
|
}
|
|
282
|
-
if (
|
|
283
|
-
if (
|
|
436
|
+
if (finalPrefix.kind === "chars" && typeof lastSource.content === "string") {
|
|
437
|
+
if (finalPrefix.length <= 0)
|
|
284
438
|
return;
|
|
285
|
-
|
|
439
|
+
const text = typeof lastWire.content === "string" ? lastWire.content : lastSource.content.toWellFormed();
|
|
440
|
+
if (finalPrefix.length >= text.length) {
|
|
286
441
|
markAnthropicMessageEnd(lastWire);
|
|
287
442
|
return;
|
|
288
443
|
}
|
|
289
|
-
const durable =
|
|
290
|
-
const volatile =
|
|
444
|
+
const durable = text.slice(0, finalPrefix.length);
|
|
445
|
+
const volatile = text.slice(finalPrefix.length);
|
|
291
446
|
if (durable.trim().length === 0)
|
|
292
447
|
return;
|
|
293
448
|
lastWire.content = volatile.trim().length > 0
|
|
@@ -298,8 +453,8 @@ const applyAnthropicPayloadAdjustments = (params, messages, cacheBreakpoints, fi
|
|
|
298
453
|
: [{ type: "text", text: durable, cache_control: { type: "ephemeral" } }];
|
|
299
454
|
return;
|
|
300
455
|
}
|
|
301
|
-
if (
|
|
302
|
-
const k = countWireParts(lastSource.content,
|
|
456
|
+
if (finalPrefix.kind === "parts" && Array.isArray(lastSource.content) && Array.isArray(lastWire.content)) {
|
|
457
|
+
const k = countWireParts(lastSource.content, finalPrefix.count);
|
|
303
458
|
if (k > 0 && k <= lastWire.content.length) {
|
|
304
459
|
const target = lastWire.content[k - 1];
|
|
305
460
|
if (isRecord(target))
|
|
@@ -314,7 +469,7 @@ const appendBedrockCachePoint = (wireMsg, sourceMsg) => {
|
|
|
314
469
|
return;
|
|
315
470
|
wireMsg.content = [...wireMsg.content, { cachePoint: { type: "default" } }];
|
|
316
471
|
};
|
|
317
|
-
const applyBedrockPayloadAdjustments = (params, messages, cacheBreakpoints,
|
|
472
|
+
const applyBedrockPayloadAdjustments = (params, messages, cacheBreakpoints, userCachePrefixes) => {
|
|
318
473
|
if (cacheBreakpoints === undefined)
|
|
319
474
|
return;
|
|
320
475
|
if (Array.isArray(params.system) && params.system.length > 0) {
|
|
@@ -325,28 +480,56 @@ const applyBedrockPayloadAdjustments = (params, messages, cacheBreakpoints, fina
|
|
|
325
480
|
const wireMessages = Array.isArray(params.messages) ? params.messages : undefined;
|
|
326
481
|
if (wireMessages === undefined || wireMessages.length === 0)
|
|
327
482
|
return;
|
|
328
|
-
const
|
|
483
|
+
const firstNonSystem = firstNonSystemMessageIndex(messages);
|
|
484
|
+
const bedrockSourceIndices = [];
|
|
485
|
+
for (let i = firstNonSystem; i < messages.length; i += 1) {
|
|
486
|
+
const m = messages[i];
|
|
487
|
+
if (m.role !== "assistant" || hasWireContent(m))
|
|
488
|
+
bedrockSourceIndices.push(i);
|
|
489
|
+
}
|
|
490
|
+
for (let w = 0; w < Math.min(wireMessages.length - 1, bedrockSourceIndices.length); w += 1) {
|
|
491
|
+
const sourceIdx = bedrockSourceIndices[w];
|
|
492
|
+
const sourceMsg = messages[sourceIdx];
|
|
493
|
+
const wireMsg = wireMessages[w];
|
|
494
|
+
if (sourceMsg.role !== "assistant" && typeof sourceMsg.content === "string" && isRecord(wireMsg) && Array.isArray(wireMsg.content)) {
|
|
495
|
+
const firstBlock = wireMsg.content[0];
|
|
496
|
+
const text = wireMsg.content.length === 1 && isRecord(firstBlock) && typeof firstBlock.text === "string"
|
|
497
|
+
? firstBlock.text
|
|
498
|
+
: sourceMsg.content.toWellFormed();
|
|
499
|
+
const split = splitHistoricalUserChars(text, userCachePrefixes?.[sourceIdx]);
|
|
500
|
+
if (split !== null) {
|
|
501
|
+
wireMsg.content = [{ text: split[0] }, { text: split[1] }];
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
}
|
|
329
505
|
if (wireMessages.length >= 2) {
|
|
330
|
-
|
|
506
|
+
const penultSourceIdx = bedrockSourceIndices[wireMessages.length - 2];
|
|
507
|
+
appendBedrockCachePoint(wireMessages[wireMessages.length - 2], penultSourceIdx === undefined ? undefined : messages[penultSourceIdx]);
|
|
331
508
|
}
|
|
332
509
|
const lastWire = wireMessages.at(-1);
|
|
333
|
-
const
|
|
334
|
-
|
|
510
|
+
const lastSourceIdx = bedrockSourceIndices.at(-1);
|
|
511
|
+
const lastSource = lastSourceIdx === undefined ? undefined : messages[lastSourceIdx];
|
|
512
|
+
if (lastSource === undefined || lastSource.role === "assistant" || !hasWireContent(lastSource) || !isRecord(lastWire) || lastWire.role !== "user" || !Array.isArray(lastWire.content)) {
|
|
335
513
|
return;
|
|
336
514
|
}
|
|
337
|
-
|
|
515
|
+
const finalPrefix = lastSourceIdx === undefined ? undefined : userCachePrefixes?.[lastSourceIdx];
|
|
516
|
+
if (finalPrefix === undefined) {
|
|
338
517
|
appendBedrockCachePoint(lastWire, lastSource);
|
|
339
518
|
return;
|
|
340
519
|
}
|
|
341
|
-
if (
|
|
342
|
-
if (
|
|
520
|
+
if (finalPrefix.kind === "chars" && typeof lastSource.content === "string") {
|
|
521
|
+
if (finalPrefix.length <= 0)
|
|
343
522
|
return;
|
|
344
|
-
|
|
523
|
+
const firstBlock = lastWire.content[0];
|
|
524
|
+
const text = lastWire.content.length === 1 && isRecord(firstBlock) && typeof firstBlock.text === "string"
|
|
525
|
+
? firstBlock.text
|
|
526
|
+
: lastSource.content.toWellFormed();
|
|
527
|
+
if (finalPrefix.length >= text.length) {
|
|
345
528
|
appendBedrockCachePoint(lastWire, lastSource);
|
|
346
529
|
return;
|
|
347
530
|
}
|
|
348
|
-
const durable =
|
|
349
|
-
const volatile =
|
|
531
|
+
const durable = text.slice(0, finalPrefix.length);
|
|
532
|
+
const volatile = text.slice(finalPrefix.length);
|
|
350
533
|
if (durable.trim().length === 0)
|
|
351
534
|
return;
|
|
352
535
|
lastWire.content = volatile.trim().length > 0
|
|
@@ -354,8 +537,8 @@ const applyBedrockPayloadAdjustments = (params, messages, cacheBreakpoints, fina
|
|
|
354
537
|
: [{ text: durable }, { cachePoint: { type: "default" } }];
|
|
355
538
|
return;
|
|
356
539
|
}
|
|
357
|
-
if (
|
|
358
|
-
const k = countWireParts(lastSource.content,
|
|
540
|
+
if (finalPrefix.kind === "parts" && Array.isArray(lastSource.content)) {
|
|
541
|
+
const k = countWireParts(lastSource.content, finalPrefix.count);
|
|
359
542
|
if (k > 0 && k <= lastWire.content.length) {
|
|
360
543
|
lastWire.content = [
|
|
361
544
|
...lastWire.content.slice(0, k),
|
|
@@ -366,7 +549,7 @@ const applyBedrockPayloadAdjustments = (params, messages, cacheBreakpoints, fina
|
|
|
366
549
|
}
|
|
367
550
|
};
|
|
368
551
|
export const executePiTransportStream = async (opts) => {
|
|
369
|
-
const { model, source, url, providerModel, fetch: configuredFetch, requestHeaders, messages, sessionId, operationSignal, requestSampling, body, outputTokenField = "max_tokens", effectiveMaxOutputTokens, nativeReasoningBudget, capacity, requestBody, effort, adaptiveEffort, nativeAdaptiveThinking, reasoningRequestBody, cacheBreakpoints,
|
|
552
|
+
const { model, source, url, providerModel, fetch: configuredFetch, requestHeaders, messages, sessionId, operationSignal, requestSampling, body, outputTokenField = "max_tokens", effectiveMaxOutputTokens, nativeReasoningBudget, capacity, requestBody, effort, adaptiveEffort, nativeAdaptiveThinking, reasoningRequestBody, cacheBreakpoints, promptCacheStyle, userCachePrefixes, cacheAffinity, resolvedControls, rawBody, capture, reasoningStepDown = 0, observers, } = opts;
|
|
370
553
|
const providerId = source.replace(/^provider:/, "");
|
|
371
554
|
const effectiveModel = providerModel ?? {
|
|
372
555
|
id: model,
|
|
@@ -483,9 +666,21 @@ export const executePiTransportStream = async (opts) => {
|
|
|
483
666
|
: payload;
|
|
484
667
|
const params = isRecord(transformed) ? { ...transformed } : {};
|
|
485
668
|
if (effectiveModel.api === "openai-completions") {
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
params.
|
|
669
|
+
const hasNonImageFile = messages.some((m) => Array.isArray(m.content) && m.content.some((p) => p.type === "file" && !p.mediaType.startsWith("image/")));
|
|
670
|
+
if (promptCacheStyle === "openai-explicit") {
|
|
671
|
+
params.prompt_cache_options = { mode: "explicit" };
|
|
672
|
+
if (cacheBreakpoints !== undefined || hasNonImageFile) {
|
|
673
|
+
params.messages = mapOpenAIExplicitCacheMessages(messages, cacheBreakpoints, userCachePrefixes);
|
|
674
|
+
}
|
|
675
|
+
else if (messages[0]?.role === "system" && Array.isArray(params.messages)) {
|
|
676
|
+
const first = params.messages[0];
|
|
677
|
+
if (!isRecord(first) || (first.role !== "system" && first.role !== "developer")) {
|
|
678
|
+
params.messages.unshift({ role: "system", content: "" });
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
else if (cacheBreakpoints !== undefined || hasNonImageFile) {
|
|
683
|
+
params.messages = mapOpenAICompatibleMessages(messages, cacheBreakpoints, userCachePrefixes);
|
|
489
684
|
}
|
|
490
685
|
else if (messages[0]?.role === "system" && Array.isArray(params.messages)) {
|
|
491
686
|
const first = params.messages[0];
|
|
@@ -511,7 +706,7 @@ export const executePiTransportStream = async (opts) => {
|
|
|
511
706
|
params.top_k = topK;
|
|
512
707
|
if (stopSequences !== undefined)
|
|
513
708
|
params.stop_sequences = stopSequences;
|
|
514
|
-
applyAnthropicPayloadAdjustments(params, messages, cacheBreakpoints,
|
|
709
|
+
applyAnthropicPayloadAdjustments(params, messages, cacheBreakpoints, userCachePrefixes, resolvedControls.serviceTier);
|
|
515
710
|
}
|
|
516
711
|
else if (effectiveModel.api === "google-generative-ai") {
|
|
517
712
|
const config = isRecord(params.config) ? { ...params.config } : {};
|
|
@@ -557,7 +752,7 @@ export const executePiTransportStream = async (opts) => {
|
|
|
557
752
|
if (resolvedControls.serviceTier !== undefined) {
|
|
558
753
|
params.serviceTier = { type: resolvedControls.serviceTier };
|
|
559
754
|
}
|
|
560
|
-
applyBedrockPayloadAdjustments(params, messages, cacheBreakpoints,
|
|
755
|
+
applyBedrockPayloadAdjustments(params, messages, cacheBreakpoints, userCachePrefixes);
|
|
561
756
|
try {
|
|
562
757
|
capture?.record({
|
|
563
758
|
method: "POST",
|
|
@@ -591,7 +786,7 @@ export const executePiTransportStream = async (opts) => {
|
|
|
591
786
|
...(stopSequences === undefined ? {} : { stop: stopSequences }),
|
|
592
787
|
...(seed === undefined ? {} : { seed }),
|
|
593
788
|
...(reasoningRequestBody === undefined ? {} : reasoningRequestBody(effort.mode, capacity.reasoningBudget)),
|
|
594
|
-
...(cacheAffinity?.target === "body" ? { [cacheAffinity.name]: sessionId } : {}),
|
|
789
|
+
...(cacheAffinity?.target === "body" ? { [cacheAffinity.name]: formatCacheAffinityValue(cacheAffinity, sessionId) } : {}),
|
|
595
790
|
...(resolvedControls.topLogprobs === undefined ? {} : { logprobs: true, top_logprobs: resolvedControls.topLogprobs }),
|
|
596
791
|
...(resolvedControls.serviceTier === undefined ? {} : { service_tier: resolvedControls.serviceTier }),
|
|
597
792
|
};
|
|
@@ -602,6 +797,7 @@ export const executePiTransportStream = async (opts) => {
|
|
|
602
797
|
...commonOptions,
|
|
603
798
|
fetch: customFetch,
|
|
604
799
|
apiKey: effectiveModel.apiKey ?? "plurnk-unset",
|
|
800
|
+
cacheRetention: "none",
|
|
605
801
|
...(reasoningEffort === undefined ? {} : { reasoningEffort }),
|
|
606
802
|
...(Object.keys(samplingParams).length === 0 ? {} : { samplingParams }),
|
|
607
803
|
onPayload,
|
|
@@ -611,7 +807,7 @@ export const executePiTransportStream = async (opts) => {
|
|
|
611
807
|
const reasoningEffort = reasoning === "none" || reasoning === "provider-default" ? undefined : reasoning;
|
|
612
808
|
const samplingParams = {
|
|
613
809
|
...(topP === undefined ? {} : { top_p: topP }),
|
|
614
|
-
...(cacheAffinity?.target === "body" ? { [cacheAffinity.name]: sessionId } : {}),
|
|
810
|
+
...(cacheAffinity?.target === "body" ? { [cacheAffinity.name]: formatCacheAffinityValue(cacheAffinity, sessionId) } : {}),
|
|
615
811
|
...(resolvedControls.topLogprobs === undefined ? {} : { top_logprobs: resolvedControls.topLogprobs }),
|
|
616
812
|
...(resolvedControls.serviceTier === undefined ? {} : { service_tier: resolvedControls.serviceTier }),
|
|
617
813
|
};
|
|
@@ -619,6 +815,7 @@ export const executePiTransportStream = async (opts) => {
|
|
|
619
815
|
...commonOptions,
|
|
620
816
|
fetch: customFetch,
|
|
621
817
|
apiKey: effectiveModel.apiKey ?? "plurnk-unset",
|
|
818
|
+
cacheRetention: "none",
|
|
622
819
|
...(reasoningEffort === undefined ? {} : { reasoningEffort }),
|
|
623
820
|
...(Object.keys(samplingParams).length === 0 ? {} : { samplingParams }),
|
|
624
821
|
onPayload,
|