@plurnk/plurnk-providers 6.1.0 → 6.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/.env.defaults +4 -5
  2. package/README.md +31 -213
  3. package/SPEC.md +8 -5
  4. package/dist/InferenceAdmission.js.map +1 -1
  5. package/dist/LeadingReasoning.js.map +1 -1
  6. package/dist/Mock.d.ts +5 -1
  7. package/dist/Mock.d.ts.map +1 -1
  8. package/dist/Mock.js +7 -1
  9. package/dist/Mock.js.map +1 -1
  10. package/dist/ModelProvider.d.ts +6 -3
  11. package/dist/ModelProvider.d.ts.map +1 -1
  12. package/dist/ModelProvider.js +9 -6
  13. package/dist/ModelProvider.js.map +1 -1
  14. package/dist/ModelRequestBody.js.map +1 -1
  15. package/dist/Pool.d.ts +2 -0
  16. package/dist/Pool.d.ts.map +1 -1
  17. package/dist/Pool.js +8 -0
  18. package/dist/Pool.js.map +1 -1
  19. package/dist/ProviderRegistry.d.ts.map +1 -1
  20. package/dist/ProviderRegistry.js +5 -10
  21. package/dist/ProviderRegistry.js.map +1 -1
  22. package/dist/RequestCapture.js.map +1 -1
  23. package/dist/accounting.js.map +1 -1
  24. package/dist/accountingPublic.js.map +1 -1
  25. package/dist/capacity.js.map +1 -1
  26. package/dist/catalogProvider.d.ts +2 -2
  27. package/dist/catalogProvider.d.ts.map +1 -1
  28. package/dist/catalogProvider.js +4 -4
  29. package/dist/catalogProvider.js.map +1 -1
  30. package/dist/compatibleProvider.js.map +1 -1
  31. package/dist/cost.js.map +1 -1
  32. package/dist/defaults.js.map +1 -1
  33. package/dist/discover.d.ts +0 -2
  34. package/dist/discover.d.ts.map +1 -1
  35. package/dist/discover.js +2 -8
  36. package/dist/discover.js.map +1 -1
  37. package/dist/env.d.ts +1 -1
  38. package/dist/env.d.ts.map +1 -1
  39. package/dist/env.js +2 -2
  40. package/dist/env.js.map +1 -1
  41. package/dist/errors.js.map +1 -1
  42. package/dist/index.d.ts +1 -1
  43. package/dist/index.d.ts.map +1 -1
  44. package/dist/index.js.map +1 -1
  45. package/dist/modelAdapters.d.ts +1 -1
  46. package/dist/modelAdapters.d.ts.map +1 -1
  47. package/dist/modelAdapters.js +19 -4
  48. package/dist/modelAdapters.js.map +1 -1
  49. package/dist/notices.js.map +1 -1
  50. package/dist/ollama.js.map +1 -1
  51. package/dist/openai.js.map +1 -1
  52. package/dist/pi-transport.d.ts +4 -3
  53. package/dist/pi-transport.d.ts.map +1 -1
  54. package/dist/pi-transport.js +241 -40
  55. package/dist/pi-transport.js.map +1 -1
  56. package/dist/promptTokens.js.map +1 -1
  57. package/dist/provider-env.js.map +1 -1
  58. package/dist/providerError.js.map +1 -1
  59. package/dist/reasoning-effort.js.map +1 -1
  60. package/dist/retry-deadline.js.map +1 -1
  61. package/dist/stream-forensics.d.ts.map +1 -1
  62. package/dist/stream-forensics.js +70 -0
  63. package/dist/stream-forensics.js.map +1 -1
  64. package/dist/types.d.ts +10 -0
  65. package/dist/types.d.ts.map +1 -1
  66. package/dist/types.js.map +1 -1
  67. package/dist/usage.js.map +1 -1
  68. package/dist/warnings.js.map +1 -1
  69. package/docs/models.md +7 -17
  70. package/package.json +5 -5
@@ -89,57 +89,157 @@ const toPiContext = (messages) => {
89
89
  messages: contextMessages,
90
90
  });
91
91
  };
92
- const mapOpenAICompatibleMessages = (messages, systemCacheBreakpoint) => {
92
+ const hasWireContent = (message) => typeof message.content === "string"
93
+ ? message.content.trim().length > 0
94
+ : message.content.some((part) => part.type !== "text" || part.text.trim().length > 0);
95
+ const countWireParts = (parts, limit) => {
96
+ let count = 0;
97
+ const end = Math.min(limit, parts.length);
98
+ for (let i = 0; i < end; i += 1) {
99
+ const part = parts[i];
100
+ if (part.type !== "text" || part.text.trim().length > 0)
101
+ count += 1;
102
+ }
103
+ return count;
104
+ };
105
+ const toOpenAIPart = (part) => {
106
+ if (part.type === "text")
107
+ return { type: "text", text: part.text };
108
+ const base64 = Buffer.from(part.data).toString("base64");
109
+ if (part.mediaType.startsWith("audio/")) {
110
+ return {
111
+ type: "input_audio",
112
+ input_audio: {
113
+ data: base64,
114
+ format: part.mediaType.slice("audio/".length),
115
+ },
116
+ };
117
+ }
118
+ return {
119
+ type: "image_url",
120
+ image_url: { url: `data:${part.mediaType};base64,${base64}` },
121
+ };
122
+ };
123
+ const withEphemeralCacheControl = (parts, atIndex) => {
124
+ if (atIndex >= 0 && atIndex < parts.length) {
125
+ parts[atIndex] = { ...parts[atIndex], cache_control: { type: "ephemeral" } };
126
+ }
127
+ return parts;
128
+ };
129
+ const mapOpenAICompatibleMessages = (messages, cacheBreakpoints, finalUserCachePrefix) => {
93
130
  let lastLeadingSystemIndex = -1;
94
- if (systemCacheBreakpoint) {
95
- for (let i = 0; i < messages.length; i += 1) {
96
- if (messages[i].role !== "system")
97
- break;
131
+ let firstNonSystemIndex = messages.length;
132
+ for (let i = 0; i < messages.length; i += 1) {
133
+ if (messages[i].role !== "system") {
134
+ firstNonSystemIndex = i;
135
+ break;
136
+ }
137
+ if (cacheBreakpoints !== undefined && hasWireContent(messages[i])) {
98
138
  lastLeadingSystemIndex = i;
99
139
  }
100
140
  }
141
+ let penultimateNonSystemIndex = -1;
142
+ let finalUserIndex = -1;
143
+ if (cacheBreakpoints === "conversation") {
144
+ const nonSystemIndices = [];
145
+ for (let i = firstNonSystemIndex; i < messages.length; i += 1) {
146
+ if (hasWireContent(messages[i]))
147
+ nonSystemIndices.push(i);
148
+ }
149
+ if (nonSystemIndices.length >= 2) {
150
+ penultimateNonSystemIndex = nonSystemIndices[nonSystemIndices.length - 2];
151
+ }
152
+ const lastIdx = nonSystemIndices.at(-1);
153
+ if (lastIdx !== undefined && messages[lastIdx].role === "user") {
154
+ finalUserIndex = lastIdx;
155
+ }
156
+ }
101
157
  return messages.map((message, index) => {
102
- if (index === lastLeadingSystemIndex && typeof message.content === "string") {
158
+ if (index === lastLeadingSystemIndex || index === penultimateNonSystemIndex) {
159
+ const parts = typeof message.content === "string"
160
+ ? [{ type: "text", text: message.content }]
161
+ : message.content.map(toOpenAIPart);
103
162
  return {
104
163
  role: message.role,
105
- content: [{
106
- type: "text",
107
- text: message.content,
108
- cache_control: { type: "ephemeral" },
109
- }],
164
+ content: withEphemeralCacheControl(parts, parts.length - 1),
110
165
  };
111
166
  }
112
- if (typeof message.content === "string") {
113
- return { role: message.role, content: message.content };
114
- }
115
- return {
116
- role: message.role,
117
- content: message.content.map((part) => {
118
- if (part.type === "text")
119
- return { type: "text", text: part.text };
120
- const base64 = Buffer.from(part.data).toString("base64");
121
- if (part.mediaType.startsWith("audio/")) {
167
+ if (index === finalUserIndex) {
168
+ if (finalUserCachePrefix === undefined) {
169
+ const parts = typeof message.content === "string"
170
+ ? [{ type: "text", text: message.content }]
171
+ : message.content.map(toOpenAIPart);
172
+ return {
173
+ role: message.role,
174
+ content: withEphemeralCacheControl(parts, parts.length - 1),
175
+ };
176
+ }
177
+ if (finalUserCachePrefix.kind === "chars" && typeof message.content === "string") {
178
+ if (finalUserCachePrefix.length <= 0) {
179
+ return { role: message.role, content: message.content };
180
+ }
181
+ if (finalUserCachePrefix.length >= message.content.length) {
122
182
  return {
123
- type: "input_audio",
124
- input_audio: {
125
- data: base64,
126
- format: part.mediaType.slice("audio/".length),
127
- },
183
+ role: message.role,
184
+ content: [{ type: "text", text: message.content, cache_control: { type: "ephemeral" } }],
128
185
  };
129
186
  }
187
+ const durable = message.content.slice(0, finalUserCachePrefix.length);
188
+ const volatile = message.content.slice(finalUserCachePrefix.length);
189
+ if (durable.trim().length === 0) {
190
+ return { role: message.role, content: message.content };
191
+ }
130
192
  return {
131
- type: "image_url",
132
- image_url: { url: `data:${part.mediaType};base64,${base64}` },
193
+ role: message.role,
194
+ content: volatile.trim().length > 0
195
+ ? [
196
+ { type: "text", text: durable, cache_control: { type: "ephemeral" } },
197
+ { type: "text", text: volatile },
198
+ ]
199
+ : [{ type: "text", text: durable, cache_control: { type: "ephemeral" } }],
133
200
  };
134
- }),
201
+ }
202
+ if (finalUserCachePrefix.kind === "parts" && Array.isArray(message.content)) {
203
+ const parts = message.content.map(toOpenAIPart);
204
+ const targetIndex = Math.min(finalUserCachePrefix.count, parts.length) - 1;
205
+ return {
206
+ role: message.role,
207
+ content: finalUserCachePrefix.count > 0
208
+ ? withEphemeralCacheControl(parts, targetIndex)
209
+ : parts,
210
+ };
211
+ }
212
+ }
213
+ if (typeof message.content === "string") {
214
+ return { role: message.role, content: message.content };
215
+ }
216
+ return {
217
+ role: message.role,
218
+ content: message.content.map(toOpenAIPart),
135
219
  };
136
220
  });
137
221
  };
138
- const applyAnthropicPayloadAdjustments = (params, messages, systemCacheBreakpoint, serviceTier) => {
222
+ const markAnthropicMessageEnd = (wireMsg) => {
223
+ if (!isRecord(wireMsg))
224
+ return;
225
+ if (typeof wireMsg.content === "string") {
226
+ if (wireMsg.content.trim().length > 0) {
227
+ wireMsg.content = [{ type: "text", text: wireMsg.content, cache_control: { type: "ephemeral" } }];
228
+ }
229
+ return;
230
+ }
231
+ if (Array.isArray(wireMsg.content) && wireMsg.content.length > 0) {
232
+ const last = wireMsg.content.at(-1);
233
+ if (isRecord(last)) {
234
+ last.cache_control = { type: "ephemeral" };
235
+ }
236
+ }
237
+ };
238
+ const applyAnthropicPayloadAdjustments = (params, messages, cacheBreakpoints, finalUserCachePrefix, serviceTier) => {
139
239
  if (serviceTier !== undefined) {
140
240
  params.service_tier = serviceTier;
141
241
  }
142
- if (systemCacheBreakpoint && Array.isArray(params.system) && params.system.length > 0) {
242
+ if (cacheBreakpoints !== undefined && Array.isArray(params.system) && params.system.length > 0) {
143
243
  const last = params.system.at(-1);
144
244
  if (isRecord(last)) {
145
245
  last.cache_control = { type: "ephemeral" };
@@ -148,14 +248,15 @@ const applyAnthropicPayloadAdjustments = (params, messages, systemCacheBreakpoin
148
248
  const wireMessages = Array.isArray(params.messages) ? params.messages : undefined;
149
249
  if (wireMessages === undefined)
150
250
  return;
151
- const nonSystem = messages.filter((m) => m.role !== "system");
152
- for (let i = 0; i < Math.min(wireMessages.length, nonSystem.length); i += 1) {
153
- const sourceMsg = nonSystem[i];
251
+ const effectiveNonSystem = messages.filter((m) => m.role !== "system" && hasWireContent(m));
252
+ for (let i = 0; i < Math.min(wireMessages.length, effectiveNonSystem.length); i += 1) {
253
+ const sourceMsg = effectiveNonSystem[i];
154
254
  const wireMsg = wireMessages[i];
155
255
  if (!Array.isArray(sourceMsg.content) || !isRecord(wireMsg) || !Array.isArray(wireMsg.content))
156
256
  continue;
157
- for (let j = 0; j < Math.min(sourceMsg.content.length, wireMsg.content.length); j += 1) {
158
- const sourcePart = sourceMsg.content[j];
257
+ const nonBlankSourceParts = sourceMsg.content.filter((p) => p.type !== "text" || p.text.trim().length > 0);
258
+ for (let j = 0; j < Math.min(nonBlankSourceParts.length, wireMsg.content.length); j += 1) {
259
+ const sourcePart = nonBlankSourceParts[j];
159
260
  const wireBlock = wireMsg.content[j];
160
261
  if (sourcePart.type === "file"
161
262
  && sourcePart.mediaType === "application/pdf"
@@ -165,9 +266,107 @@ const applyAnthropicPayloadAdjustments = (params, messages, systemCacheBreakpoin
165
266
  }
166
267
  }
167
268
  }
269
+ if (cacheBreakpoints !== "conversation" || wireMessages.length === 0)
270
+ return;
271
+ if (wireMessages.length >= 2) {
272
+ markAnthropicMessageEnd(wireMessages[wireMessages.length - 2]);
273
+ }
274
+ const lastWire = wireMessages.at(-1);
275
+ const lastSource = effectiveNonSystem.at(-1);
276
+ if (lastSource?.role !== "user" || !isRecord(lastWire) || lastWire.role !== "user")
277
+ return;
278
+ if (finalUserCachePrefix === undefined) {
279
+ markAnthropicMessageEnd(lastWire);
280
+ return;
281
+ }
282
+ if (finalUserCachePrefix.kind === "chars" && typeof lastSource.content === "string") {
283
+ if (finalUserCachePrefix.length <= 0)
284
+ return;
285
+ if (finalUserCachePrefix.length >= lastSource.content.length) {
286
+ markAnthropicMessageEnd(lastWire);
287
+ return;
288
+ }
289
+ const durable = lastSource.content.slice(0, finalUserCachePrefix.length);
290
+ const volatile = lastSource.content.slice(finalUserCachePrefix.length);
291
+ if (durable.trim().length === 0)
292
+ return;
293
+ lastWire.content = volatile.trim().length > 0
294
+ ? [
295
+ { type: "text", text: durable, cache_control: { type: "ephemeral" } },
296
+ { type: "text", text: volatile },
297
+ ]
298
+ : [{ type: "text", text: durable, cache_control: { type: "ephemeral" } }];
299
+ return;
300
+ }
301
+ if (finalUserCachePrefix.kind === "parts" && Array.isArray(lastSource.content) && Array.isArray(lastWire.content)) {
302
+ const k = countWireParts(lastSource.content, finalUserCachePrefix.count);
303
+ if (k > 0 && k <= lastWire.content.length) {
304
+ const target = lastWire.content[k - 1];
305
+ if (isRecord(target))
306
+ target.cache_control = { type: "ephemeral" };
307
+ }
308
+ }
309
+ };
310
+ const appendBedrockCachePoint = (wireMsg, sourceMsg) => {
311
+ if (sourceMsg === undefined || !hasWireContent(sourceMsg))
312
+ return;
313
+ if (!isRecord(wireMsg) || !Array.isArray(wireMsg.content) || wireMsg.content.length === 0)
314
+ return;
315
+ wireMsg.content = [...wireMsg.content, { cachePoint: { type: "default" } }];
316
+ };
317
+ const applyBedrockPayloadAdjustments = (params, messages, cacheBreakpoints, finalUserCachePrefix) => {
318
+ if (cacheBreakpoints === undefined)
319
+ return;
320
+ if (Array.isArray(params.system) && params.system.length > 0) {
321
+ params.system = [...params.system, { cachePoint: { type: "default" } }];
322
+ }
323
+ if (cacheBreakpoints !== "conversation")
324
+ return;
325
+ const wireMessages = Array.isArray(params.messages) ? params.messages : undefined;
326
+ if (wireMessages === undefined || wireMessages.length === 0)
327
+ return;
328
+ const bedrockNonSystem = messages.filter((m) => m.role === "user" || (m.role === "assistant" && hasWireContent(m)));
329
+ if (wireMessages.length >= 2) {
330
+ appendBedrockCachePoint(wireMessages[wireMessages.length - 2], bedrockNonSystem[bedrockNonSystem.length - 2]);
331
+ }
332
+ const lastWire = wireMessages.at(-1);
333
+ const lastSource = bedrockNonSystem.at(-1);
334
+ if (lastSource?.role !== "user" || !hasWireContent(lastSource) || !isRecord(lastWire) || lastWire.role !== "user" || !Array.isArray(lastWire.content)) {
335
+ return;
336
+ }
337
+ if (finalUserCachePrefix === undefined) {
338
+ appendBedrockCachePoint(lastWire, lastSource);
339
+ return;
340
+ }
341
+ if (finalUserCachePrefix.kind === "chars" && typeof lastSource.content === "string") {
342
+ if (finalUserCachePrefix.length <= 0)
343
+ return;
344
+ if (finalUserCachePrefix.length >= lastSource.content.length) {
345
+ appendBedrockCachePoint(lastWire, lastSource);
346
+ return;
347
+ }
348
+ const durable = lastSource.content.slice(0, finalUserCachePrefix.length);
349
+ const volatile = lastSource.content.slice(finalUserCachePrefix.length);
350
+ if (durable.trim().length === 0)
351
+ return;
352
+ lastWire.content = volatile.trim().length > 0
353
+ ? [{ text: durable }, { cachePoint: { type: "default" } }, { text: volatile }]
354
+ : [{ text: durable }, { cachePoint: { type: "default" } }];
355
+ return;
356
+ }
357
+ if (finalUserCachePrefix.kind === "parts" && Array.isArray(lastSource.content)) {
358
+ const k = countWireParts(lastSource.content, finalUserCachePrefix.count);
359
+ if (k > 0 && k <= lastWire.content.length) {
360
+ lastWire.content = [
361
+ ...lastWire.content.slice(0, k),
362
+ { cachePoint: { type: "default" } },
363
+ ...lastWire.content.slice(k),
364
+ ];
365
+ }
366
+ }
168
367
  };
169
368
  export const executePiTransportStream = async (opts) => {
170
- const { model, source, url, providerModel, fetch: configuredFetch, requestHeaders, messages, sessionId, operationSignal, requestSampling, body, outputTokenField = "max_tokens", effectiveMaxOutputTokens, nativeReasoningBudget, capacity, requestBody, effort, adaptiveEffort, nativeAdaptiveThinking, reasoningRequestBody, systemCacheBreakpoint, cacheAffinity, resolvedControls, rawBody, capture, reasoningStepDown = 0, observers, } = opts;
369
+ const { model, source, url, providerModel, fetch: configuredFetch, requestHeaders, messages, sessionId, operationSignal, requestSampling, body, outputTokenField = "max_tokens", effectiveMaxOutputTokens, nativeReasoningBudget, capacity, requestBody, effort, adaptiveEffort, nativeAdaptiveThinking, reasoningRequestBody, cacheBreakpoints, finalUserCachePrefix, cacheAffinity, resolvedControls, rawBody, capture, reasoningStepDown = 0, observers, } = opts;
171
370
  const providerId = source.replace(/^provider:/, "");
172
371
  const effectiveModel = providerModel ?? {
173
372
  id: model,
@@ -284,9 +483,9 @@ export const executePiTransportStream = async (opts) => {
284
483
  : payload;
285
484
  const params = isRecord(transformed) ? { ...transformed } : {};
286
485
  if (effectiveModel.api === "openai-completions") {
287
- if (systemCacheBreakpoint
486
+ if (cacheBreakpoints !== undefined
288
487
  || messages.some((m) => Array.isArray(m.content) && m.content.some((p) => p.type === "file" && !p.mediaType.startsWith("image/")))) {
289
- params.messages = mapOpenAICompatibleMessages(messages, systemCacheBreakpoint);
488
+ params.messages = mapOpenAICompatibleMessages(messages, cacheBreakpoints, finalUserCachePrefix);
290
489
  }
291
490
  else if (messages[0]?.role === "system" && Array.isArray(params.messages)) {
292
491
  const first = params.messages[0];
@@ -312,7 +511,7 @@ export const executePiTransportStream = async (opts) => {
312
511
  params.top_k = topK;
313
512
  if (stopSequences !== undefined)
314
513
  params.stop_sequences = stopSequences;
315
- applyAnthropicPayloadAdjustments(params, messages, systemCacheBreakpoint, resolvedControls.serviceTier);
514
+ applyAnthropicPayloadAdjustments(params, messages, cacheBreakpoints, finalUserCachePrefix, resolvedControls.serviceTier);
316
515
  }
317
516
  else if (effectiveModel.api === "google-generative-ai") {
318
517
  const config = isRecord(params.config) ? { ...params.config } : {};
@@ -358,6 +557,7 @@ export const executePiTransportStream = async (opts) => {
358
557
  if (resolvedControls.serviceTier !== undefined) {
359
558
  params.serviceTier = { type: resolvedControls.serviceTier };
360
559
  }
560
+ applyBedrockPayloadAdjustments(params, messages, cacheBreakpoints, finalUserCachePrefix);
361
561
  try {
362
562
  capture?.record({
363
563
  method: "POST",
@@ -502,6 +702,7 @@ export const executePiTransportStream = async (opts) => {
502
702
  : (reasoning === "xhigh" ? "max" : reasoning);
503
703
  return streamBedrockConverseStream(buildPiModel("bedrock-converse-stream"), context, {
504
704
  ...commonOptions,
705
+ cacheRetention: "none",
505
706
  ...(bedrockReasoning === undefined ? {} : { reasoning: bedrockReasoning }),
506
707
  ...(nativeReasoningBudget !== null && bedrockReasoning !== undefined
507
708
  ? {