@usagetap/sdk 1.3.1 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1058 -863
- package/dist/adapters/anthropic.cjs +990 -24
- package/dist/adapters/anthropic.cjs.map +1 -1
- package/dist/adapters/anthropic.d.cts +45 -3
- package/dist/adapters/anthropic.d.ts +45 -3
- package/dist/adapters/anthropic.mjs +990 -25
- package/dist/adapters/anthropic.mjs.map +1 -1
- package/dist/adapters/openai.cjs +1164 -44
- package/dist/adapters/openai.cjs.map +1 -1
- package/dist/adapters/openai.d.cts +46 -3
- package/dist/adapters/openai.d.ts +46 -3
- package/dist/adapters/openai.mjs +1164 -45
- package/dist/adapters/openai.mjs.map +1 -1
- package/dist/adapters/openrouter.cjs +3899 -22
- package/dist/adapters/openrouter.cjs.map +1 -1
- package/dist/adapters/openrouter.d.cts +6 -3
- package/dist/adapters/openrouter.d.ts +6 -3
- package/dist/adapters/openrouter.mjs +3897 -23
- package/dist/adapters/openrouter.mjs.map +1 -1
- package/dist/anthropic/index.cjs +990 -24
- package/dist/anthropic/index.cjs.map +1 -1
- package/dist/anthropic/index.d.cts +2 -2
- package/dist/anthropic/index.d.ts +2 -2
- package/dist/anthropic/index.mjs +990 -25
- package/dist/anthropic/index.mjs.map +1 -1
- package/dist/client-CExQ8e1T.d.cts +1225 -0
- package/dist/client-CExQ8e1T.d.ts +1225 -0
- package/dist/express/index.cjs +421 -29
- package/dist/express/index.cjs.map +1 -1
- package/dist/express/index.d.cts +2 -2
- package/dist/express/index.d.ts +2 -2
- package/dist/express/index.mjs +421 -29
- package/dist/express/index.mjs.map +1 -1
- package/dist/index.cjs +680 -15
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +5 -3
- package/dist/index.d.ts +5 -3
- package/dist/index.mjs +680 -15
- package/dist/index.mjs.map +1 -1
- package/dist/openai/index.cjs +1165 -45
- package/dist/openai/index.cjs.map +1 -1
- package/dist/openai/index.d.cts +2 -2
- package/dist/openai/index.d.ts +2 -2
- package/dist/openai/index.mjs +1165 -46
- package/dist/openai/index.mjs.map +1 -1
- package/dist/openrouter/index.cjs +1182 -47
- package/dist/openrouter/index.cjs.map +1 -1
- package/dist/openrouter/index.d.cts +3 -3
- package/dist/openrouter/index.d.ts +3 -3
- package/dist/openrouter/index.mjs +1180 -46
- package/dist/openrouter/index.mjs.map +1 -1
- package/dist/react/index.cjs +19 -1
- package/dist/react/index.cjs.map +1 -1
- package/dist/react/index.d.cts +17 -4
- package/dist/react/index.d.ts +17 -4
- package/dist/react/index.mjs +19 -1
- package/dist/react/index.mjs.map +1 -1
- package/package.json +84 -84
- package/dist/client-BD8O2J8Z.d.cts +0 -668
- package/dist/client-BD8O2J8Z.d.ts +0 -668
package/dist/express/index.mjs
CHANGED
|
@@ -174,6 +174,75 @@ async function pipeToResponse(stream, res, options = {}) {
|
|
|
174
174
|
}
|
|
175
175
|
}
|
|
176
176
|
var USAGETAP_CORRELATION_HEADER = "x-usage-correlation-id";
|
|
177
|
+
function readString(value) {
|
|
178
|
+
return typeof value === "string" && value.trim() ? value : void 0;
|
|
179
|
+
}
|
|
180
|
+
function serializeSamplingError(error) {
|
|
181
|
+
if (error instanceof Error) {
|
|
182
|
+
return { name: error.name, message: error.message };
|
|
183
|
+
}
|
|
184
|
+
return { message: String(error) };
|
|
185
|
+
}
|
|
186
|
+
function normalizeMeteredOpenAISampling(options) {
|
|
187
|
+
if (!options) return void 0;
|
|
188
|
+
if (options === true) return { provider: "openai" };
|
|
189
|
+
const { provider = "openai", ...policyFields } = options;
|
|
190
|
+
return {
|
|
191
|
+
provider,
|
|
192
|
+
policy: typeof policyFields.rate === "number" ? policyFields : void 0
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
function startMeteredOpenAISampleDecision({
|
|
196
|
+
usageTap,
|
|
197
|
+
sampling,
|
|
198
|
+
beginRequest,
|
|
199
|
+
input
|
|
200
|
+
}) {
|
|
201
|
+
if (!sampling) return Promise.resolve(false);
|
|
202
|
+
return usageTap.shouldSampleAsync(
|
|
203
|
+
{
|
|
204
|
+
customerId: beginRequest.customerId,
|
|
205
|
+
feature: beginRequest.feature,
|
|
206
|
+
input
|
|
207
|
+
},
|
|
208
|
+
sampling.policy
|
|
209
|
+
);
|
|
210
|
+
}
|
|
211
|
+
async function captureMeteredOpenAISample({
|
|
212
|
+
usageTap,
|
|
213
|
+
sampling,
|
|
214
|
+
decision,
|
|
215
|
+
ctx,
|
|
216
|
+
beginRequest,
|
|
217
|
+
input,
|
|
218
|
+
response,
|
|
219
|
+
error,
|
|
220
|
+
startedAt
|
|
221
|
+
}) {
|
|
222
|
+
if (!sampling) return;
|
|
223
|
+
let selected = false;
|
|
224
|
+
try {
|
|
225
|
+
selected = await decision;
|
|
226
|
+
} catch {
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
if (!selected) return;
|
|
230
|
+
const record = isObjectRecord(response) ? response : {};
|
|
231
|
+
await usageTap.captureSample({
|
|
232
|
+
sampleId: ctx.begin.data.callId,
|
|
233
|
+
callId: ctx.begin.data.callId,
|
|
234
|
+
customerId: beginRequest.customerId,
|
|
235
|
+
feature: beginRequest.feature,
|
|
236
|
+
tags: beginRequest.tags,
|
|
237
|
+
provider: sampling.provider,
|
|
238
|
+
model: readString(record.model) ?? readString(input.model),
|
|
239
|
+
input,
|
|
240
|
+
...response === void 0 ? {} : { output: response },
|
|
241
|
+
usage: record.usage,
|
|
242
|
+
latencyMs: Date.now() - startedAt,
|
|
243
|
+
...error === void 0 ? {} : { error: serializeSamplingError(error) }
|
|
244
|
+
}).catch(() => void 0);
|
|
245
|
+
}
|
|
177
246
|
function wrapOpenAI(client, usageTap, options = {}) {
|
|
178
247
|
if (!client) {
|
|
179
248
|
throw new UsageTapError("USAGETAP_BAD_REQUEST", "wrapOpenAI requires an OpenAI client instance");
|
|
@@ -181,6 +250,8 @@ function wrapOpenAI(client, usageTap, options = {}) {
|
|
|
181
250
|
const defaultContext = options.defaultContext;
|
|
182
251
|
const applyVendorHints = options.applyVendorHints !== false;
|
|
183
252
|
const defaultPromptCompression = normalizePromptCompressionOptions(options.promptCompression);
|
|
253
|
+
const defaultSampling = normalizeMeteredOpenAISampling(options.sampling);
|
|
254
|
+
const provider = options.provider ?? "openai";
|
|
184
255
|
const promptCompressionStats = new OpenAIPromptCompressionStats();
|
|
185
256
|
const proxiedChat = client.chat ? createChatProxy(
|
|
186
257
|
client.chat,
|
|
@@ -188,7 +259,9 @@ function wrapOpenAI(client, usageTap, options = {}) {
|
|
|
188
259
|
defaultContext,
|
|
189
260
|
applyVendorHints,
|
|
190
261
|
defaultPromptCompression,
|
|
191
|
-
promptCompressionStats
|
|
262
|
+
promptCompressionStats,
|
|
263
|
+
defaultSampling,
|
|
264
|
+
provider
|
|
192
265
|
) : void 0;
|
|
193
266
|
const proxiedResponses = typeof client.responses !== "undefined" ? createResponsesProxy(
|
|
194
267
|
client.responses,
|
|
@@ -196,7 +269,9 @@ function wrapOpenAI(client, usageTap, options = {}) {
|
|
|
196
269
|
defaultContext,
|
|
197
270
|
applyVendorHints,
|
|
198
271
|
defaultPromptCompression,
|
|
199
|
-
promptCompressionStats
|
|
272
|
+
promptCompressionStats,
|
|
273
|
+
defaultSampling,
|
|
274
|
+
provider
|
|
200
275
|
) : void 0;
|
|
201
276
|
const handler = {
|
|
202
277
|
get(target, prop, receiver) {
|
|
@@ -223,14 +298,16 @@ function wrapOpenAI(client, usageTap, options = {}) {
|
|
|
223
298
|
};
|
|
224
299
|
return new Proxy(client, handler);
|
|
225
300
|
}
|
|
226
|
-
function createChatProxy(resource, usageTap, defaultContext, applyVendorHints, defaultPromptCompression, promptCompressionStats) {
|
|
301
|
+
function createChatProxy(resource, usageTap, defaultContext, applyVendorHints, defaultPromptCompression, promptCompressionStats, defaultSampling, provider) {
|
|
227
302
|
const completions = createChatCompletionsProxy(
|
|
228
303
|
resource.completions,
|
|
229
304
|
usageTap,
|
|
230
305
|
defaultContext,
|
|
231
306
|
applyVendorHints,
|
|
232
307
|
defaultPromptCompression,
|
|
233
|
-
promptCompressionStats
|
|
308
|
+
promptCompressionStats,
|
|
309
|
+
defaultSampling,
|
|
310
|
+
provider
|
|
234
311
|
);
|
|
235
312
|
const handler = {
|
|
236
313
|
get(target, prop, receiver) {
|
|
@@ -242,7 +319,7 @@ function createChatProxy(resource, usageTap, defaultContext, applyVendorHints, d
|
|
|
242
319
|
};
|
|
243
320
|
return new Proxy(resource, handler);
|
|
244
321
|
}
|
|
245
|
-
function createResponsesProxy(resource, usageTap, defaultContext, applyVendorHints, defaultPromptCompression, promptCompressionStats) {
|
|
322
|
+
function createResponsesProxy(resource, usageTap, defaultContext, applyVendorHints, defaultPromptCompression, promptCompressionStats, defaultSampling, provider) {
|
|
246
323
|
if (!resource || typeof resource !== "object") {
|
|
247
324
|
return void 0;
|
|
248
325
|
}
|
|
@@ -257,9 +334,20 @@ function createResponsesProxy(resource, usageTap, defaultContext, applyVendorHin
|
|
|
257
334
|
withUsage: withUsage2,
|
|
258
335
|
promptCompression
|
|
259
336
|
} = splitUsageOptions(options);
|
|
260
|
-
const beginRequest =
|
|
337
|
+
const beginRequest = withRuntimeCompressionContext(
|
|
338
|
+
resolveBeginRequest(defaultContext, usageContext),
|
|
339
|
+
"openai",
|
|
340
|
+
String(params.model)
|
|
341
|
+
);
|
|
261
342
|
const wantsStream = isStreamingRequest(params);
|
|
262
343
|
return usageTap.withUsage(beginRequest, async (ctx) => {
|
|
344
|
+
const sampleStartedAt = Date.now();
|
|
345
|
+
const sampleDecision = wantsStream ? Promise.resolve(false) : startMeteredOpenAISampleDecision({
|
|
346
|
+
usageTap,
|
|
347
|
+
sampling: defaultSampling,
|
|
348
|
+
beginRequest,
|
|
349
|
+
input: params
|
|
350
|
+
});
|
|
263
351
|
const hintedParams = applyVendorHints ? applyResponsesVendorHints(params, ctx.begin.data.vendorHints) : params;
|
|
264
352
|
const finalParams = await compressResponsesParamsForCall({
|
|
265
353
|
params: hintedParams,
|
|
@@ -271,13 +359,14 @@ function createResponsesProxy(resource, usageTap, defaultContext, applyVendorHin
|
|
|
271
359
|
withUsage: withUsage2,
|
|
272
360
|
operation: "responses.create"
|
|
273
361
|
});
|
|
362
|
+
ctx.setUsage(openAIRequestExecutionMetadata(finalParams, provider));
|
|
274
363
|
const request = attachCorrelationHeader(requestOptions, ctx.begin.correlationId);
|
|
275
364
|
if (wantsStream) {
|
|
276
365
|
const apiPromise2 = originalCreate(finalParams, request);
|
|
277
366
|
const wrappedPromise2 = transformApiPromise(apiPromise2, (rawStream) => {
|
|
278
367
|
ensureAsyncIterable(rawStream, "responses.create");
|
|
279
368
|
const wrappedStream = wrapStreamForUsageTap(rawStream, async () => {
|
|
280
|
-
const usage = await extractUsageFromStream(rawStream, ctx.begin.data.vendorHints);
|
|
369
|
+
const usage = await extractUsageFromStream(rawStream, ctx.begin.data.vendorHints, provider);
|
|
281
370
|
if (usage) {
|
|
282
371
|
ctx.setUsage(usage);
|
|
283
372
|
}
|
|
@@ -287,9 +376,31 @@ function createResponsesProxy(resource, usageTap, defaultContext, applyVendorHin
|
|
|
287
376
|
return wrappedPromise2;
|
|
288
377
|
}
|
|
289
378
|
const apiPromise = originalCreate(finalParams, request);
|
|
290
|
-
const wrappedPromise = transformApiPromise(apiPromise, (response) => {
|
|
291
|
-
tryInferUsage(response, ctx.begin.data.vendorHints, void 0, ctx);
|
|
379
|
+
const wrappedPromise = transformApiPromise(apiPromise, async (response) => {
|
|
380
|
+
tryInferUsage(response, ctx.begin.data.vendorHints, void 0, ctx, provider);
|
|
381
|
+
await captureMeteredOpenAISample({
|
|
382
|
+
usageTap,
|
|
383
|
+
sampling: defaultSampling,
|
|
384
|
+
decision: sampleDecision,
|
|
385
|
+
ctx,
|
|
386
|
+
beginRequest,
|
|
387
|
+
input: params,
|
|
388
|
+
response,
|
|
389
|
+
startedAt: sampleStartedAt
|
|
390
|
+
});
|
|
292
391
|
return response;
|
|
392
|
+
}, async (error) => {
|
|
393
|
+
await captureMeteredOpenAISample({
|
|
394
|
+
usageTap,
|
|
395
|
+
sampling: defaultSampling,
|
|
396
|
+
decision: sampleDecision,
|
|
397
|
+
ctx,
|
|
398
|
+
beginRequest,
|
|
399
|
+
input: params,
|
|
400
|
+
error,
|
|
401
|
+
startedAt: sampleStartedAt
|
|
402
|
+
});
|
|
403
|
+
throw error;
|
|
293
404
|
});
|
|
294
405
|
return wrappedPromise;
|
|
295
406
|
}, withUsage2);
|
|
@@ -304,7 +415,7 @@ function createResponsesProxy(resource, usageTap, defaultContext, applyVendorHin
|
|
|
304
415
|
};
|
|
305
416
|
return new Proxy(resource, handler);
|
|
306
417
|
}
|
|
307
|
-
function createChatCompletionsProxy(resource, usageTap, defaultContext, applyVendorHints, defaultPromptCompression, promptCompressionStats) {
|
|
418
|
+
function createChatCompletionsProxy(resource, usageTap, defaultContext, applyVendorHints, defaultPromptCompression, promptCompressionStats, defaultSampling, provider) {
|
|
308
419
|
const originalCreate = resource.create.bind(resource);
|
|
309
420
|
const streamCandidate = resource.stream;
|
|
310
421
|
const originalStream = typeof streamCandidate === "function" ? streamCandidate.bind(resource) : void 0;
|
|
@@ -315,9 +426,20 @@ function createChatCompletionsProxy(resource, usageTap, defaultContext, applyVen
|
|
|
315
426
|
withUsage: withUsage2,
|
|
316
427
|
promptCompression
|
|
317
428
|
} = splitUsageOptions(options);
|
|
318
|
-
const beginRequest =
|
|
429
|
+
const beginRequest = withRuntimeCompressionContext(
|
|
430
|
+
resolveBeginRequest(defaultContext, usageContext),
|
|
431
|
+
"openai",
|
|
432
|
+
String(params.model)
|
|
433
|
+
);
|
|
319
434
|
const wantsStream = isStreamingRequest(params);
|
|
320
435
|
return usageTap.withUsage(beginRequest, async (ctx) => {
|
|
436
|
+
const sampleStartedAt = Date.now();
|
|
437
|
+
const sampleDecision = wantsStream ? Promise.resolve(false) : startMeteredOpenAISampleDecision({
|
|
438
|
+
usageTap,
|
|
439
|
+
sampling: defaultSampling,
|
|
440
|
+
beginRequest,
|
|
441
|
+
input: params
|
|
442
|
+
});
|
|
321
443
|
const hintedParams = applyVendorHints ? applyChatVendorHints(params, ctx.begin.data.vendorHints) : params;
|
|
322
444
|
const finalParams = await compressChatParamsForCall({
|
|
323
445
|
params: hintedParams,
|
|
@@ -329,13 +451,14 @@ function createChatCompletionsProxy(resource, usageTap, defaultContext, applyVen
|
|
|
329
451
|
withUsage: withUsage2,
|
|
330
452
|
operation: "chat.completions.create"
|
|
331
453
|
});
|
|
454
|
+
ctx.setUsage(openAIRequestExecutionMetadata(finalParams, provider));
|
|
332
455
|
const request = attachCorrelationHeader(requestOptions, ctx.begin.correlationId);
|
|
333
456
|
if (wantsStream) {
|
|
334
457
|
const apiPromise2 = originalCreate(finalParams, request);
|
|
335
458
|
const wrappedPromise2 = transformApiPromise(apiPromise2, (rawStream) => {
|
|
336
459
|
ensureAsyncIterable(rawStream, "chat.completions.create");
|
|
337
460
|
const wrappedStream2 = wrapStreamForUsageTap(rawStream, async () => {
|
|
338
|
-
const usage = await extractUsageFromStream(rawStream, ctx.begin.data.vendorHints);
|
|
461
|
+
const usage = await extractUsageFromStream(rawStream, ctx.begin.data.vendorHints, provider);
|
|
339
462
|
if (usage) {
|
|
340
463
|
ctx.setUsage(usage);
|
|
341
464
|
}
|
|
@@ -345,9 +468,31 @@ function createChatCompletionsProxy(resource, usageTap, defaultContext, applyVen
|
|
|
345
468
|
return wrappedPromise2;
|
|
346
469
|
}
|
|
347
470
|
const apiPromise = originalCreate(finalParams, request);
|
|
348
|
-
const wrappedPromise = transformApiPromise(apiPromise, (response) => {
|
|
349
|
-
tryInferUsage(response, ctx.begin.data.vendorHints, void 0, ctx);
|
|
471
|
+
const wrappedPromise = transformApiPromise(apiPromise, async (response) => {
|
|
472
|
+
tryInferUsage(response, ctx.begin.data.vendorHints, void 0, ctx, provider);
|
|
473
|
+
await captureMeteredOpenAISample({
|
|
474
|
+
usageTap,
|
|
475
|
+
sampling: defaultSampling,
|
|
476
|
+
decision: sampleDecision,
|
|
477
|
+
ctx,
|
|
478
|
+
beginRequest,
|
|
479
|
+
input: params,
|
|
480
|
+
response,
|
|
481
|
+
startedAt: sampleStartedAt
|
|
482
|
+
});
|
|
350
483
|
return response;
|
|
484
|
+
}, async (error) => {
|
|
485
|
+
await captureMeteredOpenAISample({
|
|
486
|
+
usageTap,
|
|
487
|
+
sampling: defaultSampling,
|
|
488
|
+
decision: sampleDecision,
|
|
489
|
+
ctx,
|
|
490
|
+
beginRequest,
|
|
491
|
+
input: params,
|
|
492
|
+
error,
|
|
493
|
+
startedAt: sampleStartedAt
|
|
494
|
+
});
|
|
495
|
+
throw error;
|
|
351
496
|
});
|
|
352
497
|
return wrappedPromise;
|
|
353
498
|
}, withUsage2);
|
|
@@ -359,7 +504,11 @@ function createChatCompletionsProxy(resource, usageTap, defaultContext, applyVen
|
|
|
359
504
|
withUsage: withUsage2,
|
|
360
505
|
promptCompression
|
|
361
506
|
} = splitUsageOptions(options);
|
|
362
|
-
const beginRequest =
|
|
507
|
+
const beginRequest = withRuntimeCompressionContext(
|
|
508
|
+
resolveBeginRequest(defaultContext, usageContext),
|
|
509
|
+
"openai",
|
|
510
|
+
String(params.model)
|
|
511
|
+
);
|
|
363
512
|
return usageTap.withUsage(beginRequest, async (ctx) => {
|
|
364
513
|
const hintedParams = applyVendorHints ? applyChatVendorHints(params, ctx.begin.data.vendorHints) : params;
|
|
365
514
|
const finalParams = await compressChatParamsForCall({
|
|
@@ -372,12 +521,17 @@ function createChatCompletionsProxy(resource, usageTap, defaultContext, applyVen
|
|
|
372
521
|
withUsage: withUsage2,
|
|
373
522
|
operation: "chat.completions.stream"
|
|
374
523
|
});
|
|
524
|
+
ctx.setUsage(openAIRequestExecutionMetadata(finalParams, provider));
|
|
375
525
|
const request = attachCorrelationHeader(requestOptions, ctx.begin.correlationId);
|
|
376
526
|
const apiPromise = originalStream(finalParams, request);
|
|
377
527
|
const wrappedPromise = transformApiPromise(apiPromise, (rawStream) => {
|
|
378
528
|
ensureAsyncIterable(rawStream, "chat.completions.stream");
|
|
379
529
|
const wrappedStreamInner = wrapStreamForUsageTap(rawStream, async () => {
|
|
380
|
-
const usage = await extractUsageFromStream(
|
|
530
|
+
const usage = await extractUsageFromStream(
|
|
531
|
+
rawStream,
|
|
532
|
+
ctx.begin.data.vendorHints,
|
|
533
|
+
provider
|
|
534
|
+
);
|
|
381
535
|
if (usage) {
|
|
382
536
|
ctx.setUsage(usage);
|
|
383
537
|
}
|
|
@@ -401,19 +555,70 @@ function createChatCompletionsProxy(resource, usageTap, defaultContext, applyVen
|
|
|
401
555
|
return new Proxy(resource, handler);
|
|
402
556
|
}
|
|
403
557
|
async function compressChatParamsForCall(args) {
|
|
404
|
-
|
|
558
|
+
let compression = resolveEffectivePromptCompressionOptions(
|
|
405
559
|
args.defaultPromptCompression,
|
|
406
560
|
args.callPromptCompression
|
|
407
561
|
);
|
|
562
|
+
const runtimePolicy = args.ctx.begin.data.runtimeCompressionPolicy;
|
|
563
|
+
const policyDriven = !compression && Boolean(runtimePolicy);
|
|
564
|
+
if (policyDriven && runtimePolicy) {
|
|
565
|
+
if (!runtimePolicy.selected) {
|
|
566
|
+
args.ctx.setUsage({
|
|
567
|
+
runtimeCompression: bypassedRuntimeMeasurement(runtimePolicy)
|
|
568
|
+
});
|
|
569
|
+
return args.params;
|
|
570
|
+
}
|
|
571
|
+
compression = openAIPolicyCompressionOptions(runtimePolicy);
|
|
572
|
+
if (runtimePolicy.mode === "SHADOW") {
|
|
573
|
+
const startedAt2 = Date.now();
|
|
574
|
+
void compressChatParams(
|
|
575
|
+
args.params,
|
|
576
|
+
args.usageTap,
|
|
577
|
+
compression,
|
|
578
|
+
args.withUsage?.signal
|
|
579
|
+
).then((outcome2) => {
|
|
580
|
+
args.ctx.setUsage({
|
|
581
|
+
runtimeCompression: openAIRuntimeMeasurement({
|
|
582
|
+
policy: runtimePolicy,
|
|
583
|
+
telemetry: buildPromptCompressionTelemetry(outcome2.segments),
|
|
584
|
+
latencyMs: Date.now() - startedAt2,
|
|
585
|
+
shadow: true,
|
|
586
|
+
originalPayload: args.params,
|
|
587
|
+
resultPayload: outcome2.params
|
|
588
|
+
})
|
|
589
|
+
});
|
|
590
|
+
}).catch(() => {
|
|
591
|
+
args.ctx.setUsage({
|
|
592
|
+
runtimeCompression: failedRuntimeMeasurement(
|
|
593
|
+
runtimePolicy,
|
|
594
|
+
Date.now() - startedAt2
|
|
595
|
+
)
|
|
596
|
+
});
|
|
597
|
+
});
|
|
598
|
+
return args.params;
|
|
599
|
+
}
|
|
600
|
+
}
|
|
408
601
|
if (!compression) {
|
|
409
602
|
return args.params;
|
|
410
603
|
}
|
|
604
|
+
const startedAt = Date.now();
|
|
411
605
|
const outcome = await compressChatParams(
|
|
412
606
|
args.params,
|
|
413
607
|
args.usageTap,
|
|
414
608
|
compression,
|
|
415
609
|
args.withUsage?.signal
|
|
416
610
|
);
|
|
611
|
+
if (policyDriven && runtimePolicy) {
|
|
612
|
+
const runtimeMeasurement = openAIRuntimeMeasurement({
|
|
613
|
+
policy: runtimePolicy,
|
|
614
|
+
telemetry: buildPromptCompressionTelemetry(outcome.segments),
|
|
615
|
+
latencyMs: Date.now() - startedAt,
|
|
616
|
+
originalPayload: args.params,
|
|
617
|
+
resultPayload: outcome.params
|
|
618
|
+
});
|
|
619
|
+
args.ctx.setUsage({ runtimeCompression: runtimeMeasurement });
|
|
620
|
+
if (runtimeMeasurement.decision !== "compressed") return args.params;
|
|
621
|
+
}
|
|
417
622
|
await recordCompressionOutcome({
|
|
418
623
|
outcome,
|
|
419
624
|
compression,
|
|
@@ -426,19 +631,70 @@ async function compressChatParamsForCall(args) {
|
|
|
426
631
|
return outcome.params;
|
|
427
632
|
}
|
|
428
633
|
async function compressResponsesParamsForCall(args) {
|
|
429
|
-
|
|
634
|
+
let compression = resolveEffectivePromptCompressionOptions(
|
|
430
635
|
args.defaultPromptCompression,
|
|
431
636
|
args.callPromptCompression
|
|
432
637
|
);
|
|
638
|
+
const runtimePolicy = args.ctx.begin.data.runtimeCompressionPolicy;
|
|
639
|
+
const policyDriven = !compression && Boolean(runtimePolicy);
|
|
640
|
+
if (policyDriven && runtimePolicy) {
|
|
641
|
+
if (!runtimePolicy.selected) {
|
|
642
|
+
args.ctx.setUsage({
|
|
643
|
+
runtimeCompression: bypassedRuntimeMeasurement(runtimePolicy)
|
|
644
|
+
});
|
|
645
|
+
return args.params;
|
|
646
|
+
}
|
|
647
|
+
compression = openAIPolicyCompressionOptions(runtimePolicy);
|
|
648
|
+
if (runtimePolicy.mode === "SHADOW") {
|
|
649
|
+
const startedAt2 = Date.now();
|
|
650
|
+
void compressResponsesParams(
|
|
651
|
+
args.params,
|
|
652
|
+
args.usageTap,
|
|
653
|
+
compression,
|
|
654
|
+
args.withUsage?.signal
|
|
655
|
+
).then((outcome2) => {
|
|
656
|
+
args.ctx.setUsage({
|
|
657
|
+
runtimeCompression: openAIRuntimeMeasurement({
|
|
658
|
+
policy: runtimePolicy,
|
|
659
|
+
telemetry: buildPromptCompressionTelemetry(outcome2.segments),
|
|
660
|
+
latencyMs: Date.now() - startedAt2,
|
|
661
|
+
shadow: true,
|
|
662
|
+
originalPayload: args.params,
|
|
663
|
+
resultPayload: outcome2.params
|
|
664
|
+
})
|
|
665
|
+
});
|
|
666
|
+
}).catch(() => {
|
|
667
|
+
args.ctx.setUsage({
|
|
668
|
+
runtimeCompression: failedRuntimeMeasurement(
|
|
669
|
+
runtimePolicy,
|
|
670
|
+
Date.now() - startedAt2
|
|
671
|
+
)
|
|
672
|
+
});
|
|
673
|
+
});
|
|
674
|
+
return args.params;
|
|
675
|
+
}
|
|
676
|
+
}
|
|
433
677
|
if (!compression) {
|
|
434
678
|
return args.params;
|
|
435
679
|
}
|
|
680
|
+
const startedAt = Date.now();
|
|
436
681
|
const outcome = await compressResponsesParams(
|
|
437
682
|
args.params,
|
|
438
683
|
args.usageTap,
|
|
439
684
|
compression,
|
|
440
685
|
args.withUsage?.signal
|
|
441
686
|
);
|
|
687
|
+
if (policyDriven && runtimePolicy) {
|
|
688
|
+
const runtimeMeasurement = openAIRuntimeMeasurement({
|
|
689
|
+
policy: runtimePolicy,
|
|
690
|
+
telemetry: buildPromptCompressionTelemetry(outcome.segments),
|
|
691
|
+
latencyMs: Date.now() - startedAt,
|
|
692
|
+
originalPayload: args.params,
|
|
693
|
+
resultPayload: outcome.params
|
|
694
|
+
});
|
|
695
|
+
args.ctx.setUsage({ runtimeCompression: runtimeMeasurement });
|
|
696
|
+
if (runtimeMeasurement.decision !== "compressed") return args.params;
|
|
697
|
+
}
|
|
442
698
|
await recordCompressionOutcome({
|
|
443
699
|
outcome,
|
|
444
700
|
compression,
|
|
@@ -450,6 +706,79 @@ async function compressResponsesParamsForCall(args) {
|
|
|
450
706
|
});
|
|
451
707
|
return outcome.params;
|
|
452
708
|
}
|
|
709
|
+
function openAIPolicyCompressionOptions(policy) {
|
|
710
|
+
const roles = Object.fromEntries(
|
|
711
|
+
["system", "user", "tool", "assistant"].filter((role) => policy.snapshot.scope.messageRoles.includes(role)).map((role) => [role, true])
|
|
712
|
+
);
|
|
713
|
+
return {
|
|
714
|
+
provider: "usagetap",
|
|
715
|
+
roles,
|
|
716
|
+
minContextTokens: policy.snapshot.scope.minimumTokens,
|
|
717
|
+
failOpen: true
|
|
718
|
+
};
|
|
719
|
+
}
|
|
720
|
+
function bypassedRuntimeMeasurement(policy) {
|
|
721
|
+
return {
|
|
722
|
+
policyId: policy.policyId,
|
|
723
|
+
policyVersionId: policy.policyVersionId,
|
|
724
|
+
policyVersion: policy.version,
|
|
725
|
+
rolloutMode: policy.mode,
|
|
726
|
+
decision: "bypassed",
|
|
727
|
+
reasonCode: "not_in_rollout_sample",
|
|
728
|
+
methodsAttempted: [],
|
|
729
|
+
methodsApplied: [],
|
|
730
|
+
originalTokens: 0,
|
|
731
|
+
resultTokens: 0,
|
|
732
|
+
savedTokens: 0,
|
|
733
|
+
reductionPercentage: 0,
|
|
734
|
+
compressionLatencyMs: 0,
|
|
735
|
+
compressionComputeCostUsd: 0,
|
|
736
|
+
financialsEstimated: true
|
|
737
|
+
};
|
|
738
|
+
}
|
|
739
|
+
function failedRuntimeMeasurement(policy, latencyMs) {
|
|
740
|
+
return {
|
|
741
|
+
...bypassedRuntimeMeasurement(policy),
|
|
742
|
+
decision: "fallback",
|
|
743
|
+
reasonCode: "transformation_failed",
|
|
744
|
+
methodsAttempted: ["deterministic"],
|
|
745
|
+
compressionLatencyMs: latencyMs
|
|
746
|
+
};
|
|
747
|
+
}
|
|
748
|
+
function openAIRuntimeMeasurement(input) {
|
|
749
|
+
const telemetry = input.telemetry;
|
|
750
|
+
const originalTokens = telemetry?.originalTokens ?? estimatePromptTokens(JSON.stringify(input.originalPayload));
|
|
751
|
+
const resultTokens = telemetry?.compressedTokens ?? originalTokens;
|
|
752
|
+
const savedTokens = Math.max(0, telemetry?.savedTokens ?? 0);
|
|
753
|
+
const reductionPercentage = originalTokens > 0 ? savedTokens / originalTokens * 100 : 0;
|
|
754
|
+
const belowMinimumSize = originalTokens < input.policy.snapshot.scope.minimumTokens;
|
|
755
|
+
const latencyExceeded = input.latencyMs > input.policy.snapshot.gates.maximumLatencyMs;
|
|
756
|
+
const belowSavings = savedTokens < input.policy.snapshot.gates.minimumTokensRemoved || reductionPercentage < input.policy.snapshot.gates.minimumReductionPercentage;
|
|
757
|
+
const accepted = !belowMinimumSize && !latencyExceeded && !belowSavings;
|
|
758
|
+
const deepText = input.policy.snapshot.methods.deepText.enabled && !input.policy.snapshot.rollout.deterministicOnly && (input.policy.mode !== "CANARY" || input.policy.snapshot.rollout.deepTextCanaryAllowed);
|
|
759
|
+
const reasonCode = belowMinimumSize ? "below_minimum_size" : latencyExceeded ? "latency_budget_exceeded" : belowSavings ? "below_minimum_savings" : void 0;
|
|
760
|
+
const measurement = {
|
|
761
|
+
policyId: input.policy.policyId,
|
|
762
|
+
policyVersionId: input.policy.policyVersionId,
|
|
763
|
+
policyVersion: input.policy.version,
|
|
764
|
+
rolloutMode: input.policy.mode,
|
|
765
|
+
decision: accepted ? input.shadow ? "bypassed" : "compressed" : latencyExceeded ? "fallback" : "skipped",
|
|
766
|
+
...reasonCode ? { reasonCode } : {},
|
|
767
|
+
methodsAttempted: [
|
|
768
|
+
"deterministic",
|
|
769
|
+
...deepText ? ["deep_text"] : []
|
|
770
|
+
],
|
|
771
|
+
methodsApplied: accepted ? ["deterministic"] : [],
|
|
772
|
+
originalTokens,
|
|
773
|
+
resultTokens: accepted ? resultTokens : originalTokens,
|
|
774
|
+
savedTokens: accepted ? savedTokens : 0,
|
|
775
|
+
reductionPercentage: accepted ? reductionPercentage : 0,
|
|
776
|
+
compressionLatencyMs: input.latencyMs,
|
|
777
|
+
compressionComputeCostUsd: 0,
|
|
778
|
+
financialsEstimated: true
|
|
779
|
+
};
|
|
780
|
+
return measurement;
|
|
781
|
+
}
|
|
453
782
|
async function recordCompressionOutcome(args) {
|
|
454
783
|
const telemetry = buildPromptCompressionTelemetry(args.outcome.segments);
|
|
455
784
|
if (!telemetry) {
|
|
@@ -503,6 +832,10 @@ async function compressChatParams(params, usageTap, compression, signal) {
|
|
|
503
832
|
const result = await usageTap.compressPromptMessages(source, {
|
|
504
833
|
provider: "usagetap",
|
|
505
834
|
failOpen: compression.failOpen,
|
|
835
|
+
mode: compression.mode,
|
|
836
|
+
latencyBudgetMs: compression.latencyBudgetMs,
|
|
837
|
+
compactEmptyUserMessages: compression.compactEmptyUserMessages,
|
|
838
|
+
compactDuplicateUserTextParts: compression.compactDuplicateUserTextParts,
|
|
506
839
|
aggressiveness: resolveMessageEndpointAggressiveness(compression),
|
|
507
840
|
signal
|
|
508
841
|
});
|
|
@@ -968,23 +1301,53 @@ function resolveBeginRequest(defaults, override) {
|
|
|
968
1301
|
}
|
|
969
1302
|
const tags = mergeTags(base.tags, current.tags);
|
|
970
1303
|
const begin = { customerId };
|
|
1304
|
+
const runtimeCompressionContext = current.runtimeCompressionContext ?? base.runtimeCompressionContext;
|
|
1305
|
+
if (runtimeCompressionContext) {
|
|
1306
|
+
begin.runtimeCompressionContext = runtimeCompressionContext;
|
|
1307
|
+
}
|
|
971
1308
|
const requested = current.requested ?? base.requested;
|
|
972
1309
|
if (requested) begin.requested = requested;
|
|
973
1310
|
const feature = current.feature ?? base.feature;
|
|
974
1311
|
if (feature) begin.feature = feature;
|
|
1312
|
+
const runId = current.runId ?? base.runId;
|
|
1313
|
+
if (runId) begin.runId = runId;
|
|
975
1314
|
const idempotency = current.idempotency ?? base.idempotency;
|
|
976
1315
|
if (idempotency) begin.idempotency = idempotency;
|
|
977
1316
|
const customerName = current.customerName ?? base.customerName;
|
|
978
1317
|
if (customerName) begin.customerName = customerName;
|
|
979
1318
|
const customerEmail = current.customerEmail ?? base.customerEmail;
|
|
980
1319
|
if (customerEmail) begin.customerEmail = customerEmail;
|
|
1320
|
+
const customerUserId = current.customerUserId ?? base.customerUserId;
|
|
1321
|
+
if (customerUserId) begin.customerUserId = customerUserId;
|
|
1322
|
+
const customerUserName = current.customerUserName ?? base.customerUserName;
|
|
1323
|
+
if (customerUserName) begin.customerUserName = customerUserName;
|
|
1324
|
+
const customerUserEmail = current.customerUserEmail ?? base.customerUserEmail;
|
|
1325
|
+
if (customerUserEmail) begin.customerUserEmail = customerUserEmail;
|
|
1326
|
+
const stripeCustomerId = current.stripeCustomerId ?? base.stripeCustomerId;
|
|
1327
|
+
if (stripeCustomerId) begin.stripeCustomerId = stripeCustomerId;
|
|
1328
|
+
const holdUsd = current.holdUsd ?? base.holdUsd;
|
|
1329
|
+
if (typeof holdUsd === "number") begin.holdUsd = holdUsd;
|
|
1330
|
+
const batch = current.batch ?? base.batch;
|
|
1331
|
+
if (typeof batch === "boolean") begin.batch = batch;
|
|
1332
|
+
const pricingMode = current.pricingMode ?? base.pricingMode;
|
|
1333
|
+
if (pricingMode) begin.pricingMode = pricingMode;
|
|
981
1334
|
if (tags?.length) {
|
|
982
1335
|
begin.tags = tags;
|
|
983
1336
|
}
|
|
984
1337
|
return begin;
|
|
985
1338
|
}
|
|
986
|
-
function
|
|
987
|
-
|
|
1339
|
+
function withRuntimeCompressionContext(begin, provider, model) {
|
|
1340
|
+
return {
|
|
1341
|
+
...begin,
|
|
1342
|
+
runtimeCompressionContext: {
|
|
1343
|
+
...begin.runtimeCompressionContext ?? {},
|
|
1344
|
+
provider,
|
|
1345
|
+
model
|
|
1346
|
+
}
|
|
1347
|
+
};
|
|
1348
|
+
}
|
|
1349
|
+
function transformApiPromise(apiPromise, onResolve, onReject) {
|
|
1350
|
+
const resolvedPromise = Promise.resolve(apiPromise).then(onResolve, onReject);
|
|
988
1351
|
if (isObjectRecord(apiPromise)) {
|
|
989
1352
|
const proto = Object.getPrototypeOf(apiPromise);
|
|
990
1353
|
if (proto) {
|
|
@@ -1119,12 +1482,12 @@ function applyResponsesVendorHints(params, hints) {
|
|
|
1119
1482
|
}
|
|
1120
1483
|
return next;
|
|
1121
1484
|
}
|
|
1122
|
-
async function extractUsageFromStream(stream, hints) {
|
|
1485
|
+
async function extractUsageFromStream(stream, hints, provider = "openai") {
|
|
1123
1486
|
const finalPayload = await resolveStreamFinalPayload(stream);
|
|
1124
1487
|
if (!finalPayload) {
|
|
1125
1488
|
return void 0;
|
|
1126
1489
|
}
|
|
1127
|
-
return inferUsageFromResponse(finalPayload, hints);
|
|
1490
|
+
return inferUsageFromResponse(finalPayload, hints, provider);
|
|
1128
1491
|
}
|
|
1129
1492
|
async function resolveStreamFinalPayload(stream) {
|
|
1130
1493
|
if (!stream || typeof stream !== "object") {
|
|
@@ -1197,13 +1560,13 @@ function setHeaderIfPossible(res, key, value) {
|
|
|
1197
1560
|
res.setHeader(key, value);
|
|
1198
1561
|
}
|
|
1199
1562
|
}
|
|
1200
|
-
function tryInferUsage(response, hints, extractor, ctx) {
|
|
1201
|
-
const inferred = inferUsageFromResponse(response, hints);
|
|
1563
|
+
function tryInferUsage(response, hints, extractor, ctx, provider = "openai") {
|
|
1564
|
+
const inferred = inferUsageFromResponse(response, hints, provider);
|
|
1202
1565
|
if (inferred) {
|
|
1203
1566
|
ctx.setUsage(inferred);
|
|
1204
1567
|
}
|
|
1205
1568
|
}
|
|
1206
|
-
function inferUsageFromResponse(response, hints) {
|
|
1569
|
+
function inferUsageFromResponse(response, hints, provider = "openai") {
|
|
1207
1570
|
if (!response || typeof response !== "object") {
|
|
1208
1571
|
return void 0;
|
|
1209
1572
|
}
|
|
@@ -1211,12 +1574,41 @@ function inferUsageFromResponse(response, hints) {
|
|
|
1211
1574
|
if (!candidate.usage) {
|
|
1212
1575
|
return void 0;
|
|
1213
1576
|
}
|
|
1214
|
-
const cachedInputTokens = candidate.usage.prompt_tokens_details?.cached_tokens ?? candidate.usage.cached_tokens;
|
|
1577
|
+
const cachedInputTokens = candidate.usage.prompt_tokens_details?.cached_tokens ?? candidate.usage.input_tokens_details?.cached_tokens ?? candidate.usage.cached_tokens;
|
|
1578
|
+
const responseEffort = normalizeExecutionReasoningEffort(
|
|
1579
|
+
candidate.reasoning?.effort
|
|
1580
|
+
);
|
|
1215
1581
|
return {
|
|
1582
|
+
providerUsed: provider,
|
|
1216
1583
|
modelUsed: candidate.model ?? hints?.preferredModel,
|
|
1217
|
-
inputTokens: candidate.usage.prompt_tokens,
|
|
1218
|
-
responseTokens: candidate.usage.completion_tokens,
|
|
1219
|
-
cachedInputTokens
|
|
1584
|
+
inputTokens: candidate.usage.prompt_tokens ?? candidate.usage.input_tokens,
|
|
1585
|
+
responseTokens: candidate.usage.completion_tokens ?? candidate.usage.output_tokens,
|
|
1586
|
+
cachedInputTokens,
|
|
1587
|
+
reasoningTokens: candidate.usage.completion_tokens_details?.reasoning_tokens ?? candidate.usage.output_tokens_details?.reasoning_tokens,
|
|
1588
|
+
...responseEffort ? {
|
|
1589
|
+
reasoningEffort: responseEffort,
|
|
1590
|
+
reasoningEffortSource: "provider_response"
|
|
1591
|
+
} : {},
|
|
1592
|
+
...typeof candidate.reasoning?.type === "string" ? { reasoningMode: candidate.reasoning.type } : typeof candidate.reasoning?.mode === "string" ? { reasoningMode: candidate.reasoning.mode } : {}
|
|
1593
|
+
};
|
|
1594
|
+
}
|
|
1595
|
+
function normalizeExecutionReasoningEffort(value) {
|
|
1596
|
+
return value === "none" || value === "minimal" || value === "low" || value === "medium" || value === "high" || value === "xhigh" || value === "max" ? value : void 0;
|
|
1597
|
+
}
|
|
1598
|
+
function openAIRequestExecutionMetadata(params, provider) {
|
|
1599
|
+
const record = params && typeof params === "object" ? params : {};
|
|
1600
|
+
const reasoning = record.reasoning && typeof record.reasoning === "object" ? record.reasoning : void 0;
|
|
1601
|
+
const effort = normalizeExecutionReasoningEffort(
|
|
1602
|
+
record.reasoning_effort ?? reasoning?.effort ?? record.thinking_level
|
|
1603
|
+
);
|
|
1604
|
+
const mode = typeof reasoning?.type === "string" ? reasoning.type : typeof reasoning?.mode === "string" ? reasoning.mode : void 0;
|
|
1605
|
+
const rawBudget = reasoning?.budget_tokens ?? record.thinking_budget ?? record.thinking_budget_tokens;
|
|
1606
|
+
const budget = typeof rawBudget === "number" && Number.isInteger(rawBudget) && rawBudget >= 0 ? rawBudget : void 0;
|
|
1607
|
+
return {
|
|
1608
|
+
providerUsed: provider,
|
|
1609
|
+
...effort ? { reasoningEffort: effort, reasoningEffortSource: "provider_request" } : {},
|
|
1610
|
+
...mode ? { reasoningMode: mode } : {},
|
|
1611
|
+
...budget !== void 0 ? { reasoningBudgetTokens: budget } : {}
|
|
1220
1612
|
};
|
|
1221
1613
|
}
|
|
1222
1614
|
function wrapStreamForUsageTap(source, finalize, ctx) {
|