@usagetap/sdk 1.4.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +156 -18
- package/dist/adapters/anthropic.cjs +195 -235
- package/dist/adapters/anthropic.cjs.map +1 -1
- package/dist/adapters/anthropic.d.cts +1 -1
- package/dist/adapters/anthropic.d.ts +1 -1
- package/dist/adapters/anthropic.mjs +195 -235
- package/dist/adapters/anthropic.mjs.map +1 -1
- package/dist/adapters/openai.cjs +256 -274
- package/dist/adapters/openai.cjs.map +1 -1
- package/dist/adapters/openai.d.cts +1 -1
- package/dist/adapters/openai.d.ts +1 -1
- package/dist/adapters/openai.mjs +256 -274
- package/dist/adapters/openai.mjs.map +1 -1
- package/dist/adapters/openrouter.cjs +256 -274
- package/dist/adapters/openrouter.cjs.map +1 -1
- package/dist/adapters/openrouter.d.cts +1 -1
- package/dist/adapters/openrouter.d.ts +1 -1
- package/dist/adapters/openrouter.mjs +256 -274
- package/dist/adapters/openrouter.mjs.map +1 -1
- package/dist/anthropic/index.cjs +195 -235
- package/dist/anthropic/index.cjs.map +1 -1
- package/dist/anthropic/index.d.cts +1 -1
- package/dist/anthropic/index.d.ts +1 -1
- package/dist/anthropic/index.mjs +195 -235
- package/dist/anthropic/index.mjs.map +1 -1
- package/dist/{client-CExQ8e1T.d.cts → client-C0UiaqVB.d.cts} +141 -61
- package/dist/{client-CExQ8e1T.d.ts → client-C0UiaqVB.d.ts} +141 -61
- package/dist/express/index.cjs +189 -246
- package/dist/express/index.cjs.map +1 -1
- package/dist/express/index.d.cts +1 -1
- package/dist/express/index.d.ts +1 -1
- package/dist/express/index.mjs +189 -246
- package/dist/express/index.mjs.map +1 -1
- package/dist/index.cjs +391 -175
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +13 -4
- package/dist/index.d.ts +13 -4
- package/dist/index.mjs +391 -175
- package/dist/index.mjs.map +1 -1
- package/dist/openai/index.cjs +256 -274
- package/dist/openai/index.cjs.map +1 -1
- package/dist/openai/index.d.cts +1 -1
- package/dist/openai/index.d.ts +1 -1
- package/dist/openai/index.mjs +256 -274
- package/dist/openai/index.mjs.map +1 -1
- package/dist/openrouter/index.cjs +256 -274
- package/dist/openrouter/index.cjs.map +1 -1
- package/dist/openrouter/index.d.cts +1 -1
- package/dist/openrouter/index.d.ts +1 -1
- package/dist/openrouter/index.mjs +256 -274
- package/dist/openrouter/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/express/index.cjs
CHANGED
|
@@ -336,13 +336,13 @@ function createResponsesProxy(resource, usageTap, defaultContext, applyVendorHin
|
|
|
336
336
|
withUsage: withUsage2,
|
|
337
337
|
promptCompression
|
|
338
338
|
} = splitUsageOptions(options);
|
|
339
|
-
const beginRequest =
|
|
339
|
+
const beginRequest = responsesBeginRequest(
|
|
340
340
|
resolveBeginRequest(defaultContext, usageContext),
|
|
341
|
-
|
|
342
|
-
String(params.model)
|
|
341
|
+
params
|
|
343
342
|
);
|
|
344
343
|
const wantsStream = isStreamingRequest(params);
|
|
345
344
|
return usageTap.withUsage(beginRequest, async (ctx) => {
|
|
345
|
+
const settle = wantsStream ? deferUsageFinalization(ctx) : void 0;
|
|
346
346
|
const sampleStartedAt = Date.now();
|
|
347
347
|
const sampleDecision = wantsStream ? Promise.resolve(false) : startMeteredOpenAISampleDecision({
|
|
348
348
|
usageTap,
|
|
@@ -367,12 +367,31 @@ function createResponsesProxy(resource, usageTap, defaultContext, applyVendorHin
|
|
|
367
367
|
const apiPromise2 = originalCreate(finalParams, request);
|
|
368
368
|
const wrappedPromise2 = transformApiPromise(apiPromise2, (rawStream) => {
|
|
369
369
|
ensureAsyncIterable(rawStream, "responses.create");
|
|
370
|
-
const wrappedStream = wrapStreamForUsageTap(rawStream, async () => {
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
370
|
+
const wrappedStream = wrapStreamForUsageTap(rawStream, async (termination) => {
|
|
371
|
+
try {
|
|
372
|
+
if (termination === "complete") {
|
|
373
|
+
const usage = await extractUsageFromStream(rawStream, ctx.begin.data.vendorHints, provider);
|
|
374
|
+
if (usage) {
|
|
375
|
+
ctx.setUsage(usage);
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
} catch (error) {
|
|
379
|
+
ctx.setError({
|
|
380
|
+
code: "USAGE_FINALIZE_ERROR",
|
|
381
|
+
message: error instanceof Error ? error.message : String(error)
|
|
382
|
+
});
|
|
383
|
+
throw error;
|
|
384
|
+
} finally {
|
|
385
|
+
await settle?.();
|
|
374
386
|
}
|
|
375
|
-
}, ctx)
|
|
387
|
+
}, ctx, (chunk) => {
|
|
388
|
+
tryInferUsageFromStreamChunk(
|
|
389
|
+
chunk,
|
|
390
|
+
ctx.begin.data.vendorHints,
|
|
391
|
+
ctx,
|
|
392
|
+
provider
|
|
393
|
+
);
|
|
394
|
+
});
|
|
376
395
|
return wrappedStream;
|
|
377
396
|
});
|
|
378
397
|
return wrappedPromise2;
|
|
@@ -428,13 +447,10 @@ function createChatCompletionsProxy(resource, usageTap, defaultContext, applyVen
|
|
|
428
447
|
withUsage: withUsage2,
|
|
429
448
|
promptCompression
|
|
430
449
|
} = splitUsageOptions(options);
|
|
431
|
-
const beginRequest =
|
|
432
|
-
resolveBeginRequest(defaultContext, usageContext),
|
|
433
|
-
"openai",
|
|
434
|
-
String(params.model)
|
|
435
|
-
);
|
|
450
|
+
const beginRequest = resolveBeginRequest(defaultContext, usageContext);
|
|
436
451
|
const wantsStream = isStreamingRequest(params);
|
|
437
452
|
return usageTap.withUsage(beginRequest, async (ctx) => {
|
|
453
|
+
const settle = wantsStream ? deferUsageFinalization(ctx) : void 0;
|
|
438
454
|
const sampleStartedAt = Date.now();
|
|
439
455
|
const sampleDecision = wantsStream ? Promise.resolve(false) : startMeteredOpenAISampleDecision({
|
|
440
456
|
usageTap,
|
|
@@ -443,7 +459,7 @@ function createChatCompletionsProxy(resource, usageTap, defaultContext, applyVen
|
|
|
443
459
|
input: params
|
|
444
460
|
});
|
|
445
461
|
const hintedParams = applyVendorHints ? applyChatVendorHints(params, ctx.begin.data.vendorHints) : params;
|
|
446
|
-
const
|
|
462
|
+
const compressedParams = await compressChatParamsForCall({
|
|
447
463
|
params: hintedParams,
|
|
448
464
|
usageTap,
|
|
449
465
|
ctx,
|
|
@@ -453,18 +469,38 @@ function createChatCompletionsProxy(resource, usageTap, defaultContext, applyVen
|
|
|
453
469
|
withUsage: withUsage2,
|
|
454
470
|
operation: "chat.completions.create"
|
|
455
471
|
});
|
|
472
|
+
const finalParams = wantsStream ? ensureOpenAIStreamUsage(compressedParams) : compressedParams;
|
|
456
473
|
ctx.setUsage(openAIRequestExecutionMetadata(finalParams, provider));
|
|
457
474
|
const request = attachCorrelationHeader(requestOptions, ctx.begin.correlationId);
|
|
458
475
|
if (wantsStream) {
|
|
459
476
|
const apiPromise2 = originalCreate(finalParams, request);
|
|
460
477
|
const wrappedPromise2 = transformApiPromise(apiPromise2, (rawStream) => {
|
|
461
478
|
ensureAsyncIterable(rawStream, "chat.completions.create");
|
|
462
|
-
const wrappedStream2 = wrapStreamForUsageTap(rawStream, async () => {
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
479
|
+
const wrappedStream2 = wrapStreamForUsageTap(rawStream, async (termination) => {
|
|
480
|
+
try {
|
|
481
|
+
if (termination === "complete") {
|
|
482
|
+
const usage = await extractUsageFromStream(rawStream, ctx.begin.data.vendorHints, provider);
|
|
483
|
+
if (usage) {
|
|
484
|
+
ctx.setUsage(usage);
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
} catch (error) {
|
|
488
|
+
ctx.setError({
|
|
489
|
+
code: "USAGE_FINALIZE_ERROR",
|
|
490
|
+
message: error instanceof Error ? error.message : String(error)
|
|
491
|
+
});
|
|
492
|
+
throw error;
|
|
493
|
+
} finally {
|
|
494
|
+
await settle?.();
|
|
466
495
|
}
|
|
467
|
-
}, ctx)
|
|
496
|
+
}, ctx, (chunk) => {
|
|
497
|
+
tryInferUsageFromStreamChunk(
|
|
498
|
+
chunk,
|
|
499
|
+
ctx.begin.data.vendorHints,
|
|
500
|
+
ctx,
|
|
501
|
+
provider
|
|
502
|
+
);
|
|
503
|
+
});
|
|
468
504
|
return wrappedStream2;
|
|
469
505
|
});
|
|
470
506
|
return wrappedPromise2;
|
|
@@ -506,14 +542,11 @@ function createChatCompletionsProxy(resource, usageTap, defaultContext, applyVen
|
|
|
506
542
|
withUsage: withUsage2,
|
|
507
543
|
promptCompression
|
|
508
544
|
} = splitUsageOptions(options);
|
|
509
|
-
const beginRequest =
|
|
510
|
-
resolveBeginRequest(defaultContext, usageContext),
|
|
511
|
-
"openai",
|
|
512
|
-
String(params.model)
|
|
513
|
-
);
|
|
545
|
+
const beginRequest = resolveBeginRequest(defaultContext, usageContext);
|
|
514
546
|
return usageTap.withUsage(beginRequest, async (ctx) => {
|
|
547
|
+
const settle = deferUsageFinalization(ctx);
|
|
515
548
|
const hintedParams = applyVendorHints ? applyChatVendorHints(params, ctx.begin.data.vendorHints) : params;
|
|
516
|
-
const
|
|
549
|
+
const compressedParams = await compressChatParamsForCall({
|
|
517
550
|
params: hintedParams,
|
|
518
551
|
usageTap,
|
|
519
552
|
ctx,
|
|
@@ -523,21 +556,41 @@ function createChatCompletionsProxy(resource, usageTap, defaultContext, applyVen
|
|
|
523
556
|
withUsage: withUsage2,
|
|
524
557
|
operation: "chat.completions.stream"
|
|
525
558
|
});
|
|
559
|
+
const finalParams = ensureOpenAIStreamUsage(compressedParams);
|
|
526
560
|
ctx.setUsage(openAIRequestExecutionMetadata(finalParams, provider));
|
|
527
561
|
const request = attachCorrelationHeader(requestOptions, ctx.begin.correlationId);
|
|
528
562
|
const apiPromise = originalStream(finalParams, request);
|
|
529
563
|
const wrappedPromise = transformApiPromise(apiPromise, (rawStream) => {
|
|
530
564
|
ensureAsyncIterable(rawStream, "chat.completions.stream");
|
|
531
|
-
const wrappedStreamInner = wrapStreamForUsageTap(rawStream, async () => {
|
|
532
|
-
|
|
533
|
-
|
|
565
|
+
const wrappedStreamInner = wrapStreamForUsageTap(rawStream, async (termination) => {
|
|
566
|
+
try {
|
|
567
|
+
if (termination === "complete") {
|
|
568
|
+
const usage = await extractUsageFromStream(
|
|
569
|
+
rawStream,
|
|
570
|
+
ctx.begin.data.vendorHints,
|
|
571
|
+
provider
|
|
572
|
+
);
|
|
573
|
+
if (usage) {
|
|
574
|
+
ctx.setUsage(usage);
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
} catch (error) {
|
|
578
|
+
ctx.setError({
|
|
579
|
+
code: "USAGE_FINALIZE_ERROR",
|
|
580
|
+
message: error instanceof Error ? error.message : String(error)
|
|
581
|
+
});
|
|
582
|
+
throw error;
|
|
583
|
+
} finally {
|
|
584
|
+
await settle();
|
|
585
|
+
}
|
|
586
|
+
}, ctx, (chunk) => {
|
|
587
|
+
tryInferUsageFromStreamChunk(
|
|
588
|
+
chunk,
|
|
534
589
|
ctx.begin.data.vendorHints,
|
|
590
|
+
ctx,
|
|
535
591
|
provider
|
|
536
592
|
);
|
|
537
|
-
|
|
538
|
-
ctx.setUsage(usage);
|
|
539
|
-
}
|
|
540
|
-
}, ctx);
|
|
593
|
+
});
|
|
541
594
|
return wrappedStreamInner;
|
|
542
595
|
});
|
|
543
596
|
return wrappedPromise;
|
|
@@ -557,70 +610,19 @@ function createChatCompletionsProxy(resource, usageTap, defaultContext, applyVen
|
|
|
557
610
|
return new Proxy(resource, handler);
|
|
558
611
|
}
|
|
559
612
|
async function compressChatParamsForCall(args) {
|
|
560
|
-
|
|
613
|
+
const compression = resolveEffectivePromptCompressionOptions(
|
|
561
614
|
args.defaultPromptCompression,
|
|
562
615
|
args.callPromptCompression
|
|
563
616
|
);
|
|
564
|
-
const runtimePolicy = args.ctx.begin.data.runtimeCompressionPolicy;
|
|
565
|
-
const policyDriven = !compression && Boolean(runtimePolicy);
|
|
566
|
-
if (policyDriven && runtimePolicy) {
|
|
567
|
-
if (!runtimePolicy.selected) {
|
|
568
|
-
args.ctx.setUsage({
|
|
569
|
-
runtimeCompression: bypassedRuntimeMeasurement(runtimePolicy)
|
|
570
|
-
});
|
|
571
|
-
return args.params;
|
|
572
|
-
}
|
|
573
|
-
compression = openAIPolicyCompressionOptions(runtimePolicy);
|
|
574
|
-
if (runtimePolicy.mode === "SHADOW") {
|
|
575
|
-
const startedAt2 = Date.now();
|
|
576
|
-
void compressChatParams(
|
|
577
|
-
args.params,
|
|
578
|
-
args.usageTap,
|
|
579
|
-
compression,
|
|
580
|
-
args.withUsage?.signal
|
|
581
|
-
).then((outcome2) => {
|
|
582
|
-
args.ctx.setUsage({
|
|
583
|
-
runtimeCompression: openAIRuntimeMeasurement({
|
|
584
|
-
policy: runtimePolicy,
|
|
585
|
-
telemetry: buildPromptCompressionTelemetry(outcome2.segments),
|
|
586
|
-
latencyMs: Date.now() - startedAt2,
|
|
587
|
-
shadow: true,
|
|
588
|
-
originalPayload: args.params,
|
|
589
|
-
resultPayload: outcome2.params
|
|
590
|
-
})
|
|
591
|
-
});
|
|
592
|
-
}).catch(() => {
|
|
593
|
-
args.ctx.setUsage({
|
|
594
|
-
runtimeCompression: failedRuntimeMeasurement(
|
|
595
|
-
runtimePolicy,
|
|
596
|
-
Date.now() - startedAt2
|
|
597
|
-
)
|
|
598
|
-
});
|
|
599
|
-
});
|
|
600
|
-
return args.params;
|
|
601
|
-
}
|
|
602
|
-
}
|
|
603
617
|
if (!compression) {
|
|
604
618
|
return args.params;
|
|
605
619
|
}
|
|
606
|
-
const startedAt = Date.now();
|
|
607
620
|
const outcome = await compressChatParams(
|
|
608
621
|
args.params,
|
|
609
622
|
args.usageTap,
|
|
610
623
|
compression,
|
|
611
624
|
args.withUsage?.signal
|
|
612
625
|
);
|
|
613
|
-
if (policyDriven && runtimePolicy) {
|
|
614
|
-
const runtimeMeasurement = openAIRuntimeMeasurement({
|
|
615
|
-
policy: runtimePolicy,
|
|
616
|
-
telemetry: buildPromptCompressionTelemetry(outcome.segments),
|
|
617
|
-
latencyMs: Date.now() - startedAt,
|
|
618
|
-
originalPayload: args.params,
|
|
619
|
-
resultPayload: outcome.params
|
|
620
|
-
});
|
|
621
|
-
args.ctx.setUsage({ runtimeCompression: runtimeMeasurement });
|
|
622
|
-
if (runtimeMeasurement.decision !== "compressed") return args.params;
|
|
623
|
-
}
|
|
624
626
|
await recordCompressionOutcome({
|
|
625
627
|
outcome,
|
|
626
628
|
compression,
|
|
@@ -633,70 +635,19 @@ async function compressChatParamsForCall(args) {
|
|
|
633
635
|
return outcome.params;
|
|
634
636
|
}
|
|
635
637
|
async function compressResponsesParamsForCall(args) {
|
|
636
|
-
|
|
638
|
+
const compression = resolveEffectivePromptCompressionOptions(
|
|
637
639
|
args.defaultPromptCompression,
|
|
638
640
|
args.callPromptCompression
|
|
639
641
|
);
|
|
640
|
-
const runtimePolicy = args.ctx.begin.data.runtimeCompressionPolicy;
|
|
641
|
-
const policyDriven = !compression && Boolean(runtimePolicy);
|
|
642
|
-
if (policyDriven && runtimePolicy) {
|
|
643
|
-
if (!runtimePolicy.selected) {
|
|
644
|
-
args.ctx.setUsage({
|
|
645
|
-
runtimeCompression: bypassedRuntimeMeasurement(runtimePolicy)
|
|
646
|
-
});
|
|
647
|
-
return args.params;
|
|
648
|
-
}
|
|
649
|
-
compression = openAIPolicyCompressionOptions(runtimePolicy);
|
|
650
|
-
if (runtimePolicy.mode === "SHADOW") {
|
|
651
|
-
const startedAt2 = Date.now();
|
|
652
|
-
void compressResponsesParams(
|
|
653
|
-
args.params,
|
|
654
|
-
args.usageTap,
|
|
655
|
-
compression,
|
|
656
|
-
args.withUsage?.signal
|
|
657
|
-
).then((outcome2) => {
|
|
658
|
-
args.ctx.setUsage({
|
|
659
|
-
runtimeCompression: openAIRuntimeMeasurement({
|
|
660
|
-
policy: runtimePolicy,
|
|
661
|
-
telemetry: buildPromptCompressionTelemetry(outcome2.segments),
|
|
662
|
-
latencyMs: Date.now() - startedAt2,
|
|
663
|
-
shadow: true,
|
|
664
|
-
originalPayload: args.params,
|
|
665
|
-
resultPayload: outcome2.params
|
|
666
|
-
})
|
|
667
|
-
});
|
|
668
|
-
}).catch(() => {
|
|
669
|
-
args.ctx.setUsage({
|
|
670
|
-
runtimeCompression: failedRuntimeMeasurement(
|
|
671
|
-
runtimePolicy,
|
|
672
|
-
Date.now() - startedAt2
|
|
673
|
-
)
|
|
674
|
-
});
|
|
675
|
-
});
|
|
676
|
-
return args.params;
|
|
677
|
-
}
|
|
678
|
-
}
|
|
679
642
|
if (!compression) {
|
|
680
643
|
return args.params;
|
|
681
644
|
}
|
|
682
|
-
const startedAt = Date.now();
|
|
683
645
|
const outcome = await compressResponsesParams(
|
|
684
646
|
args.params,
|
|
685
647
|
args.usageTap,
|
|
686
648
|
compression,
|
|
687
649
|
args.withUsage?.signal
|
|
688
650
|
);
|
|
689
|
-
if (policyDriven && runtimePolicy) {
|
|
690
|
-
const runtimeMeasurement = openAIRuntimeMeasurement({
|
|
691
|
-
policy: runtimePolicy,
|
|
692
|
-
telemetry: buildPromptCompressionTelemetry(outcome.segments),
|
|
693
|
-
latencyMs: Date.now() - startedAt,
|
|
694
|
-
originalPayload: args.params,
|
|
695
|
-
resultPayload: outcome.params
|
|
696
|
-
});
|
|
697
|
-
args.ctx.setUsage({ runtimeCompression: runtimeMeasurement });
|
|
698
|
-
if (runtimeMeasurement.decision !== "compressed") return args.params;
|
|
699
|
-
}
|
|
700
651
|
await recordCompressionOutcome({
|
|
701
652
|
outcome,
|
|
702
653
|
compression,
|
|
@@ -708,79 +659,6 @@ async function compressResponsesParamsForCall(args) {
|
|
|
708
659
|
});
|
|
709
660
|
return outcome.params;
|
|
710
661
|
}
|
|
711
|
-
function openAIPolicyCompressionOptions(policy) {
|
|
712
|
-
const roles = Object.fromEntries(
|
|
713
|
-
["system", "user", "tool", "assistant"].filter((role) => policy.snapshot.scope.messageRoles.includes(role)).map((role) => [role, true])
|
|
714
|
-
);
|
|
715
|
-
return {
|
|
716
|
-
provider: "usagetap",
|
|
717
|
-
roles,
|
|
718
|
-
minContextTokens: policy.snapshot.scope.minimumTokens,
|
|
719
|
-
failOpen: true
|
|
720
|
-
};
|
|
721
|
-
}
|
|
722
|
-
function bypassedRuntimeMeasurement(policy) {
|
|
723
|
-
return {
|
|
724
|
-
policyId: policy.policyId,
|
|
725
|
-
policyVersionId: policy.policyVersionId,
|
|
726
|
-
policyVersion: policy.version,
|
|
727
|
-
rolloutMode: policy.mode,
|
|
728
|
-
decision: "bypassed",
|
|
729
|
-
reasonCode: "not_in_rollout_sample",
|
|
730
|
-
methodsAttempted: [],
|
|
731
|
-
methodsApplied: [],
|
|
732
|
-
originalTokens: 0,
|
|
733
|
-
resultTokens: 0,
|
|
734
|
-
savedTokens: 0,
|
|
735
|
-
reductionPercentage: 0,
|
|
736
|
-
compressionLatencyMs: 0,
|
|
737
|
-
compressionComputeCostUsd: 0,
|
|
738
|
-
financialsEstimated: true
|
|
739
|
-
};
|
|
740
|
-
}
|
|
741
|
-
function failedRuntimeMeasurement(policy, latencyMs) {
|
|
742
|
-
return {
|
|
743
|
-
...bypassedRuntimeMeasurement(policy),
|
|
744
|
-
decision: "fallback",
|
|
745
|
-
reasonCode: "transformation_failed",
|
|
746
|
-
methodsAttempted: ["deterministic"],
|
|
747
|
-
compressionLatencyMs: latencyMs
|
|
748
|
-
};
|
|
749
|
-
}
|
|
750
|
-
function openAIRuntimeMeasurement(input) {
|
|
751
|
-
const telemetry = input.telemetry;
|
|
752
|
-
const originalTokens = telemetry?.originalTokens ?? estimatePromptTokens(JSON.stringify(input.originalPayload));
|
|
753
|
-
const resultTokens = telemetry?.compressedTokens ?? originalTokens;
|
|
754
|
-
const savedTokens = Math.max(0, telemetry?.savedTokens ?? 0);
|
|
755
|
-
const reductionPercentage = originalTokens > 0 ? savedTokens / originalTokens * 100 : 0;
|
|
756
|
-
const belowMinimumSize = originalTokens < input.policy.snapshot.scope.minimumTokens;
|
|
757
|
-
const latencyExceeded = input.latencyMs > input.policy.snapshot.gates.maximumLatencyMs;
|
|
758
|
-
const belowSavings = savedTokens < input.policy.snapshot.gates.minimumTokensRemoved || reductionPercentage < input.policy.snapshot.gates.minimumReductionPercentage;
|
|
759
|
-
const accepted = !belowMinimumSize && !latencyExceeded && !belowSavings;
|
|
760
|
-
const deepText = input.policy.snapshot.methods.deepText.enabled && !input.policy.snapshot.rollout.deterministicOnly && (input.policy.mode !== "CANARY" || input.policy.snapshot.rollout.deepTextCanaryAllowed);
|
|
761
|
-
const reasonCode = belowMinimumSize ? "below_minimum_size" : latencyExceeded ? "latency_budget_exceeded" : belowSavings ? "below_minimum_savings" : void 0;
|
|
762
|
-
const measurement = {
|
|
763
|
-
policyId: input.policy.policyId,
|
|
764
|
-
policyVersionId: input.policy.policyVersionId,
|
|
765
|
-
policyVersion: input.policy.version,
|
|
766
|
-
rolloutMode: input.policy.mode,
|
|
767
|
-
decision: accepted ? input.shadow ? "bypassed" : "compressed" : latencyExceeded ? "fallback" : "skipped",
|
|
768
|
-
...reasonCode ? { reasonCode } : {},
|
|
769
|
-
methodsAttempted: [
|
|
770
|
-
"deterministic",
|
|
771
|
-
...deepText ? ["deep_text"] : []
|
|
772
|
-
],
|
|
773
|
-
methodsApplied: accepted ? ["deterministic"] : [],
|
|
774
|
-
originalTokens,
|
|
775
|
-
resultTokens: accepted ? resultTokens : originalTokens,
|
|
776
|
-
savedTokens: accepted ? savedTokens : 0,
|
|
777
|
-
reductionPercentage: accepted ? reductionPercentage : 0,
|
|
778
|
-
compressionLatencyMs: input.latencyMs,
|
|
779
|
-
compressionComputeCostUsd: 0,
|
|
780
|
-
financialsEstimated: true
|
|
781
|
-
};
|
|
782
|
-
return measurement;
|
|
783
|
-
}
|
|
784
662
|
async function recordCompressionOutcome(args) {
|
|
785
663
|
const telemetry = buildPromptCompressionTelemetry(args.outcome.segments);
|
|
786
664
|
if (!telemetry) {
|
|
@@ -1303,10 +1181,6 @@ function resolveBeginRequest(defaults, override) {
|
|
|
1303
1181
|
}
|
|
1304
1182
|
const tags = mergeTags(base.tags, current.tags);
|
|
1305
1183
|
const begin = { customerId };
|
|
1306
|
-
const runtimeCompressionContext = current.runtimeCompressionContext ?? base.runtimeCompressionContext;
|
|
1307
|
-
if (runtimeCompressionContext) {
|
|
1308
|
-
begin.runtimeCompressionContext = runtimeCompressionContext;
|
|
1309
|
-
}
|
|
1310
1184
|
const requested = current.requested ?? base.requested;
|
|
1311
1185
|
if (requested) begin.requested = requested;
|
|
1312
1186
|
const feature = current.feature ?? base.feature;
|
|
@@ -1327,8 +1201,6 @@ function resolveBeginRequest(defaults, override) {
|
|
|
1327
1201
|
if (customerUserEmail) begin.customerUserEmail = customerUserEmail;
|
|
1328
1202
|
const stripeCustomerId = current.stripeCustomerId ?? base.stripeCustomerId;
|
|
1329
1203
|
if (stripeCustomerId) begin.stripeCustomerId = stripeCustomerId;
|
|
1330
|
-
const holdUsd = current.holdUsd ?? base.holdUsd;
|
|
1331
|
-
if (typeof holdUsd === "number") begin.holdUsd = holdUsd;
|
|
1332
1204
|
const batch = current.batch ?? base.batch;
|
|
1333
1205
|
if (typeof batch === "boolean") begin.batch = batch;
|
|
1334
1206
|
const pricingMode = current.pricingMode ?? base.pricingMode;
|
|
@@ -1338,16 +1210,25 @@ function resolveBeginRequest(defaults, override) {
|
|
|
1338
1210
|
}
|
|
1339
1211
|
return begin;
|
|
1340
1212
|
}
|
|
1341
|
-
function
|
|
1213
|
+
function responsesBeginRequest(begin, params) {
|
|
1214
|
+
if (!responsesRequestUsesWebSearch(params)) return begin;
|
|
1342
1215
|
return {
|
|
1343
1216
|
...begin,
|
|
1344
|
-
|
|
1345
|
-
...begin.
|
|
1346
|
-
|
|
1347
|
-
model
|
|
1217
|
+
requested: {
|
|
1218
|
+
...begin.requested ?? {},
|
|
1219
|
+
search: true
|
|
1348
1220
|
}
|
|
1349
1221
|
};
|
|
1350
1222
|
}
|
|
1223
|
+
function responsesRequestUsesWebSearch(params) {
|
|
1224
|
+
if (!params || typeof params !== "object") return false;
|
|
1225
|
+
const tools = params.tools;
|
|
1226
|
+
return Array.isArray(tools) && tools.some((tool) => {
|
|
1227
|
+
if (!tool || typeof tool !== "object") return false;
|
|
1228
|
+
const type = tool.type;
|
|
1229
|
+
return type === "web_search" || type === "web_search_preview";
|
|
1230
|
+
});
|
|
1231
|
+
}
|
|
1351
1232
|
function transformApiPromise(apiPromise, onResolve, onReject) {
|
|
1352
1233
|
const resolvedPromise = Promise.resolve(apiPromise).then(onResolve, onReject);
|
|
1353
1234
|
if (isObjectRecord(apiPromise)) {
|
|
@@ -1577,6 +1458,11 @@ function inferUsageFromResponse(response, hints, provider = "openai") {
|
|
|
1577
1458
|
return void 0;
|
|
1578
1459
|
}
|
|
1579
1460
|
const cachedInputTokens = candidate.usage.prompt_tokens_details?.cached_tokens ?? candidate.usage.input_tokens_details?.cached_tokens ?? candidate.usage.cached_tokens;
|
|
1461
|
+
const cacheWriteInputTokens = candidate.usage.prompt_tokens_details?.cache_write_tokens ?? candidate.usage.prompt_tokens_details?.cache_creation_tokens ?? candidate.usage.input_tokens_details?.cache_write_tokens ?? candidate.usage.input_tokens_details?.cache_creation_tokens ?? candidate.usage.cache_creation_input_tokens ?? candidate.usage.cache_write_input_tokens ?? candidate.usage.cache_write_tokens;
|
|
1462
|
+
const cacheWrite5mInputTokens = candidate.usage.cache_write_5m_input_tokens ?? candidate.usage.cache_creation?.ephemeral_5m_input_tokens;
|
|
1463
|
+
const cacheWrite1hInputTokens = candidate.usage.cache_write_1h_input_tokens ?? candidate.usage.cache_creation?.ephemeral_1h_input_tokens;
|
|
1464
|
+
const outputSearches = Array.isArray(candidate.output) ? candidate.output.filter((item) => item?.type === "web_search_call").length : 0;
|
|
1465
|
+
const searches = candidate.usage.searches ?? candidate.usage.web_search_queries ?? candidate.usage.server_tool_use?.web_search_requests ?? outputSearches;
|
|
1580
1466
|
const responseEffort = normalizeExecutionReasoningEffort(
|
|
1581
1467
|
candidate.reasoning?.effort
|
|
1582
1468
|
);
|
|
@@ -1586,12 +1472,21 @@ function inferUsageFromResponse(response, hints, provider = "openai") {
|
|
|
1586
1472
|
inputTokens: candidate.usage.prompt_tokens ?? candidate.usage.input_tokens,
|
|
1587
1473
|
responseTokens: candidate.usage.completion_tokens ?? candidate.usage.output_tokens,
|
|
1588
1474
|
cachedInputTokens,
|
|
1475
|
+
cacheWriteInputTokens,
|
|
1476
|
+
cacheWrite5mInputTokens,
|
|
1477
|
+
cacheWrite1hInputTokens,
|
|
1478
|
+
audioInputTokens: candidate.usage.prompt_tokens_details?.audio_tokens ?? candidate.usage.input_tokens_details?.audio_tokens,
|
|
1479
|
+
cachedAudioInputTokens: candidate.usage.prompt_tokens_details?.cached_audio_tokens ?? candidate.usage.input_tokens_details?.cached_audio_tokens ?? candidate.usage.prompt_tokens_details?.cached_tokens_details?.audio_tokens ?? candidate.usage.input_tokens_details?.cached_tokens_details?.audio_tokens,
|
|
1480
|
+
imageInputTokens: candidate.usage.prompt_tokens_details?.image_tokens ?? candidate.usage.input_tokens_details?.image_tokens,
|
|
1481
|
+
imageOutputTokens: candidate.usage.completion_tokens_details?.image_tokens ?? candidate.usage.output_tokens_details?.image_tokens,
|
|
1482
|
+
audioOutputTokens: candidate.usage.completion_tokens_details?.audio_tokens ?? candidate.usage.output_tokens_details?.audio_tokens,
|
|
1589
1483
|
reasoningTokens: candidate.usage.completion_tokens_details?.reasoning_tokens ?? candidate.usage.output_tokens_details?.reasoning_tokens,
|
|
1590
1484
|
...responseEffort ? {
|
|
1591
1485
|
reasoningEffort: responseEffort,
|
|
1592
1486
|
reasoningEffortSource: "provider_response"
|
|
1593
1487
|
} : {},
|
|
1594
|
-
...typeof candidate.reasoning?.type === "string" ? { reasoningMode: candidate.reasoning.type } : typeof candidate.reasoning?.mode === "string" ? { reasoningMode: candidate.reasoning.mode } : {}
|
|
1488
|
+
...typeof candidate.reasoning?.type === "string" ? { reasoningMode: candidate.reasoning.type } : typeof candidate.reasoning?.mode === "string" ? { reasoningMode: candidate.reasoning.mode } : {},
|
|
1489
|
+
...typeof searches === "number" && searches > 0 ? { searches } : {}
|
|
1595
1490
|
};
|
|
1596
1491
|
}
|
|
1597
1492
|
function normalizeExecutionReasoningEffort(value) {
|
|
@@ -1608,29 +1503,64 @@ function openAIRequestExecutionMetadata(params, provider) {
|
|
|
1608
1503
|
const budget = typeof rawBudget === "number" && Number.isInteger(rawBudget) && rawBudget >= 0 ? rawBudget : void 0;
|
|
1609
1504
|
return {
|
|
1610
1505
|
providerUsed: provider,
|
|
1506
|
+
...typeof record.model === "string" ? { modelUsed: record.model } : {},
|
|
1611
1507
|
...effort ? { reasoningEffort: effort, reasoningEffortSource: "provider_request" } : {},
|
|
1612
1508
|
...mode ? { reasoningMode: mode } : {},
|
|
1613
1509
|
...budget !== void 0 ? { reasoningBudgetTokens: budget } : {}
|
|
1614
1510
|
};
|
|
1615
1511
|
}
|
|
1616
|
-
function
|
|
1512
|
+
function tryInferUsageFromStreamChunk(chunk, hints, ctx, provider) {
|
|
1513
|
+
const payload = isObjectRecord(chunk) && isObjectRecord(chunk.response) ? chunk.response : chunk;
|
|
1514
|
+
const inferred = inferUsageFromResponse(payload, hints, provider);
|
|
1515
|
+
if (inferred) {
|
|
1516
|
+
ctx.setUsage(inferred);
|
|
1517
|
+
}
|
|
1518
|
+
}
|
|
1519
|
+
function ensureOpenAIStreamUsage(params) {
|
|
1520
|
+
if (!isObjectRecord(params)) {
|
|
1521
|
+
return params;
|
|
1522
|
+
}
|
|
1523
|
+
const streamOptions = isObjectRecord(params.stream_options) ? params.stream_options : {};
|
|
1524
|
+
return {
|
|
1525
|
+
...params,
|
|
1526
|
+
stream_options: {
|
|
1527
|
+
...streamOptions,
|
|
1528
|
+
include_usage: true
|
|
1529
|
+
}
|
|
1530
|
+
};
|
|
1531
|
+
}
|
|
1532
|
+
function deferUsageFinalization(ctx) {
|
|
1533
|
+
return ctx.deferFinalization?.() ?? (() => Promise.resolve());
|
|
1534
|
+
}
|
|
1535
|
+
function wrapStreamForUsageTap(source, finalize, ctx, onChunk) {
|
|
1617
1536
|
const getIterator = source[Symbol.asyncIterator];
|
|
1618
1537
|
if (typeof getIterator !== "function") {
|
|
1619
1538
|
throw new TypeError("Stream is not async iterable");
|
|
1620
1539
|
}
|
|
1621
1540
|
const iterator = getIterator.call(source);
|
|
1622
1541
|
let completed = false;
|
|
1623
|
-
const invokeFinalize = async () => {
|
|
1542
|
+
const invokeFinalize = async (termination, error) => {
|
|
1624
1543
|
if (completed) return;
|
|
1625
1544
|
completed = true;
|
|
1545
|
+
if (termination === "cancel" || termination === "manual") {
|
|
1546
|
+
ctx.setError({
|
|
1547
|
+
code: "STREAM_ABORTED",
|
|
1548
|
+
message: termination === "cancel" ? "Provider stream consumption was cancelled before completion" : "Provider stream was finalized before completion"
|
|
1549
|
+
});
|
|
1550
|
+
} else if (termination === "error") {
|
|
1551
|
+
ctx.setError({
|
|
1552
|
+
code: "VENDOR_ERROR",
|
|
1553
|
+
message: error instanceof Error ? error.message : String(error)
|
|
1554
|
+
});
|
|
1555
|
+
}
|
|
1626
1556
|
try {
|
|
1627
|
-
await finalize();
|
|
1628
|
-
} catch (
|
|
1557
|
+
await finalize(termination);
|
|
1558
|
+
} catch (error2) {
|
|
1629
1559
|
ctx.setError({
|
|
1630
1560
|
code: "USAGE_FINALIZE_ERROR",
|
|
1631
|
-
message:
|
|
1561
|
+
message: error2 instanceof Error ? error2.message : String(error2)
|
|
1632
1562
|
});
|
|
1633
|
-
throw
|
|
1563
|
+
throw error2;
|
|
1634
1564
|
}
|
|
1635
1565
|
};
|
|
1636
1566
|
const prototype = Object.getPrototypeOf(source) ?? Object.prototype;
|
|
@@ -1654,12 +1584,15 @@ function wrapStreamForUsageTap(source, finalize, ctx) {
|
|
|
1654
1584
|
value: async (...args) => {
|
|
1655
1585
|
try {
|
|
1656
1586
|
const result = await iterator.next(...args);
|
|
1587
|
+
if (!result.done) {
|
|
1588
|
+
onChunk?.(result.value);
|
|
1589
|
+
}
|
|
1657
1590
|
if (result.done) {
|
|
1658
|
-
await invokeFinalize();
|
|
1591
|
+
await invokeFinalize("complete");
|
|
1659
1592
|
}
|
|
1660
1593
|
return result;
|
|
1661
1594
|
} catch (error) {
|
|
1662
|
-
await invokeFinalize().catch(() => void 0);
|
|
1595
|
+
await invokeFinalize("error", error).catch(() => void 0);
|
|
1663
1596
|
throw error;
|
|
1664
1597
|
}
|
|
1665
1598
|
},
|
|
@@ -1668,39 +1601,49 @@ function wrapStreamForUsageTap(source, finalize, ctx) {
|
|
|
1668
1601
|
});
|
|
1669
1602
|
Object.defineProperty(wrapped, "return", {
|
|
1670
1603
|
value: async (value) => {
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1604
|
+
try {
|
|
1605
|
+
if (typeof iterator.return === "function") {
|
|
1606
|
+
const rawResult = await iterator.return(value);
|
|
1607
|
+
if (!isIteratorResult(rawResult)) {
|
|
1608
|
+
throw new TypeError("Iterator.return() returned an invalid result");
|
|
1609
|
+
}
|
|
1610
|
+
await invokeFinalize("cancel");
|
|
1611
|
+
return rawResult;
|
|
1675
1612
|
}
|
|
1676
|
-
await invokeFinalize();
|
|
1677
|
-
return
|
|
1613
|
+
await invokeFinalize("cancel");
|
|
1614
|
+
return { done: true, value };
|
|
1615
|
+
} catch (error) {
|
|
1616
|
+
await invokeFinalize("error", error).catch(() => void 0);
|
|
1617
|
+
throw error;
|
|
1678
1618
|
}
|
|
1679
|
-
await invokeFinalize();
|
|
1680
|
-
return { done: true, value };
|
|
1681
1619
|
},
|
|
1682
1620
|
configurable: true,
|
|
1683
1621
|
writable: true
|
|
1684
1622
|
});
|
|
1685
1623
|
Object.defineProperty(wrapped, "throw", {
|
|
1686
1624
|
value: async (error) => {
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1625
|
+
try {
|
|
1626
|
+
if (typeof iterator.throw === "function") {
|
|
1627
|
+
const rawResult = await iterator.throw(error);
|
|
1628
|
+
if (!isIteratorResult(rawResult)) {
|
|
1629
|
+
throw new TypeError("Iterator.throw() returned an invalid result");
|
|
1630
|
+
}
|
|
1631
|
+
await invokeFinalize("error", error);
|
|
1632
|
+
return rawResult;
|
|
1691
1633
|
}
|
|
1692
|
-
await invokeFinalize();
|
|
1693
|
-
|
|
1634
|
+
await invokeFinalize("error", error);
|
|
1635
|
+
throw error;
|
|
1636
|
+
} catch (thrownError) {
|
|
1637
|
+
await invokeFinalize("error", thrownError).catch(() => void 0);
|
|
1638
|
+
throw thrownError;
|
|
1694
1639
|
}
|
|
1695
|
-
await invokeFinalize();
|
|
1696
|
-
throw error;
|
|
1697
1640
|
},
|
|
1698
1641
|
configurable: true,
|
|
1699
1642
|
writable: true
|
|
1700
1643
|
});
|
|
1701
1644
|
Object.defineProperty(wrapped, "__usageTapFinalize", {
|
|
1702
1645
|
value: async () => {
|
|
1703
|
-
await invokeFinalize();
|
|
1646
|
+
await invokeFinalize("manual");
|
|
1704
1647
|
},
|
|
1705
1648
|
configurable: true
|
|
1706
1649
|
});
|