@opengeni/worker-bundle 0.8.3 → 0.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,6 +5,7 @@ import type { EventLogger } from "@opengeni/events";
5
5
  import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
6
6
  import {
7
7
  SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,
8
+ modelUsageTokenCountOrNull,
8
9
  type RuntimeMetricsHooks,
9
10
  type SelfhostedOpObservation,
10
11
  type SelfhostedOpObserver,
@@ -17,6 +18,13 @@ export type CreditBalanceGauge = { accountId: string; balanceMicros: number };
17
18
 
18
19
  const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
19
20
  const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
21
+ const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>();
22
+
23
+ // A worker process reaching one trillion tokens in one provider/cache counter is
24
+ // already far outside an ordinary scrape lifetime. Refuse further increments
25
+ // before floating-point precision can degrade; a restart resets both the
26
+ // process-local Prometheus registry and this guard together.
27
+ const MAX_MODEL_CACHE_COUNTER_TOTAL = 1_000_000_000_000;
20
28
 
21
29
  export function observabilityEventLogger(observability: Observability): EventLogger {
22
30
  return {
@@ -553,7 +561,8 @@ export function recordModelInputTokens(
553
561
  provider: string,
554
562
  inputTokens: number,
555
563
  ): void {
556
- if (!(inputTokens > 0)) {
564
+ const normalizedInputTokens = modelUsageTokenCountOrNull(inputTokens);
565
+ if (normalizedInputTokens === null || normalizedInputTokens === 0) {
557
566
  return;
558
567
  }
559
568
  observability.observeHistogram({
@@ -561,7 +570,7 @@ export function recordModelInputTokens(
561
570
  help: "Observed input (context) tokens per model response, by provider.",
562
571
  buckets: MODEL_INPUT_TOKENS_BUCKETS,
563
572
  labels: { provider },
564
- value: inputTokens,
573
+ value: normalizedInputTokens,
565
574
  });
566
575
  }
567
576
 
@@ -598,29 +607,56 @@ const MODEL_CACHE_HIT_RATIO_BUCKETS = [
598
607
  * - `opengeni_model_cached_tokens_total{provider}` — cumulative prompt tokens the
599
608
  * provider served from cache. Advanced only when >0, so a provider that never
600
609
  * reports cached tokens contributes nothing rather than phantom zero-increments.
610
+ * - `opengeni_model_cache_write_tokens_total{provider}` — cumulative tokens a
611
+ * reporting provider wrote into its prompt cache. Absent telemetry contributes
612
+ * nothing; in particular, Codex subscription usage does not currently expose it.
601
613
  * - `opengeni_model_cache_hit_ratio{provider}` — cached/prompt for the call,
602
- * observed whenever prompt tokens are known (>0). A call whose provider does NOT
603
- * report cached_tokens records a real 0 here — "we saw a call and the cache did
604
- * nothing" is exactly the signal the alert watches, so it must not be swallowed.
605
- * Absent/zero/non-finite `cachedTokens` (providers that don't report it) is safe: no
606
- * counter increment, ratio 0. A call with no prompt tokens has no ratio (skipped).
614
+ * observed only when prompt tokens are known (>0) AND cached tokens are reported.
615
+ * A reported cached-token zero records a real 0%; absent/null telemetry remains
616
+ * unknown and must not fabricate a low-cache signal.
617
+ * - `opengeni_model_cache_read_telemetry_total{provider,status}` — one bounded
618
+ * availability observation per authoritative call (`reported` or `missing`).
619
+ * Invalid, fractional, unsafe, or over-contract token telemetry is rejected. A call
620
+ * with no prompt tokens has no ratio (skipped).
607
621
  */
608
622
  export function recordModelCacheTokens(
609
623
  observability: Observability,
610
624
  provider: string,
611
- input: { cachedTokens: number | null | undefined; promptTokens: number | null | undefined },
625
+ input: {
626
+ cachedTokens: number | null | undefined;
627
+ cacheWriteTokens?: number | null | undefined;
628
+ promptTokens: number | null | undefined;
629
+ },
612
630
  ): void {
613
- const cached = nonNegativeTokenCount(input.cachedTokens);
614
- const prompt = nonNegativeTokenCount(input.promptTokens);
615
- if (cached > 0) {
616
- observability.incrementCounter({
631
+ const cached = modelUsageTokenCountOrNull(input.cachedTokens);
632
+ const cacheWrite = modelUsageTokenCountOrNull(input.cacheWriteTokens);
633
+ const prompt = modelUsageTokenCountOrNull(input.promptTokens);
634
+ incrementBoundedModelCacheCounter(observability, {
635
+ name: "opengeni_model_cache_read_telemetry_total",
636
+ help: "Authoritative model calls with reported or missing cache-read telemetry, by provider.",
637
+ provider,
638
+ labels: { provider, status: cached === null ? "missing" : "reported" },
639
+ amount: 1,
640
+ });
641
+ if (cached !== null && cached > 0) {
642
+ incrementBoundedModelCacheCounter(observability, {
617
643
  name: "opengeni_model_cached_tokens_total",
618
644
  help: "Total prompt tokens served from the provider's prompt cache, by provider.",
645
+ provider,
619
646
  labels: { provider },
620
647
  amount: cached,
621
648
  });
622
649
  }
623
- if (prompt > 0) {
650
+ if (cacheWrite !== null && cacheWrite > 0) {
651
+ incrementBoundedModelCacheCounter(observability, {
652
+ name: "opengeni_model_cache_write_tokens_total",
653
+ help: "Total prompt tokens written to the provider's prompt cache, by provider.",
654
+ provider,
655
+ labels: { provider },
656
+ amount: cacheWrite,
657
+ });
658
+ }
659
+ if (cached !== null && prompt !== null && prompt > 0) {
624
660
  observability.observeHistogram({
625
661
  name: "opengeni_model_cache_hit_ratio",
626
662
  help: "Per-call prompt-cache hit ratio (cached/prompt tokens) by provider.",
@@ -633,8 +669,40 @@ export function recordModelCacheTokens(
633
669
  }
634
670
  }
635
671
 
636
- function nonNegativeTokenCount(value: number | null | undefined): number {
637
- return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0;
672
+ function incrementBoundedModelCacheCounter(
673
+ observability: Observability,
674
+ input: {
675
+ name: string;
676
+ help: string;
677
+ provider: string;
678
+ labels: Record<string, string>;
679
+ amount: number;
680
+ },
681
+ ): void {
682
+ const totals = modelCacheCounterTotals.get(observability) ?? new Map<string, number>();
683
+ if (!modelCacheCounterTotals.has(observability)) {
684
+ modelCacheCounterTotals.set(observability, totals);
685
+ }
686
+ const labelKey = Object.entries(input.labels)
687
+ .sort(([left], [right]) => left.localeCompare(right))
688
+ .map(([key, value]) => `${key}=${value}`)
689
+ .join(",");
690
+ const key = `${input.name}:${labelKey}`;
691
+ const current = totals.get(key) ?? 0;
692
+ if (input.amount > MAX_MODEL_CACHE_COUNTER_TOTAL - current) {
693
+ observability.warn("model cache metric cumulative limit reached", {
694
+ provider: input.provider,
695
+ metric: input.name,
696
+ });
697
+ return;
698
+ }
699
+ observability.incrementCounter({
700
+ name: input.name,
701
+ help: input.help,
702
+ labels: input.labels,
703
+ amount: input.amount,
704
+ });
705
+ totals.set(key, current + input.amount);
638
706
  }
639
707
 
640
708
  /**