@jeffusion/bungee-core 6.1.1 → 6.2.0

package/dist/plugins/token-metering/index.js

@@ -2663,7 +2663,7 @@ class ResponsesWebSocketMetering {
     connection.active.delete(responseId);
     connection.finished.add(responseId);
     const usageComplete = count(usage?.input_tokens) && count(usage?.output_tokens);
-    await this.finish(response, event, type === "response.completed" && usageComplete ? "completed" : "failed", !usageComplete);
+    await this.finish(response, event, type === "response.completed" && usageComplete ? "completed" : "failed", !usageComplete, type === "response.completed" ? body : undefined);
     if (connection.finished.size > MAX_TOMBSTONES)
       connection.finished.delete(connection.finished.values().next().value);
   }
@@ -2697,23 +2697,23 @@ class ResponsesWebSocketMetering {
     this.callbacks.prepareRequest(response.requestId);
     return await this.emit(response, event, { phase: "selected" }) && await this.emit(response, event, { phase: "request", url: event.upstreamUrl, body: { model: response.model, stream: true } });
   }
-  async emit(response, event, phase) {
+  async emit(response, event, phase, settlement) {
     if (this.closed || !event.isActive()) {
       this.callbacks.discardRequest(response.requestId, response.attemptId);
       return false;
     }
-    await this.callbacks.emit({ requestId: response.requestId, attemptId: response.attemptId, keyId: event.keyId, routeId: event.routeId, upstreamId: event.upstreamId, isActive: event.isActive, ...phase });
+    await this.callbacks.emit({ requestId: response.requestId, attemptId: response.attemptId, observedAtMonotonicMs: event.observedAtMonotonicMs, keyId: event.keyId, routeId: event.routeId, upstreamId: event.upstreamId, isActive: event.isActive, ...phase }, settlement);
     if (this.closed || !event.isActive()) {
       this.callbacks.discardRequest(response.requestId, response.attemptId);
       return false;
     }
     return true;
   }
-  async finish(response, event, outcome, incomplete) {
+  async finish(response, event, outcome, incomplete, generationResponse) {
     this.responseCount--;
-    if (incomplete && !await this.emit(response, event, { phase: "incomplete", reason: "raw-response-incomplete" }))
+    if (incomplete && !await this.emit(response, event, { phase: "incomplete", reason: "raw-response-incomplete" }, { accountingUsageIncomplete: generationResponse !== undefined }))
       return;
-    if (!await this.emit(response, event, { phase: "end", sent: true, outcome }))
+    if (!await this.emit(response, event, { phase: "end", sent: true, outcome }, { generationResponse }))
       return;
     await this.emit(response, event, { phase: "request-end" });
   }
@@ -2741,6 +2741,93 @@ class ResponsesWebSocketMetering {
   }
 }
 
+// plugins/token-metering/server/generation-timing.ts
+var record4 = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+var text = (value) => typeof value === "string" && value.length > 0;
+var values = (value) => Array.isArray(value) ? value : [];
+var parameters = (value) => text(value) || record4(value) && Object.keys(value).length > 0;
+var content = (value) => text(value) || values(value).some((part) => record4(part) && (text(part.text) || text(part.thinking)));
+function generationSignals(provider, body, event) {
+  const type = event || body.type;
+  if (provider === "openai") {
+    const choices = values(body.choices).filter(record4);
+    const output = choices.some((choice) => text(choice.text) || record4(choice.delta) && (content(choice.delta.content) || text(choice.delta.reasoning_content) || text(choice.delta.reasoning) || record4(choice.delta.function_call) && text(choice.delta.function_call.arguments) || values(choice.delta.tool_calls).some((tool) => record4(tool) && record4(tool.function) && text(tool.function.arguments)))) || typeof type === "string" && [
+      "response.output_text.delta",
+      "response.reasoning_text.delta",
+      "response.reasoning_summary_text.delta",
+      "response.function_call_arguments.delta",
+      "response.refusal.delta"
+    ].includes(type) && text(body.delta);
+    return { output, completed: type === "response.completed" || choices.some((choice) => choice.finish_reason !== undefined && choice.finish_reason !== null) };
+  }
+  if (provider === "anthropic") {
+    const delta = record4(body.delta) ? body.delta : undefined;
+    const block = type === "content_block_start" && record4(body.content_block) ? body.content_block : undefined;
+    return {
+      output: type === "content_block_delta" && !!delta && (delta.type === "text_delta" && text(delta.text) || delta.type === "thinking_delta" && text(delta.thinking) || delta.type === "input_json_delta" && text(delta.partial_json)) || !!block && (text(block.text) || text(block.thinking) || block.type === "tool_use" && parameters(block.input)),
+      completed: type === "message_stop" || type === "message_delta" && !!delta && text(delta.stop_reason)
+    };
+  }
+  const candidates = values(body.candidates).filter(record4);
+  return {
+    output: candidates.some((candidate) => record4(candidate.content) && values(candidate.content.parts).some((part) => record4(part) && (text(part.text) || record4(part.functionCall) && parameters(part.functionCall.args)))),
+    completed: candidates.some((candidate) => text(candidate.finishReason))
+  };
+}
+
+class GenerationTiming {
+  firstOutput;
+  completedAt;
+  terminal = false;
+  missingTime = false;
+  failed = false;
+  branches = new Map;
+  observe(provider, body, at, event) {
+    const type = event || body.type;
+    const response = record4(body.response) ? body.response : undefined;
+    this.failed ||= body.error != null || response?.error != null || type === "error" || type === "response.failed" || type === "response.incomplete" || response?.status === "failed" || response?.status === "incomplete";
+    if (this.terminal)
+      return;
+    const signal = generationSignals(provider, body, event);
+    const valid = at !== undefined && Number.isFinite(at);
+    if (signal.output && this.firstOutput === undefined) {
+      if (valid)
+        this.firstOutput = at;
+      else
+        this.missingTime = true;
+    }
+    const branches = provider === "openai" ? values(body.choices) : provider === "gemini" ? values(body.candidates) : [];
+    if (branches.length) {
+      for (const [position, branch] of branches.entries()) {
+        if (!record4(branch))
+          continue;
+        const id = Number.isSafeInteger(branch.index) && branch.index >= 0 ? branch.index : position;
+        const completed = provider === "openai" ? text(branch.finish_reason) : text(branch.finishReason);
+        if (completed && !valid)
+          this.missingTime = true;
+        const previous = this.branches.get(id);
+        this.branches.set(id, completed ? { completed: true, at } : previous ?? { completed: false });
+      }
+    } else if (signal.completed) {
+      this.terminal = true;
+      if (valid)
+        this.completedAt = at;
+      else
+        this.missingTime = true;
+    }
+  }
+  duration(endedAt) {
+    if (this.failed || this.missingTime || this.firstOutput === undefined)
+      return;
+    const completedBranches = this.branches.size > 0 && [...this.branches.values()].every((branch) => branch.completed);
+    const end = this.terminal ? this.completedAt : completedBranches ? Math.max(...[...this.branches.values()].map((branch) => branch.at)) : endedAt;
+    if (end === undefined || !Number.isFinite(end))
+      return;
+    const duration = end - this.firstOutput;
+    return Number.isFinite(duration) && duration > 0 ? duration : undefined;
+  }
+}
+
 // plugins/token-metering/server/index.ts
 var MAX_ACTIVE_ATTEMPTS = 1024;
 var LOG_INTERVAL_MS = 60000;
@@ -2773,6 +2860,7 @@ function createAttempt(event) {
     keyId: event.keyId ?? null,
     routeId: event.routeId || "unknown",
     upstreamId: event.upstreamId || "unknown",
+    timing: new GenerationTiming,
     provider: "unknown",
     sent: false,
     llm: false,
@@ -2780,6 +2868,7 @@ function createAttempt(event) {
     responseSeen: false,
     responseFailed: false,
     observationIncomplete: false,
+    generationObservationIncomplete: false,
     incompleteReasonLogged: false
   };
 }
@@ -2873,6 +2962,34 @@ function finalOutcome(input, canonicalOutcome, officialComplete = false) {
 }
 function finalizeAttempt(input) {
   let event = input.observedEvent;
+  const generationComplete = (input.generationCompleted ?? input.endOutcome === "completed") && input.responseSeen && !input.responseFailed && !(input.generationObservationIncomplete ?? input.observationIncomplete) && input.generationDurationMs !== undefined;
+  let generationOutput;
+  if (generationComplete) {
+    try {
+      let measured;
+      if (input.generationResponse) {
+        const measurement = createTokenAccountingSession({
+          provider: "openai",
+          model: input.model,
+          requestId: input.requestId,
+          attemptId: input.attemptId,
+          routeId: input.routeId,
+          upstreamId: input.upstreamId,
+          streaming: true
+        }, { deferFinalization: true });
+        measurement.consumeStreamChunk({ chunk: input.generationResponse });
+        measured = measurement.finalizeCompletedStream();
+      } else
+        measured = input.session?.finalizeCompletedStream();
+      if (measured) {
+        assertCanonicalTokenAccountingEventV2(measured);
+        const output = input.generationResponse ? measured : mergeOfficialUsage({ ...measured }, event);
+        if (output.outputTokens !== undefined && ["official", "local", "heuristic"].includes(output.outputAuthority)) {
+          generationOutput = Object.freeze({ tokens: output.outputTokens, source: output.outputAuthority === "official" ? "official" : "estimated" });
+        }
+      }
+    } catch {}
+  }
   try {
     const finalized = input.endOutcome === "completed" && input.responseSeen && !input.responseFailed && !input.observationIncomplete ? input.session?.finalizeCompletedStream() : input.session?.finalizeAbortedStream();
     if (finalized) {
@@ -2914,7 +3031,8 @@ function finalizeAttempt(input) {
     observationIncomplete: input.observationIncomplete,
     complete: outcome === "completed" && event?.inputTokens !== undefined && event?.outputTokens !== undefined && event.inputAuthority !== "partial" && event.outputAuthority !== "partial" && (!input.observationIncomplete || event.inputAuthority === "official" && event.outputAuthority === "official"),
     finishedAtMs: input.finishedAtMs,
-    settlementVersion: 1
+    settlementVersion: 1,
+    ...generationComplete ? { generationDurationMs: input.generationDurationMs, generationOutput } : {}
   });
 }
 var TokenMeteringPlugin = definePlugin(class {
@@ -2929,7 +3047,7 @@ var TokenMeteringPlugin = definePlugin(class {
   requiredPending = new Map;
   closed = false;
   websocket = new ResponsesWebSocketMetering({
-    emit: (event) => this.handleAttemptObservation(event, true),
+    emit: (event, settlement) => this.handleAttemptObservation(event, true, settlement),
     prepareRequest: (requestId) => this.prepareRequest(requestId),
     discardRequest: (requestId, attemptId) => {
       this.attempts.delete(attemptId);
@@ -3063,7 +3181,7 @@ var TokenMeteringPlugin = definePlugin(class {
       });
     }
   }
-  async handleAttemptObservation(event, websocket = false) {
+  async handleAttemptObservation(event, websocket = false, settlement = {}) {
     if (!event.isActive()) {
       this.attempts.delete(event.attemptId);
       return;
@@ -3126,7 +3244,7 @@ var TokenMeteringPlugin = definePlugin(class {
           attempt.session?.consumeRequest({ body: { model } });
         }
       }
-      attempt.responseFailed ||= event.status >= 400 || body.error !== undefined;
+      attempt.responseFailed ||= event.status >= 400 || body.error != null;
       const provider = attempt.provider === "unknown" ? classifyResponse(body, attempt.llm, event.envelope?.event) : attempt.provider;
       if (!provider)
         return;
@@ -3138,6 +3256,8 @@ var TokenMeteringPlugin = definePlugin(class {
         attempt.streaming = event.protocol === "sse";
         ensureSession(attempt, provider);
       }
+      if (event.protocol === "sse")
+        attempt.timing.observe(provider, body, event.observedAtMonotonicMs, event.envelope?.event);
       const session = attempt.session;
       if (!session)
         return;
@@ -3151,6 +3271,7 @@ var TokenMeteringPlugin = definePlugin(class {
         attempt.latestEvent = attempt.responseFailed ? officialOnly(merged) : merged;
       } catch (error) {
         attempt.observationIncomplete = true;
+        attempt.generationObservationIncomplete = true;
         if (attempt.latestEvent)
           attempt.latestEvent = officialOnly(attempt.latestEvent);
         this.logger.debug("Failed to consume token metering attempt observation", { attemptId: event.attemptId, error });
@@ -3158,6 +3279,8 @@ var TokenMeteringPlugin = definePlugin(class {
       return;
     }
     if (event.phase === "incomplete") {
+      if (!settlement.accountingUsageIncomplete)
+        attempt.generationObservationIncomplete = true;
       if (!attempt.observationIncomplete) {
         attempt.observationIncomplete = true;
         if (!attempt.incompleteReasonLogged) {
@@ -3192,6 +3315,10 @@ var TokenMeteringPlugin = definePlugin(class {
         observedEvent: attempt.latestEvent ? { ...attempt.latestEvent } : undefined,
         finishedAtMs: Date.now(),
         endOutcome: event.outcome,
+        generationDurationMs: attempt.timing.duration(event.observedAtMonotonicMs),
+        generationCompleted: websocket ? settlement.generationResponse !== undefined : event.outcome === "completed",
+        generationResponse: settlement.generationResponse,
+        generationObservationIncomplete: attempt.generationObservationIncomplete,
         responseSeen: attempt.responseSeen,
         responseFailed: attempt.responseFailed,
         observationIncomplete: attempt.observationIncomplete
@@ -3216,4 +3343,4 @@ export {
   server_default as default
 };
 
-//# debugId=6E560425A55B5FB164756E2164756E21
+//# debugId=DB115B0B5828A5AC64756E2164756E21