@openhoo/hoopilot 1.1.0 → 1.2.0

package/README.md

@@ -273,7 +273,7 @@ Hoopilot tracks token usage, request counts, and latency in memory while the ser
 - `GET /v1/usage` returns JSON combining the proxy metrics snapshot with live Copilot quota fetched from GitHub and cached for 60 seconds. If quota cannot be read, `copilot` is `null` and `copilot_error` explains why. The snapshot's `proxy.githubRateLimit` field reports the most recent GitHub REST rate-limit budget per resource (`limit`, `remaining`, `used`, `resetAt`, `retryAfterSeconds`, `observedAt`).
 - `hoopilot usage` prints your Copilot plan and quota — and, when GitHub returns them, your GitHub API rate-limit budget — from the command line.
 
-Token usage is read from the upstream `usage` object. For streaming chat completions, usage is only available when the client sends `stream_options: {"include_usage": true}`; Hoopilot does not inject that flag. Responses API streaming always reports usage, so streamed Responses requests are fully accounted.
+Token usage is read from the upstream `usage` object. For streaming chat completions, usage is only available when the client sends `stream_options: {"include_usage": true}`; Hoopilot does not inject that flag. Responses API streaming always reports usage, so streamed Responses requests are fully accounted. The `hoopilot_token_extraction_total{outcome="extracted"|"missing"}` counter (mirrored in `/v1/usage` as `proxy.tokens.extraction`) tracks how often a completion reported usage versus not, so a rising `missing` count flags clients whose token usage is going unaccounted.
 
 GitHub API usage is read from the `x-ratelimit-*` response headers that `api.github.com` returns on the `copilot_internal/user` quota call Hoopilot already makes, so it costs no extra request. (The Copilot completion host `api.githubcopilot.com` does not currently emit these headers, so per-completion rate-limit data is not yet available there.)
 

package/dist/cli.js

@@ -1689,6 +1689,7 @@ var MetricsRegistry = class {
   #upstream = /* @__PURE__ */ new Map();
   #copilotQuota;
   #githubRateLimit = /* @__PURE__ */ new Map();
+  #extraction = { extracted: 0, missing: 0 };
   constructor(options = {}) {
     this.#startedAtMs = (options.now ?? Date.now)();
   }
@@ -1705,6 +1706,19 @@ var MetricsRegistry = class {
     this.#requests.set(key, (this.#requests.get(key) ?? 0) + 1);
     this.#observeDuration(observation.route, observation.durationMs / 1e3);
   }
+  /**
+   * Record whether one upstream completion reported token usage. `missing`
+   * counts responses that carried no usage object — most often streamed Chat
+   * Completions sent without `stream_options: {"include_usage": true}` — so a
+   * rising miss rate flags clients whose token usage is going unaccounted.
+   */
+  recordTokenExtraction(extracted) {
+    if (extracted) {
+      this.#extraction.extracted += 1;
+    } else {
+      this.#extraction.missing += 1;
+    }
+  }
   /** Accumulate token counts for a model from one upstream completion. */
   recordTokens(model, usage) {
     const name = this.#modelLabel(model);
@@ -1812,7 +1826,7 @@ var MetricsRegistry = class {
       inFlight: this.#inFlight,
       requests: { byRoute, byStatus, total: requestsTotal },
       startedAt: new Date(this.#startedAtMs).toISOString(),
-      tokens: { byModel, ...tokenTotals },
+      tokens: { byModel, extraction: { ...this.#extraction }, ...tokenTotals },
       upstream: { errors: upstreamErrors, total: upstreamTotal },
       uptimeSeconds: Math.max(0, Math.round((now() - this.#startedAtMs) / 1e3))
     };
@@ -1862,6 +1876,16 @@ var MetricsRegistry = class {
     for (const [model, totals] of this.#tokens) {
       lines.push(`hoopilot_model_requests_total${labels({ model })} ${totals.requests}`);
     }
+    lines.push(
+      "# HELP hoopilot_token_extraction_total Completions by whether upstream reported token usage."
+    );
+    lines.push("# TYPE hoopilot_token_extraction_total counter");
+    lines.push(
+      `hoopilot_token_extraction_total${labels({ outcome: "extracted" })} ${this.#extraction.extracted}`
+    );
+    lines.push(
+      `hoopilot_token_extraction_total${labels({ outcome: "missing" })} ${this.#extraction.missing}`
+    );
     lines.push("# HELP hoopilot_request_duration_seconds Request duration by route.");
     lines.push("# TYPE hoopilot_request_duration_seconds histogram");
     for (const [route, entry] of this.#durations) {
@@ -2017,23 +2041,25 @@ var MetricsRegistry = class {
     }
   }
 };
-function observeResponseUsage(response, fallbackModel, onUsage, signal) {
+function observeResponseUsage(response, fallbackModel, onUsage, signal, onOutcome) {
   const body = response.body;
   if (!body) {
     return response;
   }
   const [clientBranch, observerBranch] = body.tee();
   const isSse = response.headers.get("content-type")?.includes("text/event-stream") ?? false;
-  void consumeUsage(observerBranch, isSse, fallbackModel, onUsage, signal).catch(() => {
-  });
+  void consumeUsage(observerBranch, isSse, fallbackModel, onUsage, signal, onOutcome).catch(
+    () => {
+    }
+  );
   return new Response(clientBranch, {
     headers: response.headers,
     status: response.status,
     statusText: response.statusText
   });
 }
-function recordResponseTextUsage(text, isSse, fallbackModel, onUsage) {
-  const accumulator = createUsageAccumulator(fallbackModel, onUsage);
+function recordResponseTextUsage(text, isSse, fallbackModel, onUsage, onOutcome) {
+  const accumulator = createUsageAccumulator(fallbackModel, onUsage, onOutcome);
   if (isSse) {
     for (const line of text.split(/\r?\n/)) {
       considerSseLine(line, accumulator.consider);
@@ -2046,7 +2072,7 @@ function recordResponseTextUsage(text, isSse, fallbackModel, onUsage) {
   }
   accumulator.finish();
 }
-async function consumeUsage(stream, isSse, fallbackModel, onUsage, signal) {
+async function consumeUsage(stream, isSse, fallbackModel, onUsage, signal, onOutcome) {
   const reader = stream.getReader();
   const onAbort = () => {
     reader.cancel().catch(() => {
@@ -2059,7 +2085,12 @@ async function consumeUsage(stream, isSse, fallbackModel, onUsage, signal) {
     signal?.addEventListener("abort", onAbort, { once: true });
   }
   const decoder = new TextDecoder();
-  const accumulator = createUsageAccumulator(fallbackModel, onUsage);
+  const guardedOutcome = onOutcome ? (extracted) => {
+    if (!signal?.aborted) {
+      onOutcome(extracted);
+    }
+  } : void 0;
+  const accumulator = createUsageAccumulator(fallbackModel, onUsage, guardedOutcome);
   let buffer = "";
   let bufferedBytes = 0;
   let overflowed = false;
@@ -2107,7 +2138,7 @@ async function consumeUsage(stream, isSse, fallbackModel, onUsage, signal) {
   }
   accumulator.finish();
 }
-function createUsageAccumulator(fallbackModel, onUsage) {
+function createUsageAccumulator(fallbackModel, onUsage, onOutcome) {
   let model = fallbackModel;
   let usage;
   return {
@@ -2126,6 +2157,7 @@ function createUsageAccumulator(fallbackModel, onUsage) {
       if (usage) {
         onUsage(model, usage);
       }
+      onOutcome?.(usage !== void 0);
     }
   };
 }
@@ -2254,6 +2286,7 @@ function createHoopilotHandler(options = {}) {
   const metrics = options.metrics ?? new MetricsRegistry();
   const readUsage = createUsageReader(client, metrics);
   const recordTokens = (model, usage) => metrics.recordTokens(model, usage);
+  const recordExtraction = (extracted) => metrics.recordTokenExtraction(extracted);
   const streamingProxyMode = resolveStreamingProxyMode(options);
   const bufferProxyBodies = shouldBufferProxyBodies(streamingProxyMode);
   return async (request) => {
@@ -2319,6 +2352,7 @@ function createHoopilotHandler(options = {}) {
             client,
             metrics,
             recordTokens,
+            recordExtraction,
             request,
             requestLogger,
             bufferProxyBodies
@@ -2334,6 +2368,7 @@ function createHoopilotHandler(options = {}) {
             client,
             metrics,
             recordTokens,
+            recordExtraction,
             request,
             requestLogger,
             bufferProxyBodies
@@ -2346,6 +2381,7 @@ function createHoopilotHandler(options = {}) {
             client,
             metrics,
             recordTokens,
+            recordExtraction,
             request,
             requestLogger,
             bufferProxyBodies
@@ -2354,7 +2390,14 @@ function createHoopilotHandler(options = {}) {
       }
       if (request.method === "POST" && apiPath === "/v1/responses/compact") {
         return finish(
-          await handleResponsesCompact(client, metrics, recordTokens, request, requestLogger)
+          await handleResponsesCompact(
+            client,
+            metrics,
+            recordTokens,
+            recordExtraction,
+            request,
+            requestLogger
+          )
         );
       }
       if (request.method === "POST" && apiPath === "/v1/responses") {
@@ -2363,6 +2406,7 @@ function createHoopilotHandler(options = {}) {
             client,
             metrics,
             recordTokens,
+            recordExtraction,
             request,
             requestLogger,
             bufferProxyBodies
@@ -2439,7 +2483,7 @@ function startHoopilotServer(options = {}) {
     url: `http://${urlHost(host)}:${server.port}`
   };
 }
-async function handleAnthropicMessages(client, metrics, recordTokens, request, logger, bufferProxyBodies) {
+async function handleAnthropicMessages(client, metrics, recordTokens, recordExtraction, request, logger, bufferProxyBodies) {
   const anthropicRequest = await readJson(request);
   const responsesRequest = anthropicMessagesToResponsesRequest(anthropicRequest);
   const upstream = await client.responses(JSON.stringify(responsesRequest), request.signal);
@@ -2452,12 +2496,18 @@ async function handleAnthropicMessages(client, metrics, recordTokens, request, l
   if (isStreamingResponse(upstream) && upstream.body) {
     if (bufferProxyBodies) {
       const text = await upstream.text();
-      recordResponseTextUsage(text, true, model, recordTokens);
+      recordResponseTextUsage(text, true, model, recordTokens, recordExtraction);
       return proxyResponse(
         responseFromText(upstream, responsesSseTextToAnthropicSseText(text, { model }))
       );
     }
-    const observed = observeResponseUsage(upstream, model, recordTokens, request.signal);
+    const observed = observeResponseUsage(
+      upstream,
+      model,
+      recordTokens,
+      request.signal,
+      recordExtraction
+    );
     if (!observed.body) {
       return proxyResponse(observed);
     }
@@ -2475,6 +2525,7 @@ async function handleAnthropicMessages(client, metrics, recordTokens, request, l
     const responseModel = typeof body.model === "string" ? body.model.trim() : "";
     recordTokens(responseModel || model, usage);
   }
+  recordExtraction(usage !== void 0);
   return jsonResponse(responsesResponseToAnthropicMessage(body, model));
 }
 function handleAnthropicCountTokens(body) {
@@ -2500,7 +2551,7 @@ async function handleModels(client, metrics, signal, logger) {
   logUpstreamSuccess(logger, "/models", upstream.status);
   return jsonResponse(normalizeModelsResponse(await upstream.json()));
 }
-async function handleChatCompletions(client, metrics, recordTokens, request, logger, bufferProxyBodies) {
+async function handleChatCompletions(client, metrics, recordTokens, recordExtraction, request, logger, bufferProxyBodies) {
   const chatRequest = normalizeChatCompletionRequest(await readJson(request));
   const upstream = await client.chatCompletions(chatRequest, request.signal);
   metrics.recordUpstream("/chat/completions", upstream.ok);
@@ -2515,11 +2566,12 @@ async function handleChatCompletions(client, metrics, recordTokens, request, log
       model,
       recordTokens,
       request.signal,
-      bufferProxyBodies
+      bufferProxyBodies,
+      recordExtraction
     )
   );
 }
-async function handleCompletions(client, metrics, recordTokens, request, logger, bufferProxyBodies) {
+async function handleCompletions(client, metrics, recordTokens, recordExtraction, request, logger, bufferProxyBodies) {
   const body = await readJson(request);
   const upstream = await client.chatCompletions(
     completionsRequestToChatCompletion(body),
@@ -2534,7 +2586,7 @@ async function handleCompletions(client, metrics, recordTokens, request, logger,
   if (isStreamingResponse(upstream) && upstream.body) {
     if (bufferProxyBodies) {
       const upstreamText = await upstream.text();
-      recordResponseTextUsage(upstreamText, true, model, recordTokens);
+      recordResponseTextUsage(upstreamText, true, model, recordTokens, recordExtraction);
       const text = completionSseTextFromChatSseText(upstreamText);
       return proxyResponse(responseFromText(upstream, text));
     }
@@ -2547,7 +2599,8 @@ async function handleCompletions(client, metrics, recordTokens, request, logger,
         }),
         model,
         recordTokens,
-        request.signal
+        request.signal,
+        recordExtraction
       )
     );
   }
@@ -2557,9 +2610,10 @@ async function handleCompletions(client, metrics, recordTokens, request, logger,
     const responseModel = typeof completion.model === "string" ? completion.model.trim() : "";
     recordTokens(responseModel || model, usage);
   }
+  recordExtraction(usage !== void 0);
   return jsonResponse(chatCompletionToCompletion(completion));
 }
-async function handleResponses(client, metrics, recordTokens, request, logger, bufferProxyBodies) {
+async function handleResponses(client, metrics, recordTokens, recordExtraction, request, logger, bufferProxyBodies) {
   const body = await readJsonText(request);
   const upstream = await client.responses(body, request.signal);
   metrics.recordUpstream("/responses", upstream.ok);
@@ -2574,11 +2628,12 @@ async function handleResponses(client, metrics, recordTokens, request, logger, b
       model,
       recordTokens,
       request.signal,
-      bufferProxyBodies
+      bufferProxyBodies,
+      recordExtraction
     )
   );
 }
-async function handleResponsesCompact(client, metrics, recordTokens, request, logger) {
+async function handleResponsesCompact(client, metrics, recordTokens, recordExtraction, request, logger) {
   const body = await readJson(request);
   const upstream = await client.responses(
     JSON.stringify({ ...body, stream: false }),
@@ -2591,17 +2646,23 @@ async function handleResponsesCompact(client, metrics, recordTokens, request, lo
   logUpstreamSuccess(logger, "/responses", upstream.status);
   const isSse = isStreamingResponse(upstream);
   const text = await upstream.text();
-  recordResponseTextUsage(text, isSse, normalizeRequestedModel(body.model), recordTokens);
+  recordResponseTextUsage(
+    text,
+    isSse,
+    normalizeRequestedModel(body.model),
+    recordTokens,
+    recordExtraction
+  );
   return jsonResponse(responsesCompactionResult(text, isSse));
 }
-async function responseWithObservedUsage(response, fallbackModel, recordTokens, signal, bufferBody) {
+async function responseWithObservedUsage(response, fallbackModel, recordTokens, signal, bufferBody, recordExtraction) {
   const isSse = isStreamingResponse(response);
   if (bufferBody && response.body) {
     const text = await response.text();
-    recordResponseTextUsage(text, isSse, fallbackModel, recordTokens);
+    recordResponseTextUsage(text, isSse, fallbackModel, recordTokens, recordExtraction);
     return responseFromText(response, text);
   }
-  return observeResponseUsage(response, fallbackModel, recordTokens, signal);
+  return observeResponseUsage(response, fallbackModel, recordTokens, signal, recordExtraction);
 }
 function responseFromText(source, text) {
   return new Response(text, {