cartethyia 2.5.2 → 2.5.4

package/src/transport/dispatch/attempt-finalize.ts

@@ -39,6 +39,27 @@ export async function shouldCaptureProviderBody(
   return payloadCaptureKeepsBodies(db, tenantId);
 }
 
+/**
+ * Per-stage cap for the provider request/response tee at one capture depth.
+ *
+ * The tee is what the drawer's "Provider Request" and "Provider Response"
+ * panels read, so its cap is the most a provider panel can ever show. It used
+ * to be a flat 256 KiB at every depth: on a large request the translated
+ * provider body was clipped to 256 KiB before the shared record budget ever
+ * ran, so a tenant who raised the depth to Maximum — 16 MiB — still saw a
+ * quarter-megabyte provider request and an empty provider response. Derived
+ * from the same depth tiers as the record budget, so raising the depth now
+ * raises both.
+ *
+ * Capped at a quarter of the depth's total budget: four panels share it, and
+ * one of them must never be able to spend the whole thing.
+ */
+export function providerCaptureMaxBytes(depth?: TelemetryPayloadDepth): number {
+  if (depth === "moderate") return 2 * 1024 * 1024;
+  if (depth === "maximum") return 4 * 1024 * 1024;
+  return 256 * 1024;
+}
+
 /**
  * Same answer as {@link shouldCaptureProviderBody}, memoized on the request.
  *
@@ -62,6 +83,28 @@ export async function shouldCaptureProviderBodyForRequest(
   return resolved.ok === true && resolved.mode === "full";
 }
 
+/**
+ * The tenant's capture depth, memoized on the request like the mode above.
+ *
+ * Read *after* {@link shouldCaptureProviderBodyForRequest} has populated the
+ * memo, so the two answers come from one preference read. It sizes the
+ * provider tee: a tenant who raised the depth is asking for more of every
+ * panel, and a flat cap would answer with the same clipped body regardless.
+ */
+export async function resolveCaptureDepthForRequest(
+  db: CartethyiaDatabase,
+  tenantId: string | null,
+  state: { payloadCapturePreference?: unknown },
+): Promise<TelemetryPayloadDepth> {
+  if (state.payloadCapturePreference === undefined) {
+    state.payloadCapturePreference = await resolvePayloadCapture(db, tenantId);
+  }
+  const resolved = state.payloadCapturePreference as
+    | { readonly ok: true; readonly depth: TelemetryPayloadDepth }
+    | { readonly ok: false };
+  return resolved.ok === true ? resolved.depth : "minimum";
+}
+
 /**
  * Combined body cap for one capture depth. Fixed size tiers: every depth
  * stores all four drawer panels (client request, translated provider
@@ -103,7 +146,6 @@ export function parseCapturedBody(value: unknown): unknown {
   }
 }
 
-const PROVIDER_CAPTURE_MAX_BYTES = 256 * 1024;
 const PROVIDER_CAPTURE_AWAIT_MS = 2_000;
 const CAPTURABLE_RESPONSE_TYPES = /^(text\/|application\/(json|x-ndjson|sse))/i;
 
@@ -258,6 +300,7 @@ export function captureProviderExchange(
   inner: ValidatedOutboundFetch,
   sink: ProviderExchangeCapture,
   captureBody = true,
+  maxBytes = providerCaptureMaxBytes(),
 ): ValidatedOutboundFetch {
   return async (input, init) => {
     const url =
@@ -276,14 +319,12 @@ export function captureProviderExchange(
       // unbounded translated request body per attempt — up to `maxAttempts` of
       // them, all alive until the loop returns — costs more than it can tell
       // an operator.
-      ...(rawBody === null
-        ? {}
-        : { body: parseCapturedBody(rawBody.slice(0, PROVIDER_CAPTURE_MAX_BYTES)) }),
+      ...(rawBody === null ? {} : { body: parseCapturedBody(rawBody.slice(0, maxBytes)) }),
     };
     const response = await inner(input, init);
     sink.response =
       captureBody && shouldCaptureProviderResponse(response)
-        ? readBoundedBody(response.clone(), PROVIDER_CAPTURE_MAX_BYTES).then(parseCapturedBody)
+        ? readBoundedBody(response.clone(), maxBytes).then(parseCapturedBody)
         : Promise.resolve(null);
     // Read regardless of `captureBody`: this is metadata about the dispatch, not
     // payload retention, so a tenant in `metadata` mode still records which

package/src/transport/dispatch/generation.ts

@@ -34,6 +34,7 @@ import { ProxyRequestStateStore } from "../request/state";
 import { ProxyRequestPreparer } from "../request/preparer";
 import { completeAttempt, estimatedUsage } from "./attempt-finalize";
 import { runAttemptLoop } from "./attempt-loop";
+import { captureProviderRateLimit } from "./ratelimit-capture";
 
 export interface GenerationHandlerDeps {
   readonly db: CartethyiaDatabase;
@@ -125,6 +126,11 @@ async function passThrough(
   },
 ): Promise<Response> {
   const usage = estimatedUsage(0, 0);
+  // The CLI bucket comes back on every upstream response, so this is the free
+  // place to learn it: the quota surface then reads the stored value instead of
+  // spending a request of its own. Both the manual account test and automatic
+  // routing land here, so a captured reading follows real traffic.
+  captureProviderRateLimit(args.candidate.provider_id, args.candidate.provider_account_id, upstream.headers);
   await completeAttempt(state, {
     status: "completed",
     providerId: args.candidate.provider_id,

package/src/transport/dispatch/proxy-request.ts

@@ -15,7 +15,11 @@ import { messagesAdapter } from "../surface/messages/adapter";
 import { completionAdapter } from "../surface/completion";
 import { forwardedRequestHeaders, proxySuccessHeaders, buildUpstreamDispatchContext, reportedModelFrom } from "./upstream";
 import { resolvePromptCacheKey } from "../../providers/operations/session-resolution";
-import { applyTenantPreferences } from "../request/tenant-preferences";
+import {
+  applyTenantPreferences,
+  applyTenantDeadlineOverride,
+  resolveTenantImageClampMaxBytes,
+} from "../request/tenant-preferences";
 import { metrics } from "../../observability/metrics";
 import type { NetworkPoolSelector } from "../../network/pool/selector";
 import type { TelemetryBatchBuffer } from "../../observability/telemetry-buffer";
@@ -28,6 +32,8 @@ import {
   completionContext,
   estimatedUsage,
   parseCapturedBody,
+  providerCaptureMaxBytes,
+  resolveCaptureDepthForRequest,
   shouldCaptureProviderBodyForRequest,
   terminalFailure,
 } from "./attempt-finalize";
@@ -126,6 +132,22 @@ export async function handleProviderProxyRequest(
   if (!prepared)
     throw new GatewayError("admission_unavailable", 503, "The request was not prepared, so no route can be dialed. Resend the request.");
   const canonicalRequest = await applyTenantPreferences(prepared, deps.db);
+  // Per-tenant timeout, applied as soon as the tenant is known. The deadline
+  // was armed at admission with the deployment default; this re-arms it.
+  await applyTenantDeadlineOverride(
+    prepared.authorization.snapshot.tenant_id,
+    deps.db,
+    (ms) => state.extendDeadline(ms),
+  );
+  // Memoized on the request: the ceiling is a tenant setting, so it cannot
+  // change between failover attempts, and each wire encoder that clamps an
+  // image reads it from here instead of paying for a preference read per
+  // attempt.
+  const imageClampMaxBytes = await resolveTenantImageClampMaxBytes(
+    deps.db,
+    prepared.authorization.snapshot.tenant_id,
+  );
+  if (imageClampMaxBytes !== undefined) state.imageClampMaxBytes = imageClampMaxBytes;
   const candidates =
     prepared.eligibleRouteCandidates.length > 0 ? prepared.eligibleRouteCandidates : [prepared.candidate];
   const attemptCandidates = candidates;
@@ -224,6 +246,9 @@ export async function handleProviderProxyRequest(
         headers: inboundHeaders,
         requestId: state.requestId,
         requestIdentity: state,
+        ...(state.imageClampMaxBytes === undefined
+          ? {}
+          : { imageClampMaxBytes: state.imageClampMaxBytes }),
         ...(conversationAffinity ? { conversationAffinity } : {}),
         ...(candidate.user_agent === undefined ? {} : { userAgent: candidate.user_agent }),
         ...(deps.networkBindingFactory
@@ -239,6 +264,13 @@ export async function handleProviderProxyRequest(
                   prepared.authorization.snapshot.tenant_id,
                   state,
                 ),
+                providerCaptureMaxBytes(
+                  await resolveCaptureDepthForRequest(
+                    deps.db,
+                    prepared.authorization.snapshot.tenant_id,
+                    state,
+                  ),
+                ),
               ),
             }
           : {}),

package/src/transport/dispatch/ratelimit-capture.ts

@@ -0,0 +1,37 @@
+// Files a provider's rate-limit headers against the account that earned them.
+//
+// Providers answer real traffic with the bucket they are metering — Grok's CLI
+// plane returns `x-ratelimit-*` on every response — and that is a reading the
+// gateway already paid for. Capturing it here is what lets the quota surface
+// show a balance without spending a request of its own.
+//
+// Kept provider-scoped and fail-open: a provider with no bucket parser is
+// skipped, and any throw is swallowed, because a bookkeeping miss must never
+// take down a request that already succeeded.
+
+import { rememberGrokRateLimit } from "../../providers/integrations/grok/grok-ratelimit-store";
+import { readGrokRateLimitHeaders } from "../../providers/integrations/grok/grok-ratelimit";
+
+/** Providers whose response headers carry a readable bucket. */
+const GROK_PROVIDER_IDS: readonly string[] = ["grok"];
+
+/**
+ * Stores whatever bucket the response carried, keyed by account.
+ *
+ * No account id means no home for the reading: a bucket filed under the wrong
+ * account is worse than none, since it would report another account's balance.
+ */
+export function captureProviderRateLimit(
+  providerId: string,
+  accountId: string | undefined,
+  headers: { get(name: string): string | null },
+): void {
+  if (!accountId) return;
+  const id = providerId.toLowerCase();
+  if (!GROK_PROVIDER_IDS.includes(id)) return;
+  try {
+    rememberGrokRateLimit(accountId, readGrokRateLimitHeaders(headers));
+  } catch {
+    // Swallowed: advisory bookkeeping on a request that already succeeded.
+  }
+}

package/src/transport/dispatch/streaming/stream.ts

@@ -43,7 +43,13 @@ import type { AdmissionLease } from "../../../security/admission/contracts";
 import type { NetworkPoolSelector } from "../../../network/pool/selector";
 import type { TelemetryBatchBuffer } from "../../../observability/telemetry-buffer";
 import type { ProviderExchangeCapture } from "../attempt-finalize";
-import { captureProviderExchange, shouldCaptureProviderBodyForRequest } from "../attempt-finalize";
+import {
+  captureProviderExchange,
+  providerCaptureMaxBytes,
+  resolveCaptureDepthForRequest,
+  shouldCaptureProviderBodyForRequest,
+} from "../attempt-finalize";
+import { resolveTenantImageClampMaxBytes } from "../../request/tenant-preferences";
 import { proxySuccessHeaders, buildUpstreamDispatchContext } from "../upstream";
 import type { AttemptLeaseOwnership } from "../leases";
 import type { ProviderContinuationScope } from "../../../providers/provider-contracts";
@@ -165,13 +171,32 @@ export async function dispatchStreamingAttempt(input: StreamingDispatchInput): P
     networkPoolId,
     prepared.authorization.snapshot.tenant_id,
   );
+  const captureBody = outboundFetch
+    ? await shouldCaptureProviderBodyForRequest(deps.db, prepared.authorization.snapshot.tenant_id, state)
+    : false;
+  const captureDepth = await resolveCaptureDepthForRequest(
+    deps.db,
+    prepared.authorization.snapshot.tenant_id,
+    state,
+  );
   const wrappedOutboundFetch = outboundFetch
     ? captureProviderExchange(
         outboundFetch,
         providerCapture,
-        await shouldCaptureProviderBodyForRequest(deps.db, prepared.authorization.snapshot.tenant_id, state),
+        captureBody,
+        providerCaptureMaxBytes(captureDepth),
       )
     : undefined;
+  // Same memoized ceiling the non-streaming path resolves, read before the
+  // dispatch context is built so a stream gets the same limit as a normal
+  // request rather than silently falling back to the deployment default.
+  if (state.imageClampMaxBytes === undefined) {
+    const clampMaxBytes = await resolveTenantImageClampMaxBytes(
+      deps.db,
+      prepared.authorization.snapshot.tenant_id,
+    );
+    if (clampMaxBytes !== undefined) state.imageClampMaxBytes = clampMaxBytes;
+  }
   const dispatchContext = buildUpstreamDispatchContext({
     credential,
     deadline: state.deadlineMs,
@@ -179,6 +204,9 @@ export async function dispatchStreamingAttempt(input: StreamingDispatchInput): P
     headers: inboundHeaders,
     requestId: state.requestId,
     requestIdentity: state,
+    ...(state.imageClampMaxBytes === undefined
+      ? {}
+      : { imageClampMaxBytes: state.imageClampMaxBytes }),
     ...(conversationAffinity ? { conversationAffinity } : {}),
     ...(candidate.user_agent === undefined ? {} : { userAgent: candidate.user_agent }),
     ...(wrappedOutboundFetch ? { outboundFetch: wrappedOutboundFetch } : {}),

package/src/transport/dispatch/upstream.ts

@@ -56,6 +56,12 @@ export function buildUpstreamDispatchContext(input: {
   readonly continuationScope?: ProviderContinuationScope;
   /** Stable per-conversation cache affinity (preferred over random ids). */
   readonly conversationAffinity?: string;
+  /**
+   * Per-tenant inline-attachment ceiling in bytes, resolved once per request.
+   * Carried through dispatch because wire encoders clamp images but hold no
+   * database handle.
+   */
+  readonly imageClampMaxBytes?: number;
   /** Opaque identity shared by retries of one logical gateway request. */
   readonly requestIdentity?: object;
 }): Record<string, unknown> {
@@ -94,6 +100,9 @@ export function buildUpstreamDispatchContext(input: {
     ...(input.leaseOwnership ? { lease_ownership: input.leaseOwnership } : {}),
     ...(input.continuationScope ? { continuation_scope: input.continuationScope } : {}),
     ...(input.conversationAffinity ? { conversation_affinity: input.conversationAffinity } : {}),
+    ...(input.imageClampMaxBytes === undefined
+      ? {}
+      : { image_clamp_max_bytes: input.imageClampMaxBytes }),
     ...(input.requestIdentity ? { request_identity: input.requestIdentity } : {}),
   } as Record<string, unknown>;
 }

package/src/transport/request/preparer.ts

@@ -85,12 +85,13 @@ export function resolveContinuationAffinity(
     // id replayed from another key or tenant resolves nothing.
     const record = lookupPendingCall(callId, scope);
     if (record === undefined) {
-      throw new GatewayError(
-        "invalid_request",
-        400,
-        "This tool result refers to a Cursor tool call that is no longer pending. Resend the complete conversation.",
-        { callId },
-      );
+      // A replayed conversation carries every earlier tool result, and results
+      // previous legs already answered resolve nothing here — they are history,
+      // not the flight being continued. The request is a continuation only
+      // through the ids that still resolve; when none do, the ordinary routing
+      // path decides (a Cursor adapter may rebase the full history into a
+      // fresh Run).
+      continue;
     }
     if (affinity !== undefined && affinity.providerAccountId !== record.providerAccountId) {
       throw new GatewayError(

package/src/transport/request/state.ts

@@ -117,6 +117,15 @@ export interface ProxyRequestState {
   payloadCapturePreference?:
     | { readonly ok: true; readonly mode: TelemetryPayloadMode; readonly depth: TelemetryPayloadDepth }
     | { readonly ok: false };
+  /**
+   * This request's resolved inline-attachment ceiling, memoized for the same
+   * reason the capture preference is: it is a tenant setting, so it cannot
+   * change between attempts, and reading it per attempt would pay for it
+   * several times on a request that fails over.
+   *
+   * `undefined` means the deployment default.
+   */
+  imageClampMaxBytes?: number;
   /**
    * Set by the dispatch handler synchronously before returning a streaming
    * response. `afterResponse` lifecycle hooks read it to skip finalization

package/src/transport/request/tenant-preferences.ts

@@ -13,6 +13,66 @@ import type { PreparedProxyRequest } from "./preparer";
 import { normalizeThinkingConfig } from "../translation/thinking";
 import { compressRequest } from "./rtk/compress-request";
 import { preferencesReaderFor } from "../../persistence/tenant-preferences";
+import { resolveUpstreamTimeoutMs } from "../../config/timeouts";
+
+/**
+ * Re-arms the request deadline from the tenant's timeout override.
+ *
+ * The initial deadline is armed before authentication, when the tenant is not
+ * yet known, so a per-tenant timeout can only be applied here — once the
+ * snapshot says who is asking. `extendDeadline` keeps the timer, the state
+ * field, and the in-flight record in step, so the backstop sweep cannot
+ * force-release a request whose budget legitimately grew.
+ *
+ * Non-fatal: a settings read outage leaves the deployment default in place
+ * rather than failing a dispatchable request.
+ */
+export async function applyTenantDeadlineOverride(
+  tenantId: string,
+  db: CartethyiaDatabase,
+  extend: (ms: number) => void,
+): Promise<void> {
+  try {
+    const prefs = await preferencesReaderFor(db).readPreferences(tenantId);
+    const seconds = prefs?.upstreamTimeoutSeconds;
+    if (typeof seconds !== "number" || !Number.isFinite(seconds) || seconds <= 0) return;
+    const wanted = Math.floor(seconds * 1000);
+    // Only ever shorten or leave alone a deadline that has already begun: an
+    // override longer than the default cannot give back time the request has
+    // already spent, and arming a longer one would silently extend a budget
+    // the caller's own client has likely abandoned.
+    const remaining = Math.min(wanted, resolveUpstreamTimeoutMs());
+    extend(remaining);
+  } catch {
+    // Non-fatal
+  }
+}
+
+/**
+ * The tenant's inline-attachment byte ceiling, or `undefined` for the default.
+ *
+ * Read through the shared revision-keyed preference reader, so the cost is a
+ * memory hit rather than a Postgres round trip — this is consulted while
+ * building a request body, on the dispatch path.
+ *
+ * Non-fatal: a settings read outage falls back to the deployment default,
+ * which is the safe direction (the default refuses less, but it never admits
+ * something no upstream will take).
+ */
+export async function resolveTenantImageClampMaxBytes(
+  db: CartethyiaDatabase,
+  tenantId: string | undefined,
+): Promise<number | undefined> {
+  if (tenantId === undefined) return undefined;
+  try {
+    const prefs = await preferencesReaderFor(db).readPreferences(tenantId);
+    const bytes = prefs?.imageClampMaxBytes;
+    if (typeof bytes !== "number" || !Number.isFinite(bytes) || bytes <= 0) return undefined;
+    return Math.floor(bytes);
+  } catch {
+    return undefined;
+  }
+}
 
 /**
  * Applies request shaping owned by tenant preferences and deployment config: