@remodex/rmx 1.0.2

package/src/lib/translator-budget.ts

@@ -0,0 +1,366 @@
+export const TRANSLATOR_MAX_CALL_ARGUMENT_BYTES = 2 * 1024 * 1024;
+export const TRANSLATOR_MAX_TURN_BYTES = 32 * 1024 * 1024;
+export const TRANSLATOR_MAX_SSE_EVENT_BYTES = 32 * 1024 * 1024;
+export const CURSOR_MAX_CONNECT_FRAME_BYTES = 32 * 1024 * 1024;
+export const CURSOR_MAX_EFFECTIVE_CONNECT_PAYLOAD_BYTES = 16 * 1024 * 1024;
+export const CURSOR_TRANSPORT_MAX_BUFFERED_BYTES = 32 * 1024 * 1024;
+export const CURSOR_TRANSPORT_RESUME_BYTES = 16 * 1024 * 1024;
+export const CURSOR_MAX_PENDING_FRAMES = 1024;
+export const CURSOR_PENDING_FRAMES_RESUME = 512;
+export const MAX_PENDING_IMAGE_FULFILLMENTS = 64;
+export const MAX_PENDING_OAUTH_MUTATIONS = 128;
+
+export type TranslatorBufferKind =
+  | "tool_args"
+  | "retained_collectors"
+  | "live_transient"
+  | "reasoning"
+  | "item_ids"
+  | "tool_search_sources"
+  | "cursor_transport"
+  | "cursor_kv"
+  | "request_copies";
+
+export type ExternallyCappedKind = "passthrough_serialization" | "mcp_payload";
+
+export class TranslatorBudgetExceededError extends Error {
+  readonly code = "translation_buffer_limit";
+
+  constructor(readonly kind: TranslatorBufferKind, readonly limitBytes: number) {
+    super(`translator ${kind} buffer exceeded ${limitBytes} bytes`);
+    this.name = "TranslatorBudgetExceededError";
+  }
+}
+
+export function isTranslatorBudgetExceededError(error: unknown): error is TranslatorBudgetExceededError {
+  return error instanceof TranslatorBudgetExceededError
+    || (error instanceof Error && (error as { code?: unknown }).code === "translation_buffer_limit");
+}
+
+export interface TranslatorBudgetSnapshot {
+  currentBytes: number;
+  highWaterBytes: number;
+  activeCalls: number;
+  overflows: number;
+}
+
+export interface TranslatorBudgetOptions {
+  maxCallArgumentBytes?: number;
+  maxTurnBytes?: number;
+}
+
+export interface TranslatorTransientReservation {
+  commitRetained(): void;
+  release(): void;
+}
+
+export interface TranslatorBudget {
+  openCall(id: string): void;
+  closeCall(id: string): void;
+  reserveTransient(
+    bytes: number,
+    scope: { kind: TranslatorBufferKind; callId?: string },
+  ): TranslatorTransientReservation;
+  chargeRetained(delta: number, scope: { kind: TranslatorBufferKind; callId?: string }): void;
+  releaseRetained(bytes: number, scope: { kind: TranslatorBufferKind; callId?: string }): void;
+  observeAcceptedRequestCopy(bytes: number): () => void;
+  observeExternallyCapped(kind: ExternallyCappedKind, bytes: number): () => void;
+  snapshot(): TranslatorBudgetSnapshot;
+  dispose(): void;
+}
+
+const retainedEventOwnership = new WeakMap<object, { budget: TranslatorBudget; bytes: number }>();
+
+/**
+ * Charge a materialized adapter-event batch and attach its lease to the events themselves.
+ * A copied event array (for example terminal-guard collection) preserves the event objects, so
+ * the response builder can consume each source lease immediately after its replacement lands.
+ */
+export function retainTranslatedEventBatch<T extends object>(events: T[], budget: TranslatorBudget): void {
+  if (events.length === 0) return;
+  const serialized = events.map(event => JSON.stringify(event));
+  const totalBytes = Buffer.byteLength(`[${serialized.join(",")}]`);
+  budget.chargeRetained(totalBytes, { kind: "retained_collectors" });
+  for (let index = 0; index < events.length; index++) {
+    const delimiterBytes = index === events.length - 1 ? 2 : 1;
+    retainedEventOwnership.set(events[index]!, {
+      budget,
+      bytes: Buffer.byteLength(serialized[index]!) + delimiterBytes,
+    });
+  }
+}
+
+/** Transfer one charged adapter event to the response builder's newly materialized owner. */
+export function releaseTranslatedEvent(event: object, budget: TranslatorBudget): void {
+  const ownership = retainedEventOwnership.get(event);
+  if (!ownership || ownership.budget !== budget) return;
+  retainedEventOwnership.delete(event);
+  budget.releaseRetained(ownership.bytes, { kind: "retained_collectors" });
+}
+
+const liveBudgets = new Set<Budget>();
+let aggregateCurrentBytes = 0;
+let aggregateActiveCalls = 0;
+let aggregateHighWaterBytes = 0;
+let aggregateOverflows = 0;
+
+function normalizeBytes(bytes: number): number {
+  return Number.isFinite(bytes) && bytes > 0 ? Math.floor(bytes) : 0;
+}
+
+function observeAggregate(): void {
+  aggregateHighWaterBytes = Math.max(aggregateHighWaterBytes, aggregateCurrentBytes);
+}
+
+class Budget implements TranslatorBudget {
+  private readonly calls = new Map<string, { logicalBytes: number; physicalBytes: number }>();
+  private readonly charged = new Map<TranslatorBufferKind, number>();
+  private hardChargedBytes = 0;
+  private observedBytes = 0;
+  private currentBytes = 0;
+  private highWaterBytes = 0;
+  private overflowCount = 0;
+  private disposed = false;
+
+  constructor(
+    private readonly maxCallArgumentBytes: number,
+    private readonly maxTurnBytes: number,
+  ) {}
+
+  private chargedBytes(): number {
+    return this.hardChargedBytes;
+  }
+
+  observedCurrentBytes(): number {
+    return this.currentBytes;
+  }
+
+  private changed(): void {
+    const nextBytes = this.hardChargedBytes + this.observedBytes;
+    aggregateCurrentBytes += nextBytes - this.currentBytes;
+    this.currentBytes = nextBytes;
+    this.highWaterBytes = Math.max(this.highWaterBytes, nextBytes);
+    observeAggregate();
+  }
+
+  private reject(kind: TranslatorBufferKind, limitBytes: number): never {
+    this.overflowCount += 1;
+    aggregateOverflows += 1;
+    throw new TranslatorBudgetExceededError(kind, limitBytes);
+  }
+
+  private admit(kind: TranslatorBufferKind, bytes: number): number {
+    const admitted = normalizeBytes(bytes);
+    if (this.chargedBytes() + admitted > this.maxTurnBytes) this.reject(kind, this.maxTurnBytes);
+    return admitted;
+  }
+
+  openCall(id: string): void {
+    if (!this.calls.has(id)) {
+      this.calls.set(id, { logicalBytes: 0, physicalBytes: 0 });
+      aggregateActiveCalls += 1;
+    }
+  }
+
+  closeCall(id: string): void {
+    const call = this.calls.get(id);
+    if (!call) return;
+    this.hardChargedBytes = Math.max(0, this.hardChargedBytes - call.physicalBytes);
+    this.calls.delete(id);
+    aggregateActiveCalls = Math.max(0, aggregateActiveCalls - 1);
+    this.changed();
+  }
+
+  reserveTransient(
+    bytes: number,
+    scope: { kind: TranslatorBufferKind; callId?: string },
+  ): TranslatorTransientReservation {
+    const admitted = normalizeBytes(bytes);
+    if (scope.callId && admitted > this.maxCallArgumentBytes) {
+      this.reject("tool_args", this.maxCallArgumentBytes);
+    }
+    this.admit(scope.kind, admitted);
+    this.charged.set(scope.kind, (this.charged.get(scope.kind) ?? 0) + admitted);
+    this.hardChargedBytes += admitted;
+    this.changed();
+    let state: "reserved" | "committed" | "released" = "reserved";
+    return {
+      commitRetained: () => {
+        if (state !== "reserved") return;
+        state = "committed";
+        if (!scope.callId) return;
+        const call = this.calls.get(scope.callId) ?? { logicalBytes: 0, physicalBytes: 0 };
+        this.charged.set(scope.kind, Math.max(0, (this.charged.get(scope.kind) ?? 0) - admitted));
+        call.logicalBytes = admitted;
+        call.physicalBytes += admitted;
+        this.calls.set(scope.callId, call);
+        this.changed();
+      },
+      release: () => {
+        if (state !== "reserved") return;
+        state = "released";
+        this.charged.set(scope.kind, Math.max(0, (this.charged.get(scope.kind) ?? 0) - admitted));
+        this.hardChargedBytes = Math.max(0, this.hardChargedBytes - admitted);
+        this.changed();
+      },
+    };
+  }
+
+  chargeRetained(delta: number, scope: { kind: TranslatorBufferKind; callId?: string }): void {
+    const admitted = normalizeBytes(delta);
+    if (scope.callId) {
+      const call = this.calls.get(scope.callId) ?? { logicalBytes: 0, physicalBytes: 0 };
+      if (call.logicalBytes + admitted > this.maxCallArgumentBytes) {
+        this.reject("tool_args", this.maxCallArgumentBytes);
+      }
+      this.admit(scope.kind, admitted);
+      call.logicalBytes += admitted;
+      call.physicalBytes += admitted;
+      this.calls.set(scope.callId, call);
+    } else {
+      this.admit(scope.kind, admitted);
+      this.charged.set(scope.kind, (this.charged.get(scope.kind) ?? 0) + admitted);
+    }
+    this.hardChargedBytes += admitted;
+    this.changed();
+  }
+
+  releaseRetained(bytes: number, scope: { kind: TranslatorBufferKind; callId?: string }): void {
+    const released = normalizeBytes(bytes);
+    if (scope.callId) {
+      const call = this.calls.get(scope.callId);
+      if (call) {
+        const actual = Math.min(released, call.physicalBytes);
+        call.physicalBytes -= actual;
+        this.hardChargedBytes = Math.max(0, this.hardChargedBytes - actual);
+      }
+    } else {
+      const current = this.charged.get(scope.kind) ?? 0;
+      const actual = Math.min(released, current);
+      this.charged.set(scope.kind, current - actual);
+      this.hardChargedBytes = Math.max(0, this.hardChargedBytes - actual);
+    }
+    this.changed();
+  }
+
+  private observe(bytes: number): () => void {
+    const observed = normalizeBytes(bytes);
+    this.observedBytes += observed;
+    this.changed();
+    let released = false;
+    return () => {
+      if (released) return;
+      released = true;
+      this.observedBytes = Math.max(0, this.observedBytes - observed);
+      this.changed();
+    };
+  }
+
+  observeAcceptedRequestCopy(bytes: number): () => void {
+    return this.observe(bytes);
+  }
+
+  observeExternallyCapped(_kind: ExternallyCappedKind, bytes: number): () => void {
+    return this.observe(bytes);
+  }
+
+  snapshot(): TranslatorBudgetSnapshot {
+    return {
+      currentBytes: this.observedCurrentBytes(),
+      highWaterBytes: this.highWaterBytes,
+      activeCalls: this.calls.size,
+      overflows: this.overflowCount,
+    };
+  }
+
+  dispose(): void {
+    if (this.disposed) return;
+    this.disposed = true;
+    aggregateActiveCalls = Math.max(0, aggregateActiveCalls - this.calls.size);
+    aggregateCurrentBytes = Math.max(0, aggregateCurrentBytes - this.currentBytes);
+    this.calls.clear();
+    this.charged.clear();
+    this.hardChargedBytes = 0;
+    this.observedBytes = 0;
+    this.currentBytes = 0;
+    liveBudgets.delete(this);
+    observeAggregate();
+  }
+}
+
+export function createTranslatorBudget(options: TranslatorBudgetOptions = {}): TranslatorBudget {
+  const budget = new Budget(
+    options.maxCallArgumentBytes ?? TRANSLATOR_MAX_CALL_ARGUMENT_BYTES,
+    options.maxTurnBytes ?? TRANSLATOR_MAX_TURN_BYTES,
+  );
+  liveBudgets.add(budget);
+  observeAggregate();
+  return budget;
+}
+
+export function translatorObservedBufferSnapshot(): {
+  currentBytes: number;
+  highWaterBytes: number;
+  active: number;
+} {
+  return {
+    currentBytes: aggregateCurrentBytes,
+    highWaterBytes: aggregateHighWaterBytes,
+    active: aggregateActiveCalls,
+  };
+}
+
+/** Internal diagnostics used by focused tests; never registered with app-owned memory. */
+export function translatorObservedOverflowCount(): number {
+  return aggregateOverflows;
+}
+
+/** Test-only: proves owned default budgets are disposed on every stream-death path. */
+export function translatorLiveBudgetCountForTests(): number {
+  return liveBudgets.size;
+}
+
+/** Test-only: proves no charge survives against a disposed budget (cancel race). */
+export function translatorAggregateCurrentBytesForTests(): number {
+  return aggregateCurrentBytes;
+}
+
+/** Clears process-wide translator diagnostics and disposes leaked test budgets. */
+export function resetTranslatorAggregateForTests(): void {
+  for (const budget of liveBudgets) budget.dispose();
+  aggregateCurrentBytes = 0;
+  aggregateActiveCalls = 0;
+  aggregateHighWaterBytes = 0;
+  aggregateOverflows = 0;
+}
+
+export function finalizeTranslatorBudgetResponse(response: Response, budget: TranslatorBudget): Response {
+  if (!response.body) {
+    budget.dispose();
+    return response;
+  }
+  const reader = response.body.getReader();
+  let finalized = false;
+  const finalize = () => {
+    if (finalized) return;
+    finalized = true;
+    budget.dispose();
+  };
+  return new Response(new ReadableStream<Uint8Array>({
+    async pull(controller) {
+      try {
+        const result = await reader.read();
+        if (result.done) {
+          finalize();
+          controller.close();
+        } else controller.enqueue(result.value);
+      } catch (error) {
+        finalize();
+        controller.error(error);
+      }
+    },
+    async cancel(reason) {
+      try { await reader.cancel(reason); } finally { finalize(); }
+    },
+  }), { status: response.status, statusText: response.statusText, headers: response.headers });
+}

package/src/lib/upstream-reachability.ts

@@ -0,0 +1,91 @@
+/**
+ * Classification of upstream fetch rejections that occur before any request
+ * bytes can reach the origin: DNS resolution failure and TCP connect refusal.
+ *
+ * Issue #914: these failures are machine/network-wide, not account-specific —
+ * every Codex pool account shares the provider host, so rotating accounts
+ * cannot repair them. The transport layer previously mapped every non-timeout
+ * rejection to `connect_error`, and at `upstreamFailoverThreshold` that streak
+ * soft-avoided a healthy account and cleared thread affinity.
+ *
+ * Bun collapses DNS failure and TCP refusal into one label class
+ * (`ConnectionRefused` / `FailedToOpenSocket`, errno 0); Node's undici emits
+ * the classic `ECONNREFUSED` / `ENOTFOUND` / `EAI_AGAIN` / `ENETUNREACH` /
+ * `ENETDOWN` / `EHOSTUNREACH` shapes. Both shapes are matched here, on `code`
+ * values only, through a bounded cause chain. Message substrings are never
+ * trusted (a message-only match is a negative case), and labels that can also
+ * appear after the origin saw the credential — `ECONNRESET`, `EPIPE`, TLS
+ * errors, unknown shapes — deliberately stay outside the set.
+ *
+ * Classifier semantics extracted from PR #966 (Yuxin-Qiao) with attribution;
+ * sidecar blast radius intentionally not inherited.
+ *
+ * MUST stay a leaf module: imports nothing from server.ts or adapters.
+ */
+
+import { UpstreamRetryEvidenceError } from "./upstream-retry";
+
+export const PRE_CONNECT_REACHABILITY_CODES = new Set([
+  // Bun: DNS failure and TCP refusal share this class.
+  "ConnectionRefused",
+  "FailedToOpenSocket",
+  // Node undici / classic Node shapes.
+  "ECONNREFUSED",
+  "ENOTFOUND",
+  "EAI_AGAIN",
+  "ENETUNREACH",
+  "ENETDOWN",
+  "EHOSTUNREACH",
+]);
+
+/** Upper bound on how far `cause` chains are inspected. */
+export const MAX_REACHABILITY_CAUSE_DEPTH = 3;
+
+/**
+ * True when the rejection (or a bounded `cause` of it) carries a proven
+ * pre-connection reachability code. Never matches message text.
+ */
+export function isPreConnectReachabilityError(err: unknown): boolean {
+  let current: unknown = err;
+  const seen = new Set<unknown>();
+  for (let depth = 0; depth < MAX_REACHABILITY_CAUSE_DEPTH; depth++) {
+    if (!(current instanceof Error) || seen.has(current)) return false;
+    seen.add(current);
+    const code = (current as { code?: unknown }).code;
+    if (typeof code === "string" && PRE_CONNECT_REACHABILITY_CODES.has(code)) return true;
+    current = current.cause;
+  }
+  return false;
+}
+
+export type TransportFailureKind = "timeout" | "connect_neutral" | "connect_error";
+
+/**
+ * Shared transport rejection classification for Codex pool upstream sends.
+ * Timeouts keep their existing identity (account-transient); proven pre-connect
+ * reachability failures become account-neutral; everything else (ECONNRESET,
+ * EPIPE, TLS, unknown shapes) keeps the existing `connect_error`
+ * account-attributed behavior.
+ */
+export function classifyTransportFailureKind(err: unknown): TransportFailureKind {
+  const evidence = err instanceof UpstreamRetryEvidenceError ? err : undefined;
+  const rejection = evidence ? evidence.cause : err;
+  if (rejection instanceof Error && rejection.name === "TimeoutError") return "timeout";
+  if (isPreConnectReachabilityError(rejection)) {
+    // A transient upstream response (5xx) or a credential-visible connection
+    // reset before the rejection proves the host and credential path were
+    // reached: the failure is account-attributable, never the pre-connection
+    // neutral class (issue #914 review).
+    if (evidence && (evidence.transientStatuses.length > 0 || evidence.resetSeen)) return "connect_error";
+    return "connect_neutral";
+  }
+  return "connect_error";
+}
+
+/** Stable `code` carried by a transport rejection, when there is one. */
+export function transportErrorCode(err: unknown): string | undefined {
+  const rejection = err instanceof UpstreamRetryEvidenceError ? err.cause : err;
+  if (!(rejection instanceof Error)) return undefined;
+  const code = (rejection as { code?: unknown }).code;
+  return typeof code === "string" && code !== "" ? code : undefined;
+}