cartethyia 2.7.4 → 2.7.6

package/src/network/pool/selector.ts

@@ -36,8 +36,7 @@ import {
   type ProxyCooldownEntry,
 } from "./pool-state";
 
-export type { PoolRotation, PoolUsageSnapshot, ProxyCooldownEntry };
-export { parseCooldownEntry };
+export type { PoolRotation, ProxyCooldownEntry };
 
 // Pool selector
 /**
@@ -51,7 +50,7 @@ export { parseCooldownEntry };
  */
 export const PROVIDER_BLOCK_MS = 10 * 365 * 24 * 60 * 60 * 1000;
 
-export const DEFAULT_PROXY_CONCURRENCY = 10;
+const DEFAULT_PROXY_CONCURRENCY = 10;
 const DEFAULT_PROXY_WEIGHT = 100;
 const MAX_PROXY_WEIGHT = 1000;
 // Crash-recovery bound, not a lease duration: normal releases DECR/DEL the
@@ -63,7 +62,7 @@ const MAX_PROXY_WEIGHT = 1000;
 function poolInflightTtlSeconds(): number {
   return resolveInflightTtlSeconds();
 }
-export function effectiveConcurrency(limit: number): number {
+function effectiveConcurrency(limit: number): number {
   if (!Number.isInteger(limit) || limit < 1) return 0;
   return limit;
 }
@@ -100,7 +99,7 @@ function decrementLocal(counter: Map<string, number>, key: string): void {
   else counter.set(key, value - 1);
 }
 
-export type PoolSelectionFailureReason =
+type PoolSelectionFailureReason =
   | "at_capacity"
   | "provider_at_capacity"
   | "cooldown"
@@ -109,7 +108,7 @@ export type PoolSelectionFailureReason =
   | "coordination_unavailable"
   | "no_active_pool";
 
-export interface PoolCapacitySnapshot {
+interface PoolCapacitySnapshot {
   readonly poolId: string;
   readonly maxInflight: number;
   readonly currentInflight: number;
@@ -206,67 +205,67 @@ export class NetworkPoolSelector {
   }
 
   /**
-   * Raise this provider's share of a pool, enforcing both the pool ceiling and
-   * the provider ceiling. The pool half is still checked here so a caller that
-   * already admitted the pool through `acquire` does not double-count it — use
-   * this instead of `acquire` whenever a provider ceiling applies.
+   * One local slot acquisition: checks every map's ceiling, bumps each counter,
+   * and returns a release that decrements all of them exactly once. Shared by
+   * `acquire` (single pool counter) and `acquireProvider` (pool + provider
+   * share) so the release closure and the local bookkeeping live in one place.
    */
-  acquireProvider(
-    poolId: string,
-    providerId: string,
-    poolCapacity: number,
-    providerCapacity: number,
+  private acquireLocalSlot(
+    increments: ReadonlyArray<{
+      readonly map: Map<string, number>;
+      readonly key: string;
+      readonly capacity: number;
+    }>,
   ): { acquired: boolean; release: () => void } {
-    const current = this.getInflight(poolId);
-    if (current >= poolCapacity) return { acquired: false, release: () => {} };
-    const key = providerInflightStateKey(poolId, providerId);
-    const providerCurrent = this.state.providerInflight.get(key) ?? 0;
-    if (providerCurrent >= providerCapacity) return { acquired: false, release: () => {} };
-
-    enforceInflightLimit(this.state.inflight);
-    enforceInflightLimit(this.state.providerInflight);
-    this.state.inflight.set(poolId, current + 1);
-    this.state.providerInflight.set(key, providerCurrent + 1);
+    for (const increment of increments) {
+      const current = increment.map.get(increment.key) ?? 0;
+      if (current >= increment.capacity) return { acquired: false, release: () => {} };
+    }
+    for (const increment of increments) {
+      enforceInflightLimit(increment.map);
+      increment.map.set(increment.key, (increment.map.get(increment.key) ?? 0) + 1);
+    }
     this.notifyPoolUsage();
 
     let released = false;
     const release = () => {
       if (released) return;
       released = true;
-      decrementLocal(this.state.inflight, poolId);
-      decrementLocal(this.state.providerInflight, key);
+      for (const increment of increments) decrementLocal(increment.map, increment.key);
       this.notifyPoolUsage();
     };
     return { acquired: true, release };
   }
 
+  /**
+   * Raise this provider's share of a pool, enforcing both the pool ceiling and
+   * the provider ceiling. The pool half is still checked here so a caller that
+   * already admitted the pool through `acquire` does not double-count it — use
+   * this instead of `acquire` whenever a provider ceiling applies.
+   */
+  acquireProvider(
+    poolId: string,
+    providerId: string,
+    poolCapacity: number,
+    providerCapacity: number,
+  ): { acquired: boolean; release: () => void } {
+    return this.acquireLocalSlot([
+      { map: this.state.inflight, key: poolId, capacity: poolCapacity },
+      {
+        map: this.state.providerInflight,
+        key: providerInflightStateKey(poolId, providerId),
+        capacity: providerCapacity,
+      },
+    ]);
+  }
+
   acquire(
     poolId: string,
     maxInflight = DEFAULT_PROXY_CONCURRENCY,
   ): { acquired: boolean; release: () => void } {
-    const current = this.getInflight(poolId);
-    if (current >= maxInflight) {
-      return { acquired: false, release: () => {} };
-    }
-
-    enforceInflightLimit(this.state.inflight);
-    this.state.inflight.set(poolId, current + 1);
-    this.notifyPoolUsage();
-    let released = false;
-
-    const release = () => {
-      if (released) return;
-      released = true;
-      const val = this.state.inflight.get(poolId) ?? 1;
-      if (val <= 1) {
-        this.state.inflight.delete(poolId);
-      } else {
-        this.state.inflight.set(poolId, val - 1);
-      }
-      this.notifyPoolUsage();
-    };
-
-    return { acquired: true, release };
+    return this.acquireLocalSlot([
+      { map: this.state.inflight, key: poolId, capacity: maxInflight },
+    ]);
   }
 
   private cooldownKey(poolId: string, providerId: string): string {
@@ -577,6 +576,98 @@ export class NetworkPoolSelector {
     await this.redis.del(indexKey);
   }
 
+  /**
+   * Tries one local candidate: bump the local counters (pool, and provider
+   * share when a ceiling is set) and return its release. `undefined` when the
+   * candidate is full or the provider share is exhausted.
+   */
+  private localAcquire(
+    candidate: { readonly poolId: string; readonly capacity: number },
+    providerId: string,
+    providerCapacity: number | undefined,
+  ): { poolId: string; release: () => void } | undefined {
+    const slot =
+      providerCapacity === undefined
+        ? this.acquire(candidate.poolId, candidate.capacity)
+        : this.acquireProvider(candidate.poolId, providerId, candidate.capacity, providerCapacity);
+    return slot.acquired ? { poolId: candidate.poolId, release: slot.release } : undefined;
+  }
+
+  /**
+   * One atomic Redis admit (the compare-and-increment script) for a candidate:
+   * bumps both the remote and local counters on success and returns a release
+   * that decrements both. `undefined` when the pool or provider share is full
+   * (1 admitted; 0 pool full; 2 provider full — either full result fails over
+   * to the next pool, which is the whole point of the ceiling).
+   */
+  private async redisAdmit(
+    redis: RedisClient,
+    candidate: { readonly poolId: string; readonly capacity: number },
+    providerId: string,
+    providerCapacity: number | undefined,
+  ): Promise<{ poolId: string; release: () => void } | undefined> {
+    const providerKey =
+      providerCapacity === undefined
+        ? undefined
+        : proxyProviderInflightKey(candidate.poolId, providerId);
+    const result =
+      providerKey === undefined
+        ? await redisEvalNumber(
+            redis,
+            POOL_ADMIT_SCRIPT,
+            1,
+            proxyInflightKey(candidate.poolId),
+            String(candidate.capacity),
+            String(poolInflightTtlSeconds()),
+          )
+        : await redisEvalNumber(
+            redis,
+            POOL_PROVIDER_ADMIT_SCRIPT,
+            2,
+            proxyInflightKey(candidate.poolId),
+            providerKey,
+            String(candidate.capacity),
+            String(providerCapacity),
+            String(poolInflightTtlSeconds()),
+          );
+    if (result !== 1) return undefined;
+    enforceInflightLimit(this.state.inflight);
+    this.state.inflight.set(candidate.poolId, this.getInflight(candidate.poolId) + 1);
+    if (providerKey !== undefined) {
+      enforceInflightLimit(this.state.providerInflight);
+      const localProviderKey = providerInflightStateKey(candidate.poolId, providerId);
+      this.state.providerInflight.set(
+        localProviderKey,
+        this.getProviderInflight(candidate.poolId, providerId) + 1,
+      );
+    }
+    this.notifyPoolUsage();
+    let released = false;
+    return {
+      poolId: candidate.poolId,
+      release: () => {
+        if (released) return;
+        released = true;
+        decrementLocal(this.state.inflight, candidate.poolId);
+        if (providerKey !== undefined) {
+          decrementLocal(
+            this.state.providerInflight,
+            providerInflightStateKey(candidate.poolId, providerId),
+          );
+        }
+        this.notifyPoolUsage();
+        const script = providerKey === undefined ? POOL_RELEASE_SCRIPT : POOL_PROVIDER_RELEASE_SCRIPT;
+        const keys =
+          providerKey === undefined
+            ? [proxyInflightKey(candidate.poolId)]
+            : [proxyInflightKey(candidate.poolId), providerKey];
+        void redisEvalNumber(redis, script, keys.length, ...keys).catch((error: unknown) => {
+          log.error("[pool-selector] failed to release distributed slot", error as Error);
+        });
+      },
+    };
+  }
+
   /**
    * Distributed pool admission — the selection algorithm:
    *
@@ -652,17 +743,8 @@ export class NetworkPoolSelector {
         shuffleArray(scored);
         for (const candidate of scored) {
           if (this.getInflight(candidate.poolId) >= candidate.capacity) continue;
-          const slot =
-            providerCapacity === undefined
-              ? this.acquire(candidate.poolId, candidate.capacity)
-              : this.acquireProvider(
-                  candidate.poolId,
-                  providerId,
-                  candidate.capacity,
-                  providerCapacity,
-                );
-          if (!slot.acquired) continue;
-          return { poolId: candidate.poolId, release: slot.release };
+          const slot = this.localAcquire(candidate, providerId, providerCapacity);
+          if (slot) return slot;
         }
         return undefined;
       }
@@ -677,18 +759,10 @@ export class NetworkPoolSelector {
           const candidate = scored[absolute];
           if (!candidate) continue;
           if (this.getInflight(candidate.poolId) >= candidate.capacity) continue;
-          const slot =
-            providerCapacity === undefined
-              ? this.acquire(candidate.poolId, candidate.capacity)
-              : this.acquireProvider(
-                  candidate.poolId,
-                  providerId,
-                  candidate.capacity,
-                  providerCapacity,
-                );
-          if (!slot.acquired) continue;
+          const slot = this.localAcquire(candidate, providerId, providerCapacity);
+          if (!slot) continue;
           advanceRotation(this.state.rotationCursors, rotation.key, scored.length, absolute, rotation.rotateCount);
-          return { poolId: candidate.poolId, release: slot.release };
+          return slot;
         }
         return undefined;
       }
@@ -716,11 +790,7 @@ export class NetworkPoolSelector {
         }
       }
       if (!bestPool) return undefined;
-      const slot =
-        providerCapacity === undefined
-          ? this.acquire(bestPool.poolId, bestPool.capacity)
-          : this.acquireProvider(bestPool.poolId, providerId, bestPool.capacity, providerCapacity);
-      return slot.acquired ? { poolId: bestPool.poolId, release: slot.release } : undefined;
+      return this.localAcquire(bestPool, providerId, providerCapacity);
     }
 
     const redis = this.redis;
@@ -795,34 +865,8 @@ export class NetworkPoolSelector {
       });
     }
     for (const candidate of withCounts) {
-      const providerKey =
-        providerCapacity === undefined
-          ? undefined
-          : proxyProviderInflightKey(candidate.poolId, providerId);
-      const result =
-        providerKey === undefined
-          ? await redisEvalNumber(
-              redis,
-              POOL_ADMIT_SCRIPT,
-              1,
-              proxyInflightKey(candidate.poolId),
-              String(candidate.capacity),
-              String(poolInflightTtlSeconds()),
-            )
-          : await redisEvalNumber(
-              redis,
-              POOL_PROVIDER_ADMIT_SCRIPT,
-              2,
-              proxyInflightKey(candidate.poolId),
-              providerKey,
-              String(candidate.capacity),
-              String(providerCapacity),
-              String(poolInflightTtlSeconds()),
-            );
-      // 1 admitted; 0 pool full; 2 provider full. Either full result fails over
-      // to the next pool, which is the whole point of the ceiling: the request
-      // moves to an egress that still has room for this provider.
-      if (result !== 1) continue;
+      const admitted = await this.redisAdmit(redis, candidate, providerId, providerCapacity);
+      if (admitted === undefined) continue;
       if (rotation && rotation.strategy !== "random") {
         advanceRotation(
           this.state.rotationCursors,
@@ -832,37 +876,7 @@ export class NetworkPoolSelector {
           rotation.rotateCount,
         );
       }
-      enforceInflightLimit(this.state.inflight);
-      this.state.inflight.set(candidate.poolId, this.getInflight(candidate.poolId) + 1);
-      if (providerKey !== undefined) {
-        enforceInflightLimit(this.state.providerInflight);
-        const localProviderKey = providerInflightStateKey(candidate.poolId, providerId);
-        this.state.providerInflight.set(
-          localProviderKey,
-          this.getProviderInflight(candidate.poolId, providerId) + 1,
-        );
-      }
-      this.notifyPoolUsage();
-      let released = false;
-      return {
-        poolId: candidate.poolId,
-        release: () => {
-          if (released) return;
-          released = true;
-          decrementLocal(this.state.inflight, candidate.poolId);
-          if (providerKey !== undefined) {
-            decrementLocal(this.state.providerInflight, providerInflightStateKey(candidate.poolId, providerId));
-          }
-          this.notifyPoolUsage();
-          const script = providerKey === undefined ? POOL_RELEASE_SCRIPT : POOL_PROVIDER_RELEASE_SCRIPT;
-          const keys = providerKey === undefined ? [proxyInflightKey(candidate.poolId)] : [proxyInflightKey(candidate.poolId), providerKey];
-          void redisEvalNumber(redis, script, keys.length, ...keys).catch(
-            (error: unknown) => {
-              log.error("[pool-selector] failed to release distributed slot", error as Error);
-            },
-          );
-        },
-      };
+      return admitted;
     }
     return undefined;
   }

package/src/network/pool-health-machine.ts

@@ -16,7 +16,7 @@ export interface PoolHealthEvent {
   readonly createdAt: string;
 }
 
-export interface PoolHealthSnapshot {
+interface PoolHealthSnapshot {
   readonly poolId: string;
   readonly tenantId: string;
   readonly status: "active" | "cooldown" | "disabled";

package/src/network/rate-limit-verdict.ts

@@ -21,7 +21,7 @@ import { GatewayError } from "../protocol/gateway-error";
 import { providerRateLimitIsIpScoped } from "../providers/provider-metadata";
 
 /** What one upstream failure means for the egress that carried it. */
-export type EgressVerdict =
+type EgressVerdict =
   | { kind: "none" }
   | { kind: "cooldown"; retryAfterMs?: number }
   | { kind: "blocked" };

package/src/network/retry.ts

@@ -9,7 +9,7 @@
 import { exponentialBackoff, type BackoffOptions } from "./backoff";
 import { sleep as defaultSleep } from "../shared/sleep";
 
-export interface RetryOptions {
+interface RetryOptions {
   readonly maxRetries?: number;
   readonly baseDelayMs?: number;
   readonly maxDelayMs?: number;

package/src/network/ssrf.ts

@@ -3,6 +3,12 @@ import { lookup } from "node:dns/promises";
 import { isIP } from "node:net";
 import { GatewayError } from "../protocol/gateway-error";
 import type { SsrfPolicy } from "../config/network";
+import {
+  cidrContainsAddress,
+  ipv4ToNumberStrict,
+  ipv6GroupsToBytes,
+  normalizeIpv6Groups,
+} from "./ip-utils";
 
 const DNS_CACHE_TTL_MS = 60_000;
 const DNS_CACHE_MAX = 64;
@@ -31,87 +37,24 @@ type DnsLookup = (
   options: { all: true },
 ) => Promise<readonly { address: string; family: number }[]>;
 
-function ipv4Number(address: string): number {
-  const parts = address.split(".");
-  if (parts.length !== 4 || parts.some((part) => !/^\d+$/.test(part)))
-    throw new Error("invalid IPv4");
-  const octets = parts.map(Number);
-  if (octets.some((part) => part < 0 || part > 255)) throw new Error("invalid IPv4");
-  return (((octets[0]! << 24) >>> 0) | (octets[1]! << 16) | (octets[2]! << 8) | octets[3]!) >>> 0;
-}
-
 function ipv6Bytes(address: string): Uint8Array {
-  const value = address.toLowerCase().split("%", 1)[0]!;
-  if (value.includes(".")) {
-    const split = value.lastIndexOf(":");
-    if (split < 0) throw new Error("invalid IPv6");
-    const mapped = ipv4Number(value.slice(split + 1));
-    return ipv6Bytes(
-      `${value.slice(0, split)}:${(mapped >>> 16).toString(16)}:${(mapped & 0xffff).toString(16)}`,
-    );
-  }
-  const halves = value.split("::");
-  if (halves.length > 2) throw new Error("invalid IPv6");
-  const left = halves[0] ? halves[0].split(":") : [];
-  const right = halves.length === 2 && halves[1] ? halves[1].split(":") : [];
-  if (halves.length === 1 && left.length !== 8) throw new Error("invalid IPv6");
-  const missing = 8 - left.length - right.length;
-  if (missing < 0 || (halves.length === 2 && missing < 1)) throw new Error("invalid IPv6");
-  const groups = [...left, ...Array.from({ length: missing }, () => "0"), ...right];
-  if (groups.length !== 8 || groups.some((part) => !/^[0-9a-f]{1,4}$/.test(part)))
-    throw new Error("invalid IPv6");
-  const bytes = new Uint8Array(16);
-  groups.forEach((part, index) => {
-    const n = Number.parseInt(part, 16);
-    bytes[index * 2] = n >>> 8;
-    bytes[index * 2 + 1] = n & 255;
-  });
+  const bytes = ipv6GroupsToBytes(normalizeIpv6Groups(address, { allowDottedQuad: true }) ?? []);
+  if (bytes === undefined) throw new Error("invalid IPv6");
   return bytes;
 }
 
 function cidrContains(address: string, cidr: string): boolean {
-  const [network, prefixText] = cidr.split("/");
-  if (!network || prefixText === undefined) return false;
-  // A present-but-blank prefix (`"10.0.0.0/"`) must be refused, not coerced.
-  // `Number("")` is 0 and `""` is not `undefined`, so a plain truthiness check
-  // here would leave `prefix = 0` — and a /0 admits every address of its family.
-  // An operator's typo would therefore silently widen the allowlist to the whole
-  // IPv4 space, including the link-local metadata endpoint this guard exists to
-  // refuse.
-  //
-  // Requiring a run of digits (after trimming) also refuses the exotic spellings
-  // `Number` accepts for zero — `-0`, `+0`, `0x0`, `0b0`, `0o0`, `0.0` — all of
-  // which are typos rather than a deliberate "allow everything". Only a written
-  // `0` means that, which is the next branch's business.
-  const prefixDigits = prefixText.trim();
-  if (!/^\d+$/.test(prefixDigits)) return false;
-  const prefix = Number(prefixDigits);
-  const family = isIP(address);
-  if (
-    family !== isIP(network) ||
-    !Number.isInteger(prefix) ||
-    prefix < 0 ||
-    prefix > (family === 4 ? 32 : 128)
-  )
-    return false;
-  if (family === 4) {
-    const bits = 32 - prefix;
-    return bits === 32 ? true : ipv4Number(address) >>> bits === ipv4Number(network) >>> bits;
-  }
-  const a = ipv6Bytes(address);
-  const n = ipv6Bytes(network);
-  for (let index = 0; index < prefix; index++) {
-    const byte = Math.floor(index / 8);
-    const bit = 7 - (index % 8);
-    if (((a[byte]! >> bit) & 1) !== ((n[byte]! >> bit) & 1)) return false;
-  }
-  return true;
+  return cidrContainsAddress(address, cidr, {
+    requirePrefix: true,
+    unwrapMapped: false,
+    allowDottedQuad: true,
+  });
 }
 
 function unsafeAddress(address: string): boolean {
   const family = isIP(address);
   if (family === 4) {
-    const n = ipv4Number(address);
+    const n = ipv4ToNumberStrict(address);
     return [
       [0, 8],
       [0x0a000000, 8],
@@ -175,9 +118,15 @@ export function validateResolvedAddresses(
   return addresses[0]!;
 }
 
+export async function resolveAllAddresses(hostname: string): Promise<string[]>;
 export async function resolveAllAddresses(
   hostname: string,
   signal: AbortSignal,
+  lookupFn?: DnsLookup,
+): Promise<string[]>;
+export async function resolveAllAddresses(
+  hostname: string,
+  signal: AbortSignal = new AbortController().signal,
   lookupFn: DnsLookup = lookup,
 ): Promise<string[]> {
   if (signal.aborted) throw abortErrorForSignal(signal);

package/src/observability/event-log.ts

@@ -57,6 +57,12 @@ export interface TelemetryEventRecord {
   readonly lastEventAtMs: number | null;
   /** Why the drawer payloads are empty, or `captured`; absent when unrecorded. */
   readonly payloadCaptureState?: string | null;
+  /**
+   * True for a completed request whose output was far smaller than the model's
+   * ceiling and which ended with an ordinary `stop` (a "degenerate stop").
+   * NULL/unset for everything else.
+   */
+  readonly degenerateStop?: boolean;
 }
 
 function asString(value: unknown): string | null {
@@ -109,6 +115,7 @@ function normalize(raw: Record<string, unknown>): TelemetryEventRecord | undefin
     tokensPerSec: asNumber(raw["tokensPerSec"]),
     firstContentDeltaAtMs: asNumber(raw["firstContentDeltaAtMs"]),
     lastEventAtMs: asNumber(raw["lastEventAtMs"]),
+    ...(raw["degenerateStop"] === true ? { degenerateStop: true } : {}),
   };
 }
 
@@ -255,7 +262,7 @@ export async function readTelemetryEvents(): Promise<TelemetryEventRecord[]> {
   return events;
 }
 
-export interface EventQuery {
+interface EventQuery {
   readonly tenantId?: string;
   readonly since?: Date;
   /**
@@ -372,7 +379,7 @@ export async function queryTelemetryEvents(query: EventQuery): Promise<Telemetry
   return events;
 }
 
-export interface EventTotals {
+interface EventTotals {
   requests: number;
   succeeded: number;
   failed: number;