llm-cli-gateway 2.15.0 → 2.17.0

package/dist/async-job-manager.js

@@ -20,6 +20,7 @@ const DEFAULT_LEASE_RUNTIME_CONFIG = {
     role: "gateway",
 };
 const MAX_CONSECUTIVE_HEARTBEAT_FAILURES = 3;
+const MIN_CONSECUTIVE_HEARTBEAT_SUCCESSES_TO_RECOVER = 3;
 export function extractApiHttpStatus(error) {
     for (const candidate of [error, error?.cause]) {
         if (candidate instanceof ApiHttpError && typeof candidate.status === "number") {
@@ -72,6 +73,9 @@ export class JobSaturationError extends Error {
         this.name = "JobSaturationError";
     }
 }
+export function providerAtCapacity(snapshot, provider) {
+    return (snapshot.runningByProvider[provider] ?? 0) >= snapshot.maxRunningPerProvider;
+}
 class JobLimiter {
     cfg;
     logger;
@@ -258,6 +262,10 @@ export class AsyncJobManager {
     sweepTimer = null;
     durableAdmission;
     consecutiveHeartbeatFailures = 0;
+    consecutiveHeartbeatSuccesses = 0;
+    lastHeartbeatFailureAt = null;
+    lastHeartbeatRecoveryAt = null;
+    lastHeartbeatErrorName = null;
     skipSweepThisCycle = false;
     nextHeartbeatExpectedAt = 0;
     disposed = false;
@@ -278,25 +286,11 @@ export class AsyncJobManager {
             maxQueuedJobs: limits.maxQueuedJobs,
             queueTimeoutMs: limits.queueTimeoutMs,
         }, logger);
-        this.durableAdmission = store !== null;
+        this.durableAdmission = false;
         if (this.store) {
-            try {
-                this.store.registerInstance({
-                    instanceId: this.instanceId,
-                    role: this.lease.role ?? "gateway",
-                    hostname: this.hostname,
-                    pid: this.instancePid,
-                });
-            }
-            catch (err) {
-                this.durableAdmission = false;
-                this.logger.error("#139 registerInstance failed; durable async admission is disabled for this instance", err);
-            }
-            if (this.durableAdmission) {
-                this.runOrphanSweep();
-                this.startHeartbeat();
-                this.startReaper();
-            }
+            this.restoreDurableAdmission("startup");
+            this.startHeartbeat();
+            this.startReaper();
         }
         this.evictionTimer = setInterval(() => this.evictCompletedJobs(), EVICTION_INTERVAL_MS);
         if (this.evictionTimer.unref) {
@@ -310,6 +304,17 @@ export class AsyncJobManager {
     canAdmitDurableJobs() {
         return this.store !== null && this.durableAdmission;
     }
+    getDurableAdmissionHealth() {
+        return {
+            storeAttached: this.store !== null,
+            admitting: this.canAdmitDurableJobs(),
+            consecutiveHeartbeatFailures: this.consecutiveHeartbeatFailures,
+            consecutiveHeartbeatSuccesses: this.consecutiveHeartbeatSuccesses,
+            lastHeartbeatFailureAt: this.lastHeartbeatFailureAt,
+            lastHeartbeatRecoveryAt: this.lastHeartbeatRecoveryAt,
+            lastHeartbeatErrorName: this.lastHeartbeatErrorName,
+        };
+    }
     assertDurableAdmission(provider) {
         if (this.store && !this.durableAdmission) {
             throw new Error(`Durable async admission is disabled for ${provider}: this gateway instance could not register or lost its heartbeat lease. Retry after the gateway recovers.`);
@@ -394,13 +399,23 @@ export class AsyncJobManager {
         try {
             this.store.heartbeat(this.instanceId);
             this.consecutiveHeartbeatFailures = 0;
+            if (!this.durableAdmission) {
+                this.consecutiveHeartbeatSuccesses++;
+                if (this.consecutiveHeartbeatSuccesses >= MIN_CONSECUTIVE_HEARTBEAT_SUCCESSES_TO_RECOVER) {
+                    this.restoreDurableAdmission("heartbeat recovery");
+                }
+            }
         }
         catch (err) {
             this.consecutiveHeartbeatFailures++;
+            this.consecutiveHeartbeatSuccesses = 0;
+            this.lastHeartbeatFailureAt = new Date().toISOString();
+            this.lastHeartbeatErrorName = err instanceof Error ? err.name : typeof err;
             this.logger.error(`#139 heartbeat failed (${this.consecutiveHeartbeatFailures}/${MAX_CONSECUTIVE_HEARTBEAT_FAILURES})`, err);
             if (this.consecutiveHeartbeatFailures >= MAX_CONSECUTIVE_HEARTBEAT_FAILURES) {
                 if (this.durableAdmission) {
                     this.durableAdmission = false;
+                    this.consecutiveHeartbeatSuccesses = 0;
                     this.logger.error("#139 sustained heartbeat failure; disabling durable admission and orphan sweeping on this instance");
                 }
             }
@@ -412,8 +427,10 @@ export class AsyncJobManager {
     startReaper() {
         this.sweepTimer = setInterval(() => {
             this.runOrphanSweep();
+            if (!this.store || this.disposed || !this.durableAdmission)
+                return;
             try {
-                this.store?.gcInstances(this.lease.instanceGcMs);
+                this.store.gcInstances(this.lease.instanceGcMs);
             }
             catch (err) {
                 this.logger.error("#139 gateway_instances GC failed", err);
@@ -422,6 +439,35 @@ export class AsyncJobManager {
         if (this.sweepTimer.unref)
             this.sweepTimer.unref();
     }
+    restoreDurableAdmission(reason) {
+        if (!this.store || this.disposed)
+            return;
+        try {
+            this.store.registerInstance({
+                instanceId: this.instanceId,
+                role: this.lease.role ?? "gateway",
+                hostname: this.hostname,
+                pid: this.instancePid,
+            });
+            this.store.heartbeat(this.instanceId);
+            const wasDisabled = !this.durableAdmission;
+            this.durableAdmission = true;
+            this.consecutiveHeartbeatFailures = 0;
+            this.consecutiveHeartbeatSuccesses = 0;
+            this.lastHeartbeatRecoveryAt = new Date().toISOString();
+            if (wasDisabled && reason === "heartbeat recovery") {
+                this.logger.info("#139 durable heartbeat recovered; re-enabled async admission and orphan sweeping");
+            }
+            this.runOrphanSweep();
+        }
+        catch (err) {
+            this.durableAdmission = false;
+            this.consecutiveHeartbeatSuccesses = 0;
+            this.lastHeartbeatFailureAt = new Date().toISOString();
+            this.lastHeartbeatErrorName = err instanceof Error ? err.name : typeof err;
+            this.logger.error(`#139 ${reason} register/heartbeat failed; durable async admission remains disabled`, err);
+        }
+    }
     runOrphanSweep() {
         if (!this.store || this.disposed || !this.durableAdmission)
             return;
@@ -493,13 +539,13 @@ export class AsyncJobManager {
                 acq.cancel();
             this.jobs.delete(job.id);
             this.logger.error(`#139 durable recordStart failed for job ${job.id}; failing the request (fail-closed)`, err);
-            throw new Error(`Durable job admission failed for ${job.cli}: the job store rejected recordStart`);
+            throw new Error(`Durable job admission failed for ${job.cli}: the job store rejected recordStart`, { cause: err });
         }
     }
     markRunningDurable(job, pid, failClosed = false) {
         if (!this.store)
             return;
-        let transitioned = false;
+        let transitioned;
         try {
             transitioned = this.store.markRunning(job.id, { pid });
         }
@@ -645,7 +691,7 @@ export class AsyncJobManager {
         if (evicted > 0) {
             this.logger.debug(`Evicted ${evicted} completed jobs from memory (durable store retains them)`);
         }
-        if (this.store) {
+        if (this.store && this.durableAdmission) {
             try {
                 const removed = this.store.evictExpired();
                 if (removed > 0) {
@@ -657,11 +703,12 @@ export class AsyncJobManager {
             }
         }
     }
-    buildRequestKey(cli, args, env, stdin, cwd, outputFormat) {
+    buildRequestKey(cli, args, env, stdin, cwd, outputFormat, compressResponse) {
         const extraEnv = canonicaliseEnvForKey(env);
         const withStdin = stdin === undefined ? extraEnv : `${extraEnv}|stdin:${stdin}`;
         const withCwd = cwd === undefined ? withStdin : `${withStdin}|cwd:${cwd}`;
-        const extra = cli === "codex" ? `${withCwd}|fmt:${outputFormat ?? "text"}` : withCwd;
+        const withFmt = cli === "codex" ? `${withCwd}|fmt:${outputFormat ?? "text"}` : withCwd;
+        const extra = compressResponse ? `${withFmt}|compress:1` : withFmt;
         const principal = resolveOwnerPrincipal(getRequestContext());
         const scoped = principal === "local" ? extra : `${extra}|principal:${principal}`;
         return computeRequestKey(cli, args, scoped);
@@ -927,7 +974,7 @@ export class AsyncJobManager {
                 durationMs,
                 retryCount: 0,
                 circuitBreakerState: "closed",
-                optimizationApplied: false,
+                optimizationApplied: job.flightRecorderEntry.optimizationApplied ?? false,
                 exitCode,
                 httpStatus: job.transport === "http" ? (job.httpStatus ?? undefined) : undefined,
                 errorMessage,
@@ -937,6 +984,7 @@ export class AsyncJobManager {
                 cacheReadTokens: usage.cacheReadTokens,
                 cacheCreationTokens: usage.cacheCreationTokens,
                 costUsd: usage.costUsd,
+                costBasis: usage.costBasis,
                 providerSessionId: providerMeta?.sessionId,
                 stopReason: providerMeta?.stopReason,
             });
@@ -965,6 +1013,7 @@ export class AsyncJobManager {
             inputTokens: u.inputTokens,
             outputTokens: u.outputTokens,
             cacheReadTokens: u.cacheReadTokens,
+            cacheCreationTokens: u.cacheCreationTokens,
             costUsd: u.costUsd,
         };
     }
@@ -1068,6 +1117,7 @@ export class AsyncJobManager {
             exited: row.status !== "running" && row.status !== "queued",
             metricsRecorded: true,
             outputFormat: row.outputFormat ?? undefined,
+            compressResponse: row.compressResponse ?? null,
             ownerPrincipal: row.ownerPrincipal,
             outputDirty: false,
             lastOutputFlushAt: Date.now(),
@@ -1081,7 +1131,7 @@ export class AsyncJobManager {
             job = this.hydrateFromStore(jobId) ?? undefined;
         return job?.ownerPrincipal;
     }
-    startJob(cli, args, correlationId, cwd, idleTimeoutMs, outputFormat, forceRefresh, env, onComplete, flightRecorderEntry, extractUsage, writeFlightStart, stdin) {
+    startJob(cli, args, correlationId, cwd, idleTimeoutMs, outputFormat, forceRefresh, env, onComplete, flightRecorderEntry, extractUsage, writeFlightStart, stdin, compressResponse) {
         return this.startJobWithDedup(cli, args, correlationId, {
             cwd,
             idleTimeoutMs,
@@ -1093,11 +1143,12 @@ export class AsyncJobManager {
             flightRecorderEntry,
             extractUsage,
             writeFlightStart,
+            compressResponse,
         }).snapshot;
     }
     startJobWithDedup(cli, args, correlationId, opts = {}) {
-        const { cwd, idleTimeoutMs, outputFormat, forceRefresh, env: extraEnv, stdin, onComplete, flightRecorderEntry, extractUsage, writeFlightStart, } = opts;
-        const requestKey = this.buildRequestKey(cli, args, extraEnv, stdin, cwd, outputFormat);
+        const { cwd, idleTimeoutMs, outputFormat, forceRefresh, env: extraEnv, stdin, onComplete, flightRecorderEntry, extractUsage, writeFlightStart, compressResponse, } = opts;
+        const requestKey = this.buildRequestKey(cli, args, extraEnv, stdin, cwd, outputFormat, compressResponse);
         if (!forceRefresh) {
             const reused = this.tryReuseDedupedJob(requestKey, correlationId, cli, onComplete);
             if (reused)
@@ -1130,6 +1181,7 @@ export class AsyncJobManager {
             exited: false,
             metricsRecorded: false,
             outputFormat,
+            compressResponse: compressResponse ?? null,
             onComplete,
             onCompleteFired: false,
             outputDirty: false,
@@ -1178,6 +1230,7 @@ export class AsyncJobManager {
             cli,
             args: [...args],
             outputFormat,
+            compressResponse,
             startedAt,
             pid: null,
             ownerPrincipal,
@@ -1222,6 +1275,7 @@ export class AsyncJobManager {
             cwd,
             stdio: stdin === undefined ? ["ignore", "pipe", "pipe"] : ["pipe", "pipe", "pipe"],
             env: { ...baseEnv, ...(extraEnv ?? {}) },
+            logger: this.logger,
         });
         job.process = child;
         try {
@@ -1531,6 +1585,9 @@ export class AsyncJobManager {
     getJobOutputFormat(jobId) {
         return this.jobs.get(jobId)?.outputFormat;
     }
+    getJobCompressResponse(jobId) {
+        return this.jobs.get(jobId)?.compressResponse === true;
+    }
     getJobCli(jobId) {
         return this.jobs.get(jobId)?.cli;
     }

package/dist/claude-mcp-config.js

@@ -147,7 +147,7 @@ export function buildClaudeMcpConfig(servers) {
     }
     catch (error) {
         const message = error instanceof Error ? error.message : String(error);
-        throw new Error(`Failed to write Claude MCP config: ${message}`);
+        throw new Error(`Failed to write Claude MCP config: ${message}`, { cause: error });
     }
     return { path: configPath, enabled, missing };
 }

package/dist/compressor/estimate.d.ts

@@ -0,0 +1,5 @@
+import type { ContentRoute } from "./router.js";
+export type DevTokenizer = (text: string) => number;
+export declare function setDevTokenizer(tokenizer: DevTokenizer | null): void;
+export declare function estimateTokensForRoute(text: string, route: ContentRoute): number;
+export declare function estimateTokensSaved(originalText: string, compressedText: string, route: ContentRoute): number;

package/dist/compressor/estimate.js

@@ -0,0 +1,21 @@
+const DIVISORS = {
+    json: 3.0,
+    log: 3.4,
+    "ansi-text": 3.4,
+    plain: 3.6,
+    identity: 3.6,
+};
+let devTokenizer = null;
+export function setDevTokenizer(tokenizer) {
+    devTokenizer = tokenizer;
+}
+export function estimateTokensForRoute(text, route) {
+    if (devTokenizer)
+        return devTokenizer(text);
+    if (text.length === 0)
+        return 0;
+    return Math.ceil(text.length / DIVISORS[route]);
+}
+export function estimateTokensSaved(originalText, compressedText, route) {
+    return Math.max(0, estimateTokensForRoute(originalText, route) - estimateTokensForRoute(compressedText, route));
+}

package/dist/compressor/index.d.ts

@@ -0,0 +1,23 @@
+import { type ContentRoute } from "./router.js";
+export interface CompressCtx {
+    provider: string;
+    direction: "outbound";
+    outputFormat?: string;
+    outputSchemaDeclared: boolean;
+    lossless: true;
+}
+export interface CompressResult {
+    text: string;
+    originalChars: number;
+    compressedChars: number;
+    route: ContentRoute;
+    transforms: string[];
+    estimatedTokensSaved: number;
+}
+export interface Compressor {
+    compact(text: string, ctx: CompressCtx): CompressResult;
+}
+export declare class NativeCompressor implements Compressor {
+    compact(text: string, ctx: CompressCtx): CompressResult;
+}
+export declare function compressDisplayText(text: string, ctx: CompressCtx): CompressResult;

package/dist/compressor/index.js

@@ -0,0 +1,77 @@
+import { classify } from "./router.js";
+import { minifyJson } from "./transforms/json.js";
+import { dedupAdjacentLines, escapeSentinelLikeLines, noteLine, } from "./transforms/log.js";
+import { stripAnsi } from "./transforms/ansi.js";
+import { normalizeWhitespace } from "./transforms/whitespace.js";
+import { estimateTokensSaved } from "./estimate.js";
+function identityResult(text, route = "identity") {
+    return {
+        text,
+        originalChars: text.length,
+        compressedChars: text.length,
+        route,
+        transforms: [],
+        estimatedTokensSaved: 0,
+    };
+}
+export class NativeCompressor {
+    compact(text, ctx) {
+        if (ctx.outputFormat === "json" || ctx.outputSchemaDeclared) {
+            return identityResult(text);
+        }
+        const route = classify(text);
+        if (route === "identity")
+            return identityResult(text);
+        if (route === "json") {
+            const minified = minifyJson(text.trim());
+            if (minified === null || minified.length >= text.length) {
+                return identityResult(text);
+            }
+            return {
+                text: minified,
+                originalChars: text.length,
+                compressedChars: minified.length,
+                route,
+                transforms: ["json-minify"],
+                estimatedTokensSaved: estimateTokensSaved(text, minified, route),
+            };
+        }
+        const counts = { folded: 0, escaped: 0 };
+        const transforms = [];
+        let current = text;
+        const apply = (name, fn) => {
+            const next = fn(current);
+            if (next !== current)
+                transforms.push(name);
+            current = next;
+        };
+        apply("lit-escape", input => escapeSentinelLikeLines(input, counts));
+        if (route === "ansi-text") {
+            apply("ansi-strip", input => stripAnsi(input, counts));
+            apply("dedup", input => dedupAdjacentLines(input, counts));
+        }
+        if (route === "log") {
+            apply("dedup", input => dedupAdjacentLines(input, counts));
+        }
+        apply("whitespace", normalizeWhitespace);
+        if (counts.folded + counts.escaped > 0) {
+            current = `${noteLine(counts.folded, counts.escaped)}\n${current}`;
+            transforms.push("leading-note");
+        }
+        if (current.length >= text.length) {
+            return identityResult(text, route);
+        }
+        return {
+            text: current,
+            originalChars: text.length,
+            compressedChars: current.length,
+            route,
+            transforms,
+            estimatedTokensSaved: estimateTokensSaved(text, current, route),
+        };
+    }
+}
+const defaultCompressor = new NativeCompressor();
+export function compressDisplayText(text, ctx) {
+    return defaultCompressor.compact(text, ctx);
+}

package/dist/compressor/router.d.ts

@@ -0,0 +1,2 @@
+export type ContentRoute = "json" | "log" | "ansi-text" | "plain" | "identity";
+export declare function classify(text: string): ContentRoute;

package/dist/compressor/router.js

@@ -0,0 +1,57 @@
+import { hasDangerousSequences } from "./transforms/ansi.js";
+import { minifyJson } from "./transforms/json.js";
+import { splitFences } from "./transforms/whitespace.js";
+import { MIN_RUN } from "./transforms/log.js";
+const HAS_ESCAPE = /\x1b/;
+const HAS_INTERNAL_CR = /\r(?!\n|$)/;
+const HAS_FENCE = /^ {0,3}(?:`{3,}|~{3,})/m;
+function hasFence(text) {
+    return HAS_FENCE.test(text);
+}
+function hasRepeatedRuns(text) {
+    for (const segment of splitFences(text)) {
+        if (segment.fenced)
+            continue;
+        const lines = segment.text.split("\n");
+        let run = 1;
+        for (let i = 1; i < lines.length; i += 1) {
+            if (lines[i] === lines[i - 1] && lines[i].trim() !== "") {
+                run += 1;
+                if (run >= MIN_RUN)
+                    return true;
+            }
+            else {
+                run = 1;
+            }
+        }
+    }
+    return false;
+}
+function isStructuralJson(text) {
+    const trimmed = text.trim();
+    if (trimmed.length < 2)
+        return false;
+    const first = trimmed[0];
+    const last = trimmed[trimmed.length - 1];
+    const structural = (first === "{" && last === "}") || (first === "[" && last === "]");
+    if (!structural)
+        return false;
+    return minifyJson(trimmed) !== null;
+}
+export function classify(text) {
+    if (text.length === 0)
+        return "identity";
+    const escapes = HAS_ESCAPE.test(text);
+    const internalCr = HAS_INTERNAL_CR.test(text);
+    if (escapes || internalCr) {
+        if (hasDangerousSequences(text))
+            return "identity";
+        if (!hasFence(text))
+            return "ansi-text";
+    }
+    if (isStructuralJson(text))
+        return "json";
+    if (hasRepeatedRuns(text))
+        return "log";
+    return "plain";
+}

package/dist/compressor/transforms/ansi.d.ts

@@ -0,0 +1,3 @@
+import { type MarkerCounts } from "./log.js";
+export declare function hasDangerousSequences(text: string): boolean;
+export declare function stripAnsi(text: string, counts: MarkerCounts): string;

package/dist/compressor/transforms/ansi.js

@@ -0,0 +1,99 @@
+import { mapUnfenced } from "./whitespace.js";
+import { crMarker } from "./log.js";
+const CSI = /\x1b\[([0-9:;<=>?]*)[ -/]*([@-~])/g;
+const OSC = /\x1b\][\s\S]*?(?:\x07|\x1b\\)/g;
+const DCS_APC_PM_SOS = /\x1b[PX^_][\s\S]*?\x1b\\/g;
+const ESC_TWO_BYTE = /\x1b[@-Z\\-_]/g;
+const STRAY_C0 = /[\x00-\x07\x0b\x0c\x0e-\x1f]/g;
+const DANGEROUS_CSI_FINALS = new Set([
+    "A",
+    "B",
+    "C",
+    "D",
+    "E",
+    "F",
+    "G",
+    "H",
+    "f",
+    "S",
+    "T",
+    "J",
+    "s",
+    "u",
+]);
+export function hasDangerousSequences(text) {
+    if (text.includes("\b"))
+        return true;
+    CSI.lastIndex = 0;
+    let match;
+    while ((match = CSI.exec(text)) !== null) {
+        const params = match[1];
+        const final = match[2];
+        if (DANGEROUS_CSI_FINALS.has(final))
+            return true;
+        if ((final === "h" || final === "l") && params.includes("?")) {
+            return true;
+        }
+    }
+    return false;
+}
+function stripEscapes(text) {
+    return text
+        .replace(OSC, "")
+        .replace(DCS_APC_PM_SOS, "")
+        .replace(CSI, "")
+        .replace(ESC_TWO_BYTE, "")
+        .replace(STRAY_C0, "");
+}
+function isSingleColumnAsciiFrame(text) {
+    for (const character of text) {
+        const codePoint = character.codePointAt(0);
+        if (codePoint !== 0x09 &&
+            codePoint !== 0x0d &&
+            (codePoint === undefined || codePoint < 0x20 || codePoint > 0x7e)) {
+            return false;
+        }
+    }
+    return true;
+}
+function overlayFrames(frames) {
+    const cells = [];
+    for (const frame of frames) {
+        for (let i = 0; i < frame.length; i += 1)
+            cells[i] = frame[i];
+    }
+    return cells.join("");
+}
+function collapseLine(line, out, counts) {
+    const crlf = line.endsWith("\r");
+    const body = crlf ? line.slice(0, -1) : line;
+    if (!body.includes("\r")) {
+        out.push(line);
+        return;
+    }
+    if (!isSingleColumnAsciiFrame(body)) {
+        out.push(line);
+        return;
+    }
+    const frames = body.split("\r");
+    const visible = overlayFrames(frames);
+    out.push(crlf ? `${visible}\r` : visible);
+    out.push(crMarker(frames.length - 1));
+    counts.folded += 1;
+}
+export function stripAnsi(text, counts) {
+    if (hasDangerousSequences(text))
+        return text;
+    return mapUnfenced(text, segment => {
+        const stripped = stripEscapes(segment);
+        const out = [];
+        for (const line of stripped.split("\n")) {
+            if (line.includes("`")) {
+                out.push(line);
+                continue;
+            }
+            collapseLine(line, out, counts);
+        }
+        return out.join("\n");
+    });
+}

package/dist/compressor/transforms/json.d.ts

@@ -0,0 +1 @@
+export declare function minifyJson(text: string): string | null;