llm-cli-gateway 2.12.1 → 2.13.0

package/dist/http-transport.js

@@ -3,7 +3,7 @@ import { randomUUID } from "node:crypto";
 import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
 import { isInitializeRequest } from "@modelcontextprotocol/sdk/types.js";
 import { authorizeBearerRequest, getRequiredBearerToken, resolveTrustedPrincipal, writeAuthFailure, } from "./auth.js";
-import { loadRemoteOAuthConfig } from "./config.js";
+import { loadRemoteOAuthConfig, loadLimitsConfig } from "./config.js";
 import { OAuthServer, oauthBaseUrlFromRequest } from "./oauth.js";
 import { runWithRequestContext } from "./request-context.js";
 import { readCappedRawBody, maxHttpBodyBytes } from "./request-limits.js";
@@ -79,6 +79,8 @@ export async function startHttpGateway(options) {
     const noAuthPaths = parseNoAuthPaths(process.env.LLM_GATEWAY_NO_AUTH_PATHS, path);
     const logger = options.logger ?? noopLogger;
     const sessions = new Map();
+    let pendingInitializes = 0;
+    const httpLimits = options.httpLimits ?? loadLimitsConfig(logger).http;
     const token = getRequiredBearerToken();
     const oauthConfig = loadRemoteOAuthConfig(logger);
     if (oauthConfig.enabled &&
@@ -101,14 +103,33 @@ export async function startHttpGateway(options) {
             .catch(error => logger.error("HTTP transport close failed", error));
         await entry.server.close().catch(error => logger.error("HTTP MCP server close failed", error));
     }
-    async function createSession() {
+    function touchSessionComplete(entry) {
+        entry.inFlight = Math.max(0, entry.inFlight - 1);
+        entry.lastActivityAt = Date.now();
+    }
+    async function createSession(releaseInitializeReservation) {
         const gatewayServer = options.createGatewayServer(options.deps);
+        let entry;
         const transport = new StreamableHTTPServerTransport({
             sessionIdGenerator: () => randomUUID(),
             onsessioninitialized: sessionId => {
-                sessions.set(sessionId, { server: gatewayServer, transport });
+                const now = Date.now();
+                releaseInitializeReservation();
+                entry.sessionId = sessionId;
+                entry.createdAt = now;
+                entry.lastActivityAt = now;
+                entry.inFlight++;
+                sessions.set(sessionId, entry);
             },
         });
+        const now = Date.now();
+        entry = {
+            server: gatewayServer,
+            transport,
+            createdAt: now,
+            lastActivityAt: now,
+            inFlight: 0,
+        };
         transport.onclose = () => {
             if (transport.sessionId) {
                 sessions.delete(transport.sessionId);
@@ -116,7 +137,70 @@ export async function startHttpGateway(options) {
         };
         transport.onerror = error => logger.error("HTTP MCP transport error", error);
         await gatewayServer.connect(transport);
-        return { server: gatewayServer, transport };
+        return entry;
+    }
+    function reapIdleSessions() {
+        const now = Date.now();
+        for (const [sessionId, entry] of sessions) {
+            if (entry.inFlight > 0)
+                continue;
+            if (now - entry.lastActivityAt < httpLimits.sessionIdleTtlMs)
+                continue;
+            logger.info(`Reaping idle HTTP MCP session (idle ${now - entry.lastActivityAt}ms >= ${httpLimits.sessionIdleTtlMs}ms)`);
+            void closeSession(sessionId);
+        }
+    }
+    const reaperTimer = setInterval(reapIdleSessions, httpLimits.sessionReaperIntervalMs);
+    if (reaperTimer.unref)
+        reaperTimer.unref();
+    function sessionHealth() {
+        const now = Date.now();
+        let oldestAgeMs = 0;
+        for (const entry of sessions.values()) {
+            const age = now - entry.createdAt;
+            if (age > oldestAgeMs)
+                oldestAgeMs = age;
+        }
+        return {
+            current: sessions.size,
+            max: httpLimits.maxSessions,
+            oldestAgeMs,
+            idleTtlMs: httpLimits.sessionIdleTtlMs,
+            reaperIntervalMs: httpLimits.sessionReaperIntervalMs,
+            saturated: sessions.size + pendingInitializes >= httpLimits.maxSessions,
+        };
+    }
+    function healthPayload() {
+        const manager = options.deps?.asyncJobManager;
+        const mem = process.memoryUsage();
+        const payload = {
+            sessions: sessionHealth(),
+            memory: {
+                rss: mem.rss,
+                heapUsed: mem.heapUsed,
+                heapTotal: mem.heapTotal,
+                external: mem.external,
+            },
+        };
+        if (manager) {
+            const limiter = manager.getLimiterSnapshot();
+            const limits = manager.getConfiguredLimits();
+            payload.jobs = {
+                running: limiter.running,
+                queued: limiter.queued,
+                runningByProvider: limiter.runningByProvider,
+                queuedByProvider: limiter.queuedByProvider,
+                maxRunning: limiter.maxRunning,
+                maxRunningPerProvider: limiter.maxRunningPerProvider,
+                maxQueued: limiter.maxQueued,
+                rejected: limiter.rejected,
+                timedOut: limiter.timedOut,
+                saturated: limiter.saturated,
+                completedJobMemoryTtlMs: limits.completedJobMemoryTtlMs,
+                maxJobOutputBytes: limits.maxJobOutputBytes,
+            };
+        }
+        return payload;
     }
     const httpServer = createServer(async (req, res) => {
         try {
@@ -127,7 +211,7 @@ export async function startHttpGateway(options) {
             const resourceMetadataUrl = oauthServer && oauthOrigin ? oauthServer.resourceMetadataUrl(oauthOrigin) : undefined;
             if (url.pathname === "/healthz") {
                 res.writeHead(200, { "content-type": "application/json" });
-                res.end(JSON.stringify({ ok: true, sessions: sessions.size }));
+                res.end(JSON.stringify({ ok: true, ...healthPayload() }));
                 return;
             }
             if (oauthServer) {
@@ -188,7 +272,14 @@ export async function startHttpGateway(options) {
                     return;
                 }
                 const body = req.method === "POST" ? await readBody(req) : undefined;
-                await runWithRequestContext(requestContext, () => entry.transport.handleRequest(req, res, body));
+                entry.lastActivityAt = Date.now();
+                entry.inFlight++;
+                try {
+                    await runWithRequestContext(requestContext, () => entry.transport.handleRequest(req, res, body));
+                }
+                finally {
+                    touchSessionComplete(entry);
+                }
                 return;
             }
             if (req.method !== "POST") {
@@ -204,8 +295,45 @@ export async function startHttpGateway(options) {
                 jsonError(res, 400, "First request must be initialize");
                 return;
             }
-            const entry = await createSession();
-            await runWithRequestContext(requestContext, () => entry.transport.handleRequest(req, res, body));
+            const capacityInUse = sessions.size + pendingInitializes;
+            if (capacityInUse >= httpLimits.maxSessions) {
+                res.writeHead(429, {
+                    "content-type": "application/json",
+                    "retry-after": "5",
+                });
+                res.end(JSON.stringify({
+                    error: "Gateway at session capacity",
+                    code: "session_capacity",
+                    retryable: true,
+                    sessions: {
+                        current: sessions.size,
+                        pending: pendingInitializes,
+                        max: httpLimits.maxSessions,
+                    },
+                }));
+                logger.info(`Rejected new HTTP MCP session: at capacity (${capacityInUse}/${httpLimits.maxSessions})`);
+                return;
+            }
+            pendingInitializes++;
+            let reservationReleased = false;
+            const releaseInitializeReservation = () => {
+                if (reservationReleased)
+                    return;
+                reservationReleased = true;
+                pendingInitializes = Math.max(0, pendingInitializes - 1);
+            };
+            let entry;
+            try {
+                const created = await createSession(releaseInitializeReservation);
+                entry = created;
+                await runWithRequestContext(requestContext, () => created.transport.handleRequest(req, res, body));
+            }
+            finally {
+                releaseInitializeReservation();
+                if (entry?.sessionId && sessions.get(entry.sessionId) === entry && entry.inFlight > 0) {
+                    touchSessionComplete(entry);
+                }
+            }
         }
         catch (error) {
             logger.error("HTTP transport request failed", error);
@@ -241,11 +369,13 @@ export async function startHttpGateway(options) {
         server: httpServer,
         url,
         close: async () => {
+            clearInterval(reaperTimer);
             await Promise.all([...sessions.keys()].map(closeSession));
             await new Promise((resolve, reject) => {
                 httpServer.close(error => (error ? reject(error) : resolve()));
             });
         },
         sessionCount: () => sessions.size,
+        sessionHealth,
     };
 }

package/dist/index.d.ts

@@ -7,7 +7,7 @@ import { PerformanceMetrics } from "./metrics.js";
 import { type PersistenceConfig, type CacheAwarenessConfig, type ProvidersConfig, type ApiProviderRuntime, type AcpConfig } from "./config.js";
 import { DatabaseConnection } from "./db.js";
 import { AsyncJobManager } from "./async-job-manager.js";
-import { ApprovalManager, ApprovalRecord } from "./approval-manager.js";
+import { ApprovalManager, ApprovalPolicy, ApprovalRecord } from "./approval-manager.js";
 import { ReviewIntegrityResult } from "./review-integrity.js";
 import { ClaudeMcpConfigResult, ClaudeMcpServerName } from "./claude-mcp-config.js";
 import { type MistralAgentMode, type ClaudePermissionMode, type CodexSandboxMode, type CodexAskForApproval, type ClaudeEffortLevel } from "./request-helpers.js";
@@ -60,8 +60,8 @@ export declare const WORKTREE_SCHEMA: z.ZodUnion<[z.ZodBoolean, z.ZodObject<{
     ref?: string | undefined;
 }>]>;
 export declare const WORKSPACE_ALIAS_SCHEMA: z.ZodString;
-export declare const SESSION_PROVIDER_VALUES: readonly ["claude", "codex", "gemini", "grok", "mistral", "devin", "grok-api"];
-export declare const SESSION_PROVIDER_ENUM: z.ZodEnum<["claude", "codex", "gemini", "grok", "mistral", "devin", "grok-api"]>;
+export declare const SESSION_PROVIDER_VALUES: readonly ["claude", "codex", "gemini", "grok", "mistral", "devin", "cursor", "grok-api"];
+export declare const SESSION_PROVIDER_ENUM: z.ZodEnum<["claude", "codex", "gemini", "grok", "mistral", "devin", "cursor", "grok-api"]>;
 export declare function sessionProviderValuesFor(providers: ProvidersConfig): string[];
 export type SessionProvider = ProviderType;
 export interface GatewayServerDeps {
@@ -124,6 +124,20 @@ export declare function createErrorResponse(cli: string, code: number, stderr: s
     httpStatus?: number | null;
     responseBody?: string;
 }): {
+    content: {
+        type: "text";
+        text: string;
+    }[];
+    isError: boolean;
+    structuredContent: {
+        response: string;
+        correlationId: string | null;
+        cli: string;
+        exitCode: number;
+        errorCategory: string;
+        retryable: boolean;
+    };
+} | {
     content: {
         type: "text";
         text: string;
@@ -137,9 +151,10 @@ export declare function createErrorResponse(cli: string, code: number, stderr: s
         cli: string;
         exitCode: number;
         errorCategory: string;
+        retryable?: undefined;
     };
 };
-export declare function extractUsageAndCost(cli: "claude" | "codex" | "gemini" | "grok" | "mistral" | "devin", output: string, outputFormat?: string, ctx?: {
+export declare function extractUsageAndCost(cli: CliType, output: string, outputFormat?: string, ctx?: {
     sessionId?: string;
     home?: string;
 }): {
@@ -333,7 +348,7 @@ export declare function buildMistralRetryPrep(params: Pick<MistralRequestParams,
     env: Record<string, string>;
     ignoredDisallowedTools: boolean;
 };
-export declare function buildCliResponse(cli: "claude" | "codex" | "gemini" | "grok" | "mistral" | "devin", stdout: string, optimizeResponse: boolean, corrId: string, sessionId: string | undefined, prep: CliRequestPrep, durationMs: number, resumable?: boolean, outputFormat?: string, warnings?: WarningEntry[]): ExtendedToolResponse;
+export declare function buildCliResponse(cli: CliType, stdout: string, optimizeResponse: boolean, corrId: string, sessionId: string | undefined, prep: CliRequestPrep, durationMs: number, resumable?: boolean, outputFormat?: string, warnings?: WarningEntry[]): ExtendedToolResponse;
 export interface GrokApiRequestParams {
     prompt?: string;
     promptParts?: PromptParts;
@@ -507,6 +522,48 @@ export declare function prepareDevinRequest(params: {
 }, _runtime: GatewayServerRuntime): CliRequestPrep | ExtendedToolResponse;
 export declare function handleDevinRequest(deps: HandlerDeps, params: DevinRequestParams): Promise<ExtendedToolResponse>;
 export declare function handleDevinRequestAsync(deps: AsyncHandlerDeps, params: Omit<DevinRequestParams, "optimizeResponse">): Promise<ExtendedToolResponse>;
+export interface CursorRequestParams {
+    prompt?: string;
+    model?: string;
+    mode?: "plan" | "ask";
+    outputFormat?: "text" | "json" | "stream-json";
+    transport?: "cli" | "acp";
+    force?: boolean;
+    autoReview?: boolean;
+    sandbox?: "enabled" | "disabled";
+    trust?: boolean;
+    workspace?: string;
+    addDir?: string[];
+    sessionId?: string;
+    resumeLatest?: boolean;
+    createNewSession?: boolean;
+    approvalStrategy?: "legacy" | "mcp_managed";
+    approvalPolicy?: ApprovalPolicy;
+    correlationId?: string;
+    optimizePrompt: boolean;
+    optimizeResponse?: boolean;
+    idleTimeoutMs?: number;
+    forceRefresh?: boolean;
+}
+export declare function prepareCursorRequest(params: {
+    prompt?: string;
+    model?: string;
+    mode?: CursorRequestParams["mode"];
+    outputFormat?: CursorRequestParams["outputFormat"];
+    force?: boolean;
+    autoReview?: boolean;
+    sandbox?: CursorRequestParams["sandbox"];
+    trust?: boolean;
+    workspace?: string;
+    addDir?: string[];
+    approvalStrategy?: CursorRequestParams["approvalStrategy"];
+    approvalPolicy?: CursorRequestParams["approvalPolicy"];
+    correlationId?: string;
+    optimizePrompt: boolean;
+    operation: string;
+}, runtime: GatewayServerRuntime): CliRequestPrep | ExtendedToolResponse;
+export declare function handleCursorRequest(deps: HandlerDeps, params: CursorRequestParams): Promise<ExtendedToolResponse>;
+export declare function handleCursorRequestAsync(deps: AsyncHandlerDeps, params: Omit<CursorRequestParams, "optimizeResponse">): Promise<ExtendedToolResponse>;
 export interface MistralRequestParams {
     prompt?: string;
     promptParts?: PromptParts;