llm-cli-gateway 2.16.0 → 2.17.0

package/dist/index.js

@@ -7,7 +7,8 @@ import { existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileS
 import { basename, dirname, isAbsolute, join, relative } from "path";
 import { fileURLToPath } from "url";
 import { z } from "zod/v3";
-import { executeCli, killAllProcessGroups, providerCommandName } from "./executor.js";
+import { executeCli, killAllProcessGroups, providerCommandName, cliBreakerState, } from "./executor.js";
+import { isDefaultTransient } from "./retry.js";
 import { parseStreamJson } from "./stream-json-parser.js";
 import { parseCodexJsonStream, codexDisplayText, codexFrResponse } from "./codex-json-parser.js";
 import { grokDisplayText } from "./grok-json-parser.js";
@@ -22,7 +23,11 @@ import { generateResourceDescriptors, generateProviderAcpDescriptors, } from "./
 import { PerformanceMetrics } from "./metrics.js";
 import { estimateTokens, optimizePrompt as optimizePromptText, optimizeResponse as optimizeResponseText, } from "./optimizer.js";
 import { compressDisplayText } from "./compressor/index.js";
-import { loadConfig, loadPersistenceConfig, loadCacheAwarenessConfig, loadProvidersConfig, loadAcpConfig, loadAdminConfig, loadLimitsConfig, loadCompressionConfig, loadSkillsConfig, defaultGatewayConfigPath, isXaiProviderEnabled, enabledApiProviders, minStableTokensForModel, } from "./config.js";
+import { loadConfig, loadPersistenceConfig, loadCacheAwarenessConfig, loadProvidersConfig, loadAcpConfig, loadAdminConfig, loadLimitsConfig, loadCompressionConfig, loadLeastCostConfig, loadSkillsConfig, defaultGatewayConfigPath, isXaiProviderEnabled, enabledApiProviders, minStableTokensForModel, } from "./config.js";
+import { buildRouterEnv, toRouterConfig } from "./lcr-router-env.js";
+import { getModelCost, composeCost, modelIdToFamily } from "./pricing.js";
+import { telemetryTierFor } from "./lcr-telemetry.js";
+import { selectCandidate, } from "./least-cost-router.js";
 import { loadGatewaySkills } from "./skill-loader.js";
 import { runAcpRequest } from "./acp/runtime.js";
 import { isAcpError } from "./acp/errors.js";
@@ -230,6 +235,7 @@ let flightRecorder = null;
 let persistenceConfig = null;
 let cacheAwarenessConfig = null;
 let compressionConfig = null;
+let leastCostConfig = null;
 let providersConfig = null;
 let acpConfig = null;
 let adminConfig = null;
@@ -266,6 +272,10 @@ function getCompressionConfig(runtimeLogger = logger) {
     compressionConfig ??= loadCompressionConfig(runtimeLogger);
     return compressionConfig;
 }
+function getLeastCostConfig(runtimeLogger = logger) {
+    leastCostConfig ??= loadLeastCostConfig(runtimeLogger);
+    return leastCostConfig;
+}
 function getProvidersConfig(runtimeLogger = logger) {
     providersConfig ??= loadProvidersConfig(runtimeLogger);
     return providersConfig;
@@ -278,7 +288,7 @@ function getJobStore(runtimeLogger = logger) {
         jobStore = createJobStore(getPersistenceConfig(runtimeLogger), runtimeLogger);
     }
     catch (err) {
-        runtimeLogger.error("Failed to open durable job store; async tools will be unavailable", err);
+        runtimeLogger.error("Failed to open configured durable job store", err);
         jobStore = null;
     }
     return jobStore;
@@ -464,7 +474,7 @@ export function resolveGatewayServerRuntime(deps = {}, options = {}) {
         sessionManager: runtimeSessionManager,
         resourceProvider: deps.resourceProvider ??
             (options.isolateState
-                ? new ResourceProvider(runtimeSessionManager, runtimePerformanceMetrics, runtimeFlightRecorder, deps.cacheAwareness ?? getCacheAwarenessConfig(runtimeLogger), deps.providers ?? getProvidersConfig(runtimeLogger), undefined, deps.acpConfig ?? getAcpConfig(runtimeLogger))
+                ? new ResourceProvider(runtimeSessionManager, runtimePerformanceMetrics, runtimeFlightRecorder, deps.cacheAwareness ?? getCacheAwarenessConfig(runtimeLogger), deps.providers ?? getProvidersConfig(runtimeLogger), undefined, deps.acpConfig ?? getAcpConfig(runtimeLogger), deps.leastCost ?? getLeastCostConfig(runtimeLogger))
                 : resourceProvider),
         db: "db" in deps ? (deps.db ?? null) : db,
         performanceMetrics: runtimePerformanceMetrics,
@@ -479,6 +489,7 @@ export function resolveGatewayServerRuntime(deps = {}, options = {}) {
         acpConfig: deps.acpConfig ?? getAcpConfig(runtimeLogger),
         adminConfig: deps.adminConfig ?? getAdminConfig(runtimeLogger),
         workspaces: deps.workspaces ?? loadWorkspaceRegistry(runtimeLogger),
+        leastCost: deps.leastCost ?? getLeastCostConfig(runtimeLogger),
     };
 }
 export function shouldRegisterGrokApiTools(providers) {
@@ -1179,9 +1190,10 @@ function buildAsyncFlightRecorderHandoff(cliName, prep, sessionId, outputFormat,
     const fmt = outputFormat;
     const sid = sessionId;
     const home = homedir();
+    const model = prep.resolvedModel || "default";
     return {
         flightRecorderEntry: {
-            model: prep.resolvedModel || "default",
+            model,
             prompt: prep.effectivePrompt,
             sessionId,
             optimizationApplied,
@@ -1190,7 +1202,11 @@ function buildAsyncFlightRecorderHandoff(cliName, prep, sessionId, outputFormat,
             cacheControlBlocks: prep.cacheControlBlocks,
             cacheControlTtlSeconds: prep.cacheControlTtlSeconds,
         },
-        extractUsage: (stdout) => extractUsageAndCost(cli, stdout, fmt, { sessionId: sid, home }),
+        extractUsage: (stdout) => {
+            const usage = extractUsageAndCost(cli, stdout, fmt, { sessionId: sid, home });
+            const { costUsd, costBasis } = deriveCostBasis(cli, model, usage);
+            return { ...usage, costUsd, costBasis };
+        },
     };
 }
 function safeFlightStart(entry, runtime = resolveGatewayServerRuntime()) {
@@ -1235,6 +1251,14 @@ function safeRecordCompression(correlationId, compression, runtime) {
         runtime.logger.error("Flight recorder recordCompressionTelemetry failed", error);
     }
 }
+function safeRecordRouting(correlationId, routing, runtime) {
+    try {
+        runtime.flightRecorder.recordRouting(correlationId, routing);
+    }
+    catch (error) {
+        runtime.logger.error("Flight recorder recordRouting failed", error);
+    }
+}
 function createApprovalDeniedResponse(operation, decision) {
     return {
         content: [
@@ -2177,7 +2201,7 @@ export function prepareGrokRequest(params, runtime = resolveGatewayServerRuntime
             args.push("--permission-mode", "acceptEdits");
         }
     }
-    else if (Boolean(params.alwaysApprove)) {
+    else if (params.alwaysApprove) {
         args.push("--always-approve");
     }
     else if (params.permissionMode) {
@@ -2981,6 +3005,7 @@ export async function handleApiProviderRequest(runtimeArg, providerRuntime, para
             inputTokens: result.usage?.inputTokens,
             outputTokens: result.usage?.outputTokens,
             cacheReadTokens: result.usage?.cacheReadTokens,
+            cacheCreationTokens: result.usage?.cacheCreationTokens,
             costUsd: result.usage?.costUsd,
         }, runtimeArg);
         return buildApiSuccessResponse(text, corrId, providerRuntime.name, {
@@ -3252,7 +3277,7 @@ export async function handleGeminiRequest(deps, params) {
             return createErrorResponse("gemini", code, stderr, corrId);
         }
         wasSuccessful = true;
-        let effectiveSessionId = effectiveSessionIdHint;
+        const effectiveSessionId = effectiveSessionIdHint;
         if (effectiveSessionId) {
             const existing = await deps.sessionManager.getSession(effectiveSessionId);
             if (!existing) {
@@ -3277,6 +3302,7 @@ export async function handleGeminiRequest(deps, params) {
             }
         }
         const geminiUsage = extractUsageAndCost("gemini", stdout, params.outputFormat);
+        const { costUsd: geminiCostUsd, costBasis: geminiCostBasis } = deriveCostBasis("gemini", prep.resolvedModel || "default", geminiUsage);
         const geminiMeta = extractProviderOutputMetadata("gemini", stdout, params.outputFormat);
         safeFlightComplete(corrId, {
             response: stdout,
@@ -3291,7 +3317,8 @@ export async function handleGeminiRequest(deps, params) {
             outputTokens: geminiUsage.outputTokens,
             cacheReadTokens: geminiUsage.cacheReadTokens,
             cacheCreationTokens: geminiUsage.cacheCreationTokens,
-            costUsd: geminiUsage.costUsd,
+            costUsd: geminiCostUsd,
+            costBasis: geminiCostBasis,
             providerSessionId: geminiMeta.sessionId,
             stopReason: geminiMeta.stopReason,
         }, runtime);
@@ -3352,7 +3379,7 @@ export async function handleGeminiRequestAsync(deps, params) {
             resumeLatest: params.resumeLatest,
             createNewSession: params.createNewSession,
         });
-        let effectiveSessionId = sessionPlan.resumed ? params.sessionId : undefined;
+        const effectiveSessionId = sessionPlan.resumed ? params.sessionId : undefined;
         const existingSession = await getExistingSessionForProvider(deps.sessionManager, effectiveSessionId, "gemini");
         args.push(...sessionPlan.args);
         if (effectiveSessionId) {
@@ -4730,14 +4757,8 @@ export async function handleCodexRequestAsync(deps, params) {
             outputSchemaDeclared: params.outputSchema !== undefined,
         });
         const codexAsyncFrHandoff = buildAsyncFlightRecorderHandoff("codex", prep, effectiveSessionId, params.outputFormat, params.optimizePrompt);
-        let job;
-        try {
-            job = deps.asyncJobManager.startJob("codex", args, corrId, worktreeResolution.cwd, resolveIdleTimeout("codex", params.idleTimeoutMs), params.outputFormat, params.forceRefresh, undefined, prepCleanup, codexAsyncFrHandoff.flightRecorderEntry, codexAsyncFrHandoff.extractUsage, true, undefined, effectiveCompress);
-            prepCleanupOwnedHere = false;
-        }
-        catch (startErr) {
-            throw startErr;
-        }
+        const job = deps.asyncJobManager.startJob("codex", args, corrId, worktreeResolution.cwd, resolveIdleTimeout("codex", params.idleTimeoutMs), params.outputFormat, params.forceRefresh, undefined, prepCleanup, codexAsyncFrHandoff.flightRecorderEntry, codexAsyncFrHandoff.extractUsage, true, undefined, effectiveCompress);
+        prepCleanupOwnedHere = false;
         deps.logger.info(`[${corrId}] codex_request_async started job ${job.id}`);
         const asyncResponse = {
             success: true,
@@ -4766,6 +4787,439 @@ export async function handleCodexRequestAsync(deps, params) {
         return createErrorResponse("codex_request_async", 1, "", corrId, error);
     }
 }
+function routedOutputFormat(provider) {
+    return provider === "claude" || provider === "gemini" ? "stream-json" : "text";
+}
+function prepareRoutedCli(runtime, cli, params, outputFormat) {
+    const base = {
+        prompt: params.prompt,
+        promptParts: params.promptParts,
+        model: params.model,
+        correlationId: params.correlationId,
+        optimizePrompt: params.optimizePrompt,
+        operation: "route_request",
+        outputFormat,
+    };
+    switch (cli) {
+        case "claude": {
+            const p = {
+                ...base,
+                dangerouslySkipPermissions: false,
+                approvalStrategy: "legacy",
+                strictMcpConfig: false,
+            };
+            return prepareClaudeRequest(p, runtime);
+        }
+        case "codex": {
+            const p = {
+                ...base,
+                outputFormat: outputFormat,
+                fullAuto: false,
+                dangerouslyBypassApprovalsAndSandbox: false,
+                approvalStrategy: "legacy",
+            };
+            return prepareCodexRequest(p, runtime);
+        }
+        case "gemini": {
+            const p = { ...base, approvalStrategy: "legacy" };
+            return prepareGeminiRequest(p, runtime);
+        }
+        case "grok": {
+            const p = { ...base, approvalStrategy: "legacy" };
+            return prepareGrokRequest(p, runtime);
+        }
+        case "mistral": {
+            const p = { ...base, approvalStrategy: "legacy" };
+            return prepareMistralRequest(p, runtime);
+        }
+        case "devin": {
+            const p = { ...base };
+            return prepareDevinRequest(p, runtime);
+        }
+        case "cursor": {
+            const p = { ...base };
+            return prepareCursorRequest(p, runtime);
+        }
+        default: {
+            const exhaustive = cli;
+            return exhaustive;
+        }
+    }
+}
+async function dispatchRoutedCli(runtime, cli, params) {
+    const startTime = Date.now();
+    let durationMs = 0;
+    let wasSuccessful = false;
+    const outputFormat = routedOutputFormat(cli);
+    const prep = prepareRoutedCli(runtime, cli, params, outputFormat);
+    if (!("args" in prep))
+        return prep;
+    const { corrId, args } = prep;
+    const prepCleanup = "cleanup" in prep && typeof prep.cleanup === "function"
+        ? prep.cleanup
+        : undefined;
+    safeFlightStart({
+        correlationId: corrId,
+        cli,
+        model: prep.resolvedModel || "default",
+        prompt: prep.effectivePrompt,
+        stablePrefixHash: prep.stablePrefixHash ?? undefined,
+        stablePrefixTokens: prep.stablePrefixTokens ?? undefined,
+    }, runtime);
+    try {
+        const effectiveCompress = resolveEffectiveCompression(runtime.compression, {
+            compressResponse: params.compressResponse,
+            outputFormat,
+            outputSchemaDeclared: false,
+        });
+        const frHandoff = buildAsyncFlightRecorderHandoff(cli, prep, undefined, outputFormat, params.optimizePrompt);
+        const result = await awaitJobOrDefer(cli, args, corrId, resolveIdleTimeout(cli, undefined), outputFormat, params.forceRefresh, runtime, undefined, prepCleanup, frHandoff.flightRecorderEntry, frHandoff.extractUsage, prep.stdinPayload, undefined, effectiveCompress);
+        if (isDeferredResponse(result)) {
+            return buildDeferredToolResponse(result, undefined);
+        }
+        const { stdout, stderr, code } = result;
+        durationMs = Math.max(0, Date.now() - startTime);
+        if (code !== 0) {
+            safeFlightComplete(corrId, {
+                response: stderr || "",
+                durationMs,
+                retryCount: 0,
+                circuitBreakerState: "closed",
+                optimizationApplied: params.optimizePrompt || params.optimizeResponse,
+                exitCode: code,
+                errorMessage: stderr || `Exit code ${code}`,
+                status: "failed",
+            }, runtime);
+            return createErrorResponse(cli, code, stderr, corrId);
+        }
+        wasSuccessful = true;
+        const claudeStream = cli === "claude" && outputFormat === "stream-json";
+        const parsedClaude = claudeStream ? parseStreamJson(stdout) : null;
+        const responseText = parsedClaude ? parsedClaude.text : stdout;
+        const usage = parsedClaude
+            ? {
+                inputTokens: parsedClaude.usage?.inputTokens,
+                outputTokens: parsedClaude.usage?.outputTokens,
+                cacheReadTokens: parsedClaude.usage?.cacheReadInputTokens || undefined,
+                cacheCreationTokens: parsedClaude.usage?.cacheCreationInputTokens || undefined,
+                costUsd: parsedClaude.costUsd ?? undefined,
+            }
+            : extractUsageAndCost(cli, stdout, outputFormat);
+        const meta = extractProviderOutputMetadata(cli, stdout, outputFormat);
+        const { costUsd: recordedCostUsd, costBasis } = deriveCostBasis(cli, prep.resolvedModel || "default", usage);
+        safeFlightComplete(corrId, {
+            response: cli === "codex" ? codexFrResponse(outputFormat, stdout) : responseText,
+            durationMs,
+            retryCount: 0,
+            circuitBreakerState: "closed",
+            optimizationApplied: params.optimizePrompt || params.optimizeResponse,
+            exitCode: 0,
+            status: "completed",
+            inputTokens: usage.inputTokens,
+            outputTokens: usage.outputTokens,
+            cacheReadTokens: usage.cacheReadTokens,
+            cacheCreationTokens: usage.cacheCreationTokens,
+            costUsd: recordedCostUsd,
+            costBasis,
+            providerSessionId: parsedClaude ? (parsedClaude.sessionId ?? undefined) : meta.sessionId,
+            stopReason: parsedClaude ? (parsedClaude.stopReason ?? undefined) : meta.stopReason,
+        }, runtime);
+        const response = buildCliResponse(cli, responseText, params.optimizeResponse, corrId, undefined, prep, durationMs, undefined, outputFormat, undefined, effectiveCompress);
+        safeRecordCompression(corrId, response.compression, runtime);
+        return response;
+    }
+    catch (error) {
+        const elapsedMs = Math.max(0, Date.now() - startTime);
+        safeFlightComplete(corrId, {
+            response: "",
+            durationMs: elapsedMs,
+            retryCount: 0,
+            circuitBreakerState: "closed",
+            optimizationApplied: params.optimizePrompt || params.optimizeResponse,
+            exitCode: 1,
+            errorMessage: error.message,
+            status: "failed",
+        }, runtime);
+        return createErrorResponse(cli, 1, "", corrId, error);
+    }
+    finally {
+        const finalizedDurationMs = Math.max(0, durationMs || Date.now() - startTime);
+        runtime.performanceMetrics.recordRequest(cli, finalizedDurationMs, wasSuccessful);
+    }
+}
+async function dispatchRoutedRequest(runtime, chosen, params) {
+    if (isCliTypeValue(chosen.provider)) {
+        return dispatchRoutedCli(runtime, chosen.provider, params);
+    }
+    const api = enabledApiProviders(runtime.providers).find(p => p.name === chosen.provider);
+    if (!api) {
+        return createErrorResponse("route_request", 1, `Routed provider '${chosen.provider}' is not an enabled API provider`, params.correlationId);
+    }
+    const apiParams = {
+        prompt: params.prompt,
+        promptParts: params.promptParts,
+        model: chosen.model,
+        correlationId: params.correlationId,
+        optimizePrompt: params.optimizePrompt,
+        optimizeResponse: params.optimizeResponse,
+        compressResponse: params.compressResponse,
+    };
+    return handleApiProviderRequest(runtime, api, apiParams);
+}
+function isCliTypeValue(provider) {
+    return CLI_TYPES.includes(provider);
+}
+function routeCandidateKey(c) {
+    return `${c.provider}:${c.model}`;
+}
+export function isTransientRouteFailure(provider, response) {
+    const breakerOpen = isCliTypeValue(provider)
+        ? cliBreakerState(provider) !== "CLOSED"
+        : apiProviderBreakerState(provider) !== "CLOSED";
+    if (breakerOpen)
+        return true;
+    const sc = response.structuredContent;
+    if (isDefaultTransient({ code: sc?.exitCode }))
+        return true;
+    const haystack = `${response.content.map(c => c.text).join(" ")} ${sc?.errorCategory ?? ""}`;
+    return /\b(ETIMEDOUT|ECONNRESET|ECONNREFUSED|EPIPE)\b/i.test(haystack);
+}
+function attachRouting(response, block) {
+    response.routing = block;
+    response.structuredContent = { ...(response.structuredContent ?? {}), routing: block };
+    const chosen = block.chosen ? `${block.chosen.provider}/${block.chosen.model}` : "none";
+    const cost = block.estCostUsd !== undefined ? `~$${block.estCostUsd.toFixed(6)}` : "unpriced";
+    const basis = block.costBasis ? ` (${block.costBasis}, ${block.confidence})` : "";
+    const banner = `[routing] chosen=${chosen} est=${cost}${basis} considered=${block.consideredCount} reroutes=${block.reroutes}`;
+    const first = response.content[0];
+    if (first && first.type === "text") {
+        first.text = `${banner}\n${first.text}`;
+    }
+    else {
+        response.content.unshift({ type: "text", text: banner });
+    }
+    return response;
+}
+function buildRoutingBlock(decision, reroutes) {
+    return {
+        chosen: decision.chosen ? { ...decision.chosen } : null,
+        tier: decision.tier,
+        estCostUsd: decision.estCostUsd,
+        costBasis: decision.costBasis,
+        confidence: decision.confidence,
+        nearTie: decision.nearTie,
+        estInputTokens: decision.estInputTokens,
+        estOutputTokens: decision.estOutputTokens,
+        priceAsOf: decision.priceAsOf,
+        priceSource: decision.priceSource,
+        consideredCount: decision.consideredCount,
+        rejected: decision.rejected.map(r => ({ candidate: { ...r.candidate }, reason: r.reason })),
+        reroutes,
+        error: decision.error,
+    };
+}
+export function deriveCostBasis(provider, resolvedModel, usage) {
+    if (usage.costUsd != null) {
+        return { costUsd: usage.costUsd, costBasis: "provider-reported" };
+    }
+    const hasCounts = usage.inputTokens != null || usage.outputTokens != null;
+    if (hasCounts) {
+        const modelCost = getModelCost(provider, resolvedModel);
+        if (modelCost.source !== "unknown") {
+            const counts = {
+                inputTokens: usage.inputTokens ?? 0,
+                outputTokens: usage.outputTokens ?? 0,
+                cacheReadTokens: usage.cacheReadTokens,
+                cacheCreationTokens: usage.cacheCreationTokens,
+            };
+            const composed = composeCost(counts, { estInputTokens: 0, estOutputTokens: 0 }, modelCost);
+            return { costUsd: composed.costUsd, costBasis: composed.cost_basis };
+        }
+    }
+    return {};
+}
+function recordRoutingDecision(runtime, correlationId, decision, reroutes) {
+    const reason = decision.chosen
+        ? `cheapest${decision.nearTie ? ":near-tie" : ""}`
+        : (decision.error ?? "no-eligible");
+    safeRecordRouting(correlationId, {
+        estCostUsd: decision.estCostUsd ?? null,
+        estConfidence: decision.confidence ?? null,
+        reason,
+        considered: decision.consideredCount,
+        reroutes,
+    }, runtime);
+}
+function withDispatchRejections(decision, dispatchRejections) {
+    if (dispatchRejections.length === 0)
+        return decision;
+    const seen = new Set(decision.rejected.map(r => routeCandidateKey(r.candidate)));
+    const extra = dispatchRejections.filter(r => !seen.has(routeCandidateKey(r.candidate)));
+    return { ...decision, rejected: [...decision.rejected, ...extra] };
+}
+function routeErrorResponse(decision, reroutes, corrId, lastFailure) {
+    const reasons = decision.rejected
+        .map(r => `${routeCandidateKey(r.candidate)} (${r.reason})`)
+        .join(", ");
+    const kind = decision.error ?? "NoEligibleCandidate";
+    const base = kind === "BudgetExceeded"
+        ? `route_request: no candidate fits the budget. Rejected: ${reasons || "none eligible"}.`
+        : `route_request: no eligible candidate. Rejected: ${reasons || "empty pool"}.`;
+    const diag = lastFailure?.content?.[0]?.text
+        ? `\nLast dispatch failure: ${lastFailure.content[0].text}`
+        : "";
+    const resp = createErrorResponse("route_request", 1, `${base}${diag}`, corrId);
+    return attachRouting(resp, buildRoutingBlock({ ...decision, error: kind }, reroutes));
+}
+async function runRouteRequest(runtime, params) {
+    const corrId = params.correlationId ?? randomUUID();
+    const cfg = runtime.leastCost;
+    const { env, routerConfig, req } = buildRouteContext(runtime, params);
+    const excluded = new Set();
+    const dispatchRejections = [];
+    let reroutes = 0;
+    let transientReroutes = 0;
+    let lastFailure = null;
+    let lastDecision = null;
+    const maxIterations = env.providers().length * 8 + cfg.maxReroutes + 2;
+    for (let i = 0; i < maxIterations; i++) {
+        const decision = selectCandidate(req, env, routerConfig, excluded);
+        lastDecision = decision;
+        if (!decision.chosen) {
+            return routeErrorResponse(withDispatchRejections(decision, dispatchRejections), reroutes, corrId, lastFailure);
+        }
+        const chosen = decision.chosen;
+        const attemptCorrId = i === 0 ? corrId : randomUUID();
+        const dispatchParams = {
+            prompt: params.prompt,
+            promptParts: params.promptParts,
+            model: chosen.model,
+            correlationId: attemptCorrId,
+            optimizePrompt: params.optimizePrompt ?? false,
+            optimizeResponse: params.optimizeResponse ?? false,
+            compressResponse: params.compressResponse,
+            forceRefresh: params.forceRefresh,
+        };
+        const result = await dispatchRoutedRequest(runtime, chosen, dispatchParams);
+        if (!result.isError) {
+            recordRoutingDecision(runtime, attemptCorrId, decision, reroutes);
+            return attachRouting(result, buildRoutingBlock(decision, reroutes));
+        }
+        lastFailure = result;
+        const transient = isTransientRouteFailure(chosen.provider, result);
+        excluded.add(routeCandidateKey(chosen));
+        dispatchRejections.push({
+            candidate: chosen,
+            reason: transient ? "dispatch-failed:transient" : "dispatch-failed:non-transient",
+        });
+        reroutes++;
+        if (transient) {
+            transientReroutes++;
+            if (transientReroutes > cfg.maxReroutes) {
+                return attachRouting(result, buildRoutingBlock(decision, reroutes));
+            }
+        }
+    }
+    if (lastDecision) {
+        return routeErrorResponse(withDispatchRejections(lastDecision, dispatchRejections), reroutes, corrId, lastFailure);
+    }
+    return createErrorResponse("route_request", 1, "route_request: routing did not converge", corrId);
+}
+function buildRouteContext(runtime, params) {
+    return {
+        env: buildRouterEnv({
+            performanceMetrics: runtime.performanceMetrics,
+            limiterSnapshot: runtime.asyncJobManager.getLimiterSnapshot(),
+            apiProviders: enabledApiProviders(runtime.providers),
+            preferCatalogPrice: runtime.leastCost.preferCatalogPrice,
+            flightRecorder: runtime.flightRecorder,
+            priorsScope: runtime.leastCost.priorsScope,
+            ownerPrincipal: resolveOwnerPrincipal(getRequestContext()),
+        }),
+        routerConfig: toRouterConfig(runtime.leastCost),
+        req: {
+            prompt: params.prompt ?? "",
+            candidates: params.candidates,
+            minTier: params.minTier,
+            maxCostUsd: params.maxCostUsd,
+            expectedOutputTokens: params.expectedOutputTokens,
+            maxOutputTokens: params.maxOutputTokens,
+            requiredCapabilities: params.requiredCapabilities,
+            allowUnpriced: params.allowUnpriced,
+            budgetWaiver: params.budgetWaiver,
+            fallback: params.fallback,
+        },
+    };
+}
+async function dispatchRoutedCliAsync(runtime, cli, params) {
+    const outputFormat = routedOutputFormat(cli);
+    const prep = prepareRoutedCli(runtime, cli, params, outputFormat);
+    if (!("args" in prep))
+        return prep;
+    const { corrId, args } = prep;
+    const prepCleanup = "cleanup" in prep && typeof prep.cleanup === "function"
+        ? prep.cleanup
+        : undefined;
+    try {
+        const effectiveCompress = resolveEffectiveCompression(runtime.compression, {
+            compressResponse: params.compressResponse,
+            outputFormat,
+            outputSchemaDeclared: false,
+        });
+        const frHandoff = buildAsyncFlightRecorderHandoff(cli, prep, undefined, outputFormat, params.optimizePrompt);
+        const job = runtime.asyncJobManager.startJob(cli, args, corrId, undefined, resolveIdleTimeout(cli, undefined), outputFormat, params.forceRefresh, undefined, prepCleanup, frHandoff.flightRecorderEntry, frHandoff.extractUsage, true, prep.stdinPayload, effectiveCompress);
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: JSON.stringify({ success: true, job, sessionId: null }, null, 2),
+                },
+            ],
+        };
+    }
+    catch (error) {
+        return createErrorResponse("route_request_async", 1, "", corrId, error);
+    }
+}
+async function runRouteRequestAsync(runtime, params) {
+    const corrId = params.correlationId ?? randomUUID();
+    const { env, routerConfig, req } = buildRouteContext(runtime, params);
+    const decision = selectCandidate(req, env, routerConfig);
+    if (!decision.chosen) {
+        return routeErrorResponse(decision, 0, corrId);
+    }
+    const chosen = decision.chosen;
+    const dispatchParams = {
+        prompt: params.prompt,
+        promptParts: params.promptParts,
+        model: chosen.model,
+        correlationId: corrId,
+        optimizePrompt: params.optimizePrompt ?? false,
+        optimizeResponse: params.optimizeResponse ?? false,
+        compressResponse: params.compressResponse,
+        forceRefresh: params.forceRefresh,
+    };
+    let result;
+    if (isCliTypeValue(chosen.provider)) {
+        result = await dispatchRoutedCliAsync(runtime, chosen.provider, dispatchParams);
+    }
+    else {
+        const api = enabledApiProviders(runtime.providers).find(p => p.name === chosen.provider);
+        if (!api) {
+            return createErrorResponse("route_request_async", 1, `Routed provider '${chosen.provider}' is not an enabled API provider`, corrId);
+        }
+        result = await handleApiProviderRequestAsync(runtime, api, {
+            prompt: params.prompt,
+            promptParts: params.promptParts,
+            model: chosen.model,
+            correlationId: corrId,
+            optimizePrompt: params.optimizePrompt ?? false,
+            optimizeResponse: params.optimizeResponse ?? false,
+        });
+    }
+    recordRoutingDecision(runtime, corrId, decision, 0);
+    return attachRouting(result, buildRoutingBlock(decision, 0));
+}
 export function createGatewayServer(deps = {}) {
     const runtime = resolveGatewayServerRuntime(deps, { isolateState: true });
     const { sessionManager, asyncJobManager, approvalManager, performanceMetrics, logger, persistence, flightRecorder, cacheAwareness, providers, } = runtime;
@@ -4779,6 +5233,9 @@ export function createGatewayServer(deps = {}) {
         asyncJobManager,
         apiProviders: enabledApiProviders(providers),
         validationRunStore: asyncJobManager.getValidationRunStore(),
+        leastCost: runtime.leastCost,
+        performanceMetrics: runtime.performanceMetrics,
+        flightRecorder: runtime.flightRecorder,
     });
     registerWorkspaceTools(server, runtime);
     registerProviderAdminTools(server, {
@@ -4790,6 +5247,93 @@ export function createGatewayServer(deps = {}) {
     if (apiProviderTools.length > 0) {
         runtime.logger.info(`Registered API provider tools: ${apiProviderTools.join(", ")}`);
     }
+    if (runtime.leastCost.enabled) {
+        const RouteCandidateShape = z.object({
+            provider: z.string().min(1).describe("A CLI_TYPES member or an enabled API-provider name."),
+            model: z.string().min(1).describe("A concrete model id/alias that provider can serve."),
+        });
+        const RouteRequiredCapabilitiesShape = z
+            .object({
+            images: z.boolean().optional(),
+            attachments: z.boolean().optional(),
+            toolCalling: z.boolean().optional(),
+            jsonSchema: z.boolean().optional(),
+            outputFormat: z.string().optional(),
+            effort: z.string().optional(),
+        })
+            .describe("Hard capability constraints; a candidate missing any required capability is excluded.");
+        const routeInputShape = {
+            prompt: z
+                .string()
+                .min(1)
+                .max(100000)
+                .optional()
+                .describe("Prompt text to route (mutually exclusive with promptParts)."),
+            promptParts: PromptPartsSchema.optional().describe("Cache-aware structured prompt (mutually exclusive with prompt)."),
+            candidates: z
+                .array(RouteCandidateShape)
+                .optional()
+                .describe("Explicit (provider, model) pool to restrict routing to; also whitelists untiered / maintain-only candidates. Empty/omitted = all eligible."),
+            minTier: z
+                .enum(["economy", "standard", "frontier"])
+                .optional()
+                .describe("Minimum quality tier floor (default from config, usually 'standard'). No silent downgrade below it."),
+            maxCostUsd: z
+                .number()
+                .positive()
+                .optional()
+                .describe("Per-request budget cap (USD). Over-budget fails closed unless waived via an explicit fallback + budgetWaiver."),
+            maxOutputTokens: z
+                .number()
+                .int()
+                .positive()
+                .optional()
+                .describe("Caller output cap; bounds the conservative budget estimate."),
+            expectedOutputTokens: z
+                .number()
+                .int()
+                .positive()
+                .optional()
+                .describe("Expected output tokens for the ranking estimate (default from config)."),
+            requiredCapabilities: RouteRequiredCapabilitiesShape.optional(),
+            allowUnpriced: z
+                .boolean()
+                .optional()
+                .describe("Admit unpriced (source 'unknown') candidates. They still rank strictly last and cannot pass the budget gate without budgetWaiver."),
+            budgetWaiver: z
+                .boolean()
+                .optional()
+                .describe("Explicit acknowledgment to admit an unpriced or over-budget fallback candidate (with allowUnpriced / fallback)."),
+            fallback: RouteCandidateShape.optional().describe("A pinned (provider, model) used only when the eligible pool is empty; bypasses cost ranking but still passes eligibility + budget (unless budgetWaiver)."),
+            optimizePrompt: z
+                .boolean()
+                .optional()
+                .describe("Apply prompt token optimization before dispatch."),
+            optimizeResponse: z.boolean().optional().describe("Apply response token optimization."),
+            compressResponse: z
+                .boolean()
+                .optional()
+                .describe("Apply the native display-text compressor to the response."),
+            correlationId: z.string().optional().describe("Caller-supplied correlation id for tracing."),
+            forceRefresh: z
+                .boolean()
+                .optional()
+                .describe("Bypass the async dedup window for this request."),
+        };
+        const routeAnnotations = {
+            readOnlyHint: false,
+            destructiveHint: true,
+            idempotentHint: false,
+            openWorldHint: true,
+        };
+        server.tool("route_request", "Least-cost routing: pick the cheapest eligible (provider, model) candidate that meets the caller's quality tier, capability, and budget constraints, then dispatch it through the normal provider path. Returns the provider response plus a `routing` block (chosen candidate, cost estimate + basis/confidence, rejections, reroutes). Registered only when [least_cost].enabled is true.", routeInputShape, { title: "Least-cost route (sync)", ...routeAnnotations }, async (params) => runRouteRequest(runtime, params));
+        let registeredRouteTools = "route_request";
+        if (asyncJobsEnabled) {
+            server.tool("route_request_async", "Async least-cost routing: select the cheapest eligible (provider, model) candidate synchronously, enqueue it for background execution, and return a pollable job handle plus the `routing` block. Poll with llm_job_status, collect with llm_job_result. Registered only when [least_cost].enabled is true and async jobs are enabled.", routeInputShape, { title: "Least-cost route (async)", ...routeAnnotations }, async (params) => runRouteRequestAsync(runtime, params));
+            registeredRouteTools += ", route_request_async";
+        }
+        runtime.logger.info(`Registered least-cost routing tools: ${registeredRouteTools}`);
+    }
     if (grokApiToolsEnabled) {
         server.tool("grok_api_request", "Run an xAI Grok API request synchronously through the Responses API. Requires exactly one of prompt or promptParts. Registered only when [providers.xai] is configured and its API-key env var is present.", {
             prompt: z
@@ -5181,6 +5725,13 @@ export function createGatewayServer(deps = {}) {
                 if (parsed.costUsd !== null) {
                     logger.debug(`[${corrId}] stream-json cost=$${parsed.costUsd}, model=${parsed.model}, turns=${parsed.numTurns}`);
                 }
+                const { costUsd: claudeCostUsd, costBasis: claudeCostBasis } = deriveCostBasis("claude", prep.resolvedModel || "default", {
+                    inputTokens: parsed.usage?.inputTokens,
+                    outputTokens: parsed.usage?.outputTokens,
+                    cacheReadTokens: parsed.usage?.cacheReadInputTokens || undefined,
+                    cacheCreationTokens: parsed.usage?.cacheCreationInputTokens || undefined,
+                    costUsd: parsed.costUsd ?? undefined,
+                });
                 safeFlightComplete(corrId, {
                     response: parsed.text,
                     inputTokens: parsed.usage?.inputTokens,
@@ -5190,7 +5741,8 @@ export function createGatewayServer(deps = {}) {
                     durationMs,
                     retryCount: 0,
                     circuitBreakerState: "closed",
-                    costUsd: parsed.costUsd ?? undefined,
+                    costUsd: claudeCostUsd,
+                    costBasis: claudeCostBasis,
                     optimizationApplied: optimizePrompt || optimizeResponse,
                     exitCode: 0,
                     status: "completed",
@@ -5432,7 +5984,7 @@ export function createGatewayServer(deps = {}) {
         }, runtime);
         logger.info(`[${corrId}] codex_request invoked with model=${prep.resolvedModel || "default"}, fullAuto=${fullAuto}, prompt length=${prep.effectivePrompt.length}`);
         const prepCleanup = "cleanup" in prep && typeof prep.cleanup === "function" ? prep.cleanup : undefined;
-        let worktreeResolution = {};
+        let worktreeResolution;
         try {
             worktreeResolution = await resolveWorkspaceAndWorktreeForRequest({
                 provider: "codex",
@@ -5505,6 +6057,7 @@ export function createGatewayServer(deps = {}) {
             }
             logger.info(`[${corrId}] codex_request completed successfully in ${durationMs}ms`);
             const codexUsage = extractUsageAndCost("codex", stdout, outputFormat);
+            const { costUsd: codexCostUsd, costBasis: codexCostBasis } = deriveCostBasis("codex", prep.resolvedModel || "default", codexUsage);
             const codexMeta = extractProviderOutputMetadata("codex", stdout, outputFormat);
             safeFlightComplete(corrId, {
                 response: codexFrResponse(outputFormat, stdout),
@@ -5518,7 +6071,8 @@ export function createGatewayServer(deps = {}) {
                 outputTokens: codexUsage.outputTokens,
                 cacheReadTokens: codexUsage.cacheReadTokens,
                 cacheCreationTokens: codexUsage.cacheCreationTokens,
-                costUsd: codexUsage.costUsd,
+                costUsd: codexCostUsd,
+                costBasis: codexCostBasis,
                 providerSessionId: codexMeta.sessionId,
                 stopReason: codexMeta.stopReason,
             }, runtime);
@@ -7683,21 +8237,28 @@ export function createGatewayServer(deps = {}) {
     }, async () => {
         const health = asyncJobManager.getJobHealth();
         const storeAttached = asyncJobManager.hasStore();
-        const asyncJobsEffective = persistence.backend !== "none" && persistence.asyncJobsEnabled && storeAttached;
+        const durableAdmission = asyncJobManager.getDurableAdmissionHealth();
+        const asyncJobsConfigured = persistence.backend !== "none" && persistence.asyncJobsEnabled;
+        const asyncJobsEffective = asyncJobsConfigured && storeAttached && durableAdmission.admitting;
         const persistenceBlock = {
             backend: persistence.backend,
             dbPath: persistence.path,
             dsn: persistence.dsn ? "[redacted]" : null,
             retentionDays: persistence.retentionDays,
             dedupWindowMs: persistence.dedupWindowMs,
+            asyncJobsConfigured,
+            storeAttached,
             asyncJobsEnabled: asyncJobsEffective,
+            durableAdmission,
             acknowledgeEphemeral: persistence.acknowledgeEphemeral,
             sources: persistence.sources,
             warning: asyncJobsEffective
                 ? null
                 : persistence.backend === "none"
                     ? "Async job persistence is disabled (backend = 'none'). *_request_async tools are NOT registered on this gateway. Set [persistence].backend = 'sqlite' (or 'memory' + acknowledgeEphemeral = true) to enable them."
-                    : `Async job persistence is configured (backend = '${persistence.backend}') but the durable job store failed to open, so *_request_async tools are NOT registered on this gateway. Check gateway startup logs for the store-open error.`,
+                    : !storeAttached
+                        ? `Async job persistence is configured (backend = '${persistence.backend}') but the durable job store failed to open, so *_request_async / llm_job_* tools are NOT registered on this gateway. Check gateway startup logs for the store-open error.`
+                        : "Async job persistence is attached but durable admission is temporarily disabled while its heartbeat lease recovers. Existing async tools fail closed until admission is restored.",
         };
         const outboundProviders = {
             xai: providers.xai
@@ -7763,6 +8324,46 @@ export function createGatewayServer(deps = {}) {
                 external: mem.external,
             },
         };
+        const leastCostBlock = (() => {
+            const lc = runtime.leastCost;
+            if (!lc.enabled)
+                return { enabled: false };
+            const env = buildRouterEnv({
+                performanceMetrics: runtime.performanceMetrics,
+                limiterSnapshot: runtime.asyncJobManager.getLimiterSnapshot(),
+                apiProviders: enabledApiProviders(runtime.providers),
+                preferCatalogPrice: lc.preferCatalogPrice,
+            });
+            const providerHealth = env.providers().map(provider => ({
+                provider,
+                telemetryTier: telemetryTierFor(provider),
+                authed: env.isAuthed(provider),
+                breakerState: env.breakerState(provider),
+                atCapacity: env.atCapacity(provider),
+                candidates: env.models(provider).map(model => {
+                    const mc = env.modelCost(provider, model);
+                    const family = modelIdToFamily(model);
+                    const tier = lc.tiers[`${provider}:${family}`];
+                    return {
+                        model,
+                        family: mc.family,
+                        priced: mc.source !== "unknown",
+                        priceSource: mc.source,
+                        tier: tier ?? null,
+                    };
+                }),
+            }));
+            return {
+                enabled: true,
+                minTier: lc.minTier,
+                maxCostUsd: lc.maxCostUsd,
+                priorsScope: lc.priorsScope,
+                preferCatalogPrice: lc.preferCatalogPrice,
+                allowUnpriced: lc.allowUnpriced,
+                maxReroutes: lc.maxReroutes,
+                providers: providerHealth,
+            };
+        })();
         return {
             content: [
                 {
@@ -7773,6 +8374,7 @@ export function createGatewayServer(deps = {}) {
                         backpressure,
                         persistence: persistenceBlock,
                         outboundProviders,
+                        leastCost: leastCostBlock,
                     }, null, 2),
                 },
             ],
@@ -8441,7 +9043,7 @@ async function initializeSessionManager() {
         });
         logger.info("File-based session manager initialized");
     }
-    resourceProvider = new ResourceProvider(sessionManager, performanceMetrics, getFlightRecorder(logger), getCacheAwarenessConfig(logger), getProvidersConfig(logger), undefined, getAcpConfig(logger));
+    resourceProvider = new ResourceProvider(sessionManager, performanceMetrics, getFlightRecorder(logger), getCacheAwarenessConfig(logger), getProvidersConfig(logger), undefined, getAcpConfig(logger), getLeastCostConfig(logger));
 }
 function registerHealthResource(server) {
     if (db) {
@@ -8468,20 +9070,26 @@ function registerHealthResource(server) {
         description: "Async job health (CPU, memory, zombie detection)",
         mimeType: "application/json",
     }, async (uri) => {
-        const health = getAsyncJobManager().getJobHealth();
+        const manager = getAsyncJobManager();
+        const health = manager.getJobHealth();
         return {
             contents: [
                 {
                     uri: uri.href,
                     mimeType: "application/json",
-                    text: JSON.stringify(health, null, 2),
+                    text: JSON.stringify({ ...health, durableAdmission: manager.getDurableAdmissionHealth() }, null, 2),
                 },
             ],
         };
     });
     logger.info("Process health resource registered");
 }
-async function shutdown(signal) {
+let shutdownPromise = null;
+function shutdown(signal, exitCode = 0) {
+    shutdownPromise ??= performShutdown(signal, exitCode);
+    return shutdownPromise;
+}
+async function performShutdown(signal, exitCode) {
     logger.info(`Received ${signal}, shutting down gracefully...`);
     try {
         if (asyncJobManager) {
@@ -8505,6 +9113,12 @@ async function shutdown(signal) {
             logger.info("MCP server closed");
             activeServer = null;
         }
+        if (jobStore) {
+            jobStore.close();
+            logger.info("Durable job store closed");
+            jobStore = null;
+            jobStoreInitialized = false;
+        }
         if (db) {
             await db.disconnect();
             logger.info("Database connections closed");
@@ -8513,7 +9127,7 @@ async function shutdown(signal) {
             flightRecorder.close();
             logger.info("Flight recorder closed");
         }
-        process.exit(0);
+        process.exit(exitCode);
     }
     catch (error) {
         logger.error("Error during shutdown:", error);
@@ -8907,15 +9521,23 @@ async function main() {
     logger.info(`Starting llm-cli-gateway MCP server with ${transportMode} transport`);
     await initializeSessionManager();
     void warmProviderCapabilities({ logger }).catch(() => undefined);
+    const persistence = getPersistenceConfig(logger);
+    const runtimeAsyncJobManager = getAsyncJobManager(logger);
+    if (persistence.backend !== "none" &&
+        persistence.asyncJobsEnabled &&
+        !runtimeAsyncJobManager.hasStore()) {
+        throw new Error(`Configured ${persistence.backend} job store could not be opened; refusing to start without durable async persistence`);
+    }
     const serverDeps = {
         sessionManager,
         resourceProvider,
         db,
         performanceMetrics,
-        asyncJobManager: getAsyncJobManager(logger),
+        asyncJobManager: runtimeAsyncJobManager,
         approvalManager: getApprovalManager(logger),
         flightRecorder: getFlightRecorder(logger),
         logger,
+        persistence,
     };
     if (transportMode === "http") {
         activeHttpGateway = await startHttpGateway({
@@ -8932,6 +9554,12 @@ async function main() {
     activeServer = createGatewayServer({
         ...serverDeps,
     });
+    activeServer.server.onclose = () => {
+        void shutdown("stdio MCP connection closed");
+    };
+    process.stdin.once("end", () => {
+        void shutdown("stdio stdin EOF");
+    });
     registerHealthResource(activeServer);
     const transport = new StdioServerTransport();
     await activeServer.connect(transport);
@@ -8941,6 +9569,6 @@ const __entryUrl = entrypointFileURL(process.argv[1]);
 if (__entryUrl === import.meta.url) {
     main().catch(error => {
         logger.error("Fatal server error:", error);
-        process.exit(1);
+        void shutdown("fatal startup error", 1);
     });
 }