llm-simple-router 0.3.7 → 0.4.0

package/dist/proxy/proxy-logging.js

@@ -0,0 +1,122 @@
+import { randomUUID } from "crypto";
+import { insertRequestLog, insertMetrics } from "../db/index.js";
+import { insertSuccessLog } from "./log-helpers.js";
+import { MetricsExtractor } from "../metrics/metrics-extractor.js";
+// Re-export shared constants & types from types.ts (canonical home)
+export { UPSTREAM_SUCCESS } from "./types.js";
+// Internal imports from types.ts
+import { UPSTREAM_SUCCESS } from "./types.js";
+import { HTTP_BAD_GATEWAY } from "../constants.js";
+// ---------- Header sanitization ----------
+const AUTH_HEADER_RE = /^authorization$/i;
+/** 日志存储前脱敏 Authorization header，避免 API Key 被持久化 */
+export function sanitizeHeadersForLog(headers) {
+    const sanitized = {};
+    for (const [key, value] of Object.entries(headers)) {
+        sanitized[key] = AUTH_HEADER_RE.test(key) ? value.replace(/(Bearer\s+)\S+/, "$1sk-***") : value;
+    }
+    return sanitized;
+}
+// ---------- Logging helpers (extracted from proxy-core) ----------
+export function handleIntercept(db, apiType, request, reply, interceptResponse, clientModel) {
+    const logId = randomUUID();
+    const isStream = request.body.stream === true;
+    const respBody = JSON.stringify(interceptResponse.body);
+    insertRequestLog(db, {
+        id: logId, api_type: apiType, model: clientModel, provider_id: "router",
+        status_code: interceptResponse.statusCode, latency_ms: 0,
+        is_stream: isStream ? 1 : 0, error_message: null,
+        created_at: new Date().toISOString(),
+        request_body: JSON.stringify(request.body),
+        response_body: respBody,
+        client_request: JSON.stringify({ headers: request.headers, body: request.body }),
+        upstream_request: interceptResponse.meta ? JSON.stringify(interceptResponse.meta) : null,
+        client_response: JSON.stringify({ statusCode: interceptResponse.statusCode, body: respBody }),
+        is_retry: 0, is_failover: 0, original_request_id: null,
+        router_key_id: request.routerKey?.id ?? null, original_model: null,
+    });
+    return reply.status(interceptResponse.statusCode).send(interceptResponse.body);
+}
+// ---------- New-architecture logging ----------
+export function logResilienceResult(db, params, attempts, result, startTime) {
+    const isFailoverIteration = params.failover?.isFailoverIteration ?? false;
+    const rootLogId = params.failover?.rootLogId ?? params.logId;
+    let lastSuccessLogId = params.logId;
+    for (const attempt of attempts) {
+        const isOriginal = attempt.attemptIndex === 0;
+        const attemptLogId = isOriginal ? params.logId : randomUUID();
+        const isFailoverLog = isOriginal && isFailoverIteration;
+        const parentId = isOriginal ? (isFailoverIteration ? rootLogId : null) : params.logId;
+        if (attempt.error) {
+            insertRequestLog(db, {
+                id: attemptLogId, api_type: params.apiType, model: params.model,
+                provider_id: attempt.target.provider_id,
+                status_code: HTTP_BAD_GATEWAY, latency_ms: attempt.latencyMs,
+                is_stream: params.isStream ? 1 : 0, error_message: attempt.error,
+                created_at: new Date().toISOString(), request_body: params.reqBodyStr,
+                client_request: params.clientReq, upstream_request: params.upstreamReqBase,
+                is_retry: isOriginal ? 0 : 1, is_failover: isFailoverLog ? 1 : 0,
+                original_request_id: parentId,
+                router_key_id: params.routerKeyId, original_model: params.originalModel,
+            });
+        }
+        else if (attempt.statusCode !== UPSTREAM_SUCCESS) {
+            insertRequestLog(db, {
+                id: attemptLogId, api_type: params.apiType, model: params.model,
+                provider_id: attempt.target.provider_id,
+                status_code: attempt.statusCode, latency_ms: attempt.latencyMs,
+                is_stream: params.isStream ? 1 : 0, error_message: null,
+                created_at: new Date().toISOString(), request_body: params.reqBodyStr,
+                response_body: attempt.responseBody,
+                client_request: params.clientReq, upstream_request: params.upstreamReqBase,
+                upstream_response: JSON.stringify({ statusCode: attempt.statusCode, body: attempt.responseBody }),
+                client_response: JSON.stringify({ statusCode: attempt.statusCode, body: attempt.responseBody }),
+                is_retry: isOriginal ? 0 : 1, is_failover: isFailoverLog ? 1 : 0,
+                original_request_id: parentId,
+                router_key_id: params.routerKeyId, original_model: params.originalModel,
+            });
+        }
+        else {
+            const upHdrs = (result.kind === "stream_success" || result.kind === "stream_abort")
+                ? (result.upstreamResponseHeaders ?? {})
+                : ("headers" in result ? result.headers : {});
+            insertSuccessLog(db, {
+                apiType: params.apiType, model: params.model,
+                provider: { id: attempt.target.provider_id },
+                isStream: params.isStream, startTime,
+                reqBody: params.reqBodyStr, clientReq: params.clientReq,
+                upstreamReq: params.upstreamReqBase, id: attemptLogId,
+                status: attempt.statusCode, respBody: attempt.responseBody,
+                upHdrs, cliHdrs: upHdrs,
+                isRetry: !isOriginal, isFailover: isFailoverLog,
+                originalRequestId: parentId,
+                routerKeyId: params.routerKeyId, originalModel: params.originalModel,
+            });
+            lastSuccessLogId = attemptLogId;
+        }
+    }
+    return lastSuccessLogId;
+}
+export function collectTransportMetrics(db, apiType, result, isStream, lastSuccessLogId, providerId, backendModel, request) {
+    const base = { request_log_id: lastSuccessLogId, provider_id: providerId, backend_model: backendModel, api_type: apiType };
+    try {
+        if (isStream && (result.kind === "stream_success" || result.kind === "stream_abort")) {
+            if (result.metrics) {
+                insertMetrics(db, { ...base, ...result.metrics });
+                return;
+            }
+        }
+        else if (result.kind === "success") {
+            const mr = MetricsExtractor.fromNonStreamResponse(apiType, result.body);
+            if (mr) {
+                insertMetrics(db, { ...base, ...mr });
+                return;
+            }
+        }
+        // 无法提取完整 metrics 的 fallback，标记为未完成
+        insertMetrics(db, { ...base, is_complete: 0 });
+    }
+    catch (err) {
+        request.log.error({ err }, "Failed to insert metrics");
+    }
+}

package/dist/proxy/resilience.d.ts

@@ -0,0 +1,63 @@
+import type { RetryRuleMatcher } from "./retry-rules.js";
+import type { TransportResult } from "./types.js";
+import type { Target } from "./strategy/types.js";
+export interface RetryStrategy {
+    getDelay(attempt: number): number;
+}
+export declare class FixedIntervalStrategy implements RetryStrategy {
+    private delayMs;
+    constructor(delayMs: number);
+    getDelay(): number;
+}
+export declare class ExponentialBackoffStrategy implements RetryStrategy {
+    private baseMs;
+    private capMs;
+    constructor(baseMs: number, capMs: number);
+    getDelay(attempt: number): number;
+}
+export declare function createStrategy(rule: {
+    retry_strategy: string;
+    retry_delay_ms: number;
+    max_delay_ms: number;
+}): RetryStrategy;
+export interface ResilienceConfig {
+    maxRetries: number;
+    baseDelayMs: number;
+    failoverThreshold: number;
+    ruleMatcher?: RetryRuleMatcher;
+    isFailover: boolean;
+}
+export interface ResilienceAttempt {
+    target: Target;
+    attemptIndex: number;
+    statusCode: number | null;
+    error: string | null;
+    latencyMs: number;
+    responseBody: string | null;
+}
+export interface ResilienceResult {
+    result: TransportResult;
+    attempts: ResilienceAttempt[];
+    excludedTargets: Target[];
+}
+export type ResilienceDecision = {
+    action: "done";
+} | {
+    action: "retry";
+    delayMs: number;
+} | {
+    action: "failover";
+    excludeTarget: Target;
+} | {
+    action: "abort";
+    reason: string;
+};
+export interface ResilienceState {
+    attemptCount: number;
+    currentTarget: Target;
+    excludedTargets: Target[];
+}
+export declare class ResilienceLayer {
+    decide(result: TransportResult, state: ResilienceState, config: ResilienceConfig): ResilienceDecision;
+    execute(targets: () => Target[], fn: (target: Target) => Promise<TransportResult>, config: ResilienceConfig): Promise<ResilienceResult>;
+}

package/dist/proxy/resilience.js

@@ -0,0 +1,188 @@
+import { ProviderSwitchNeeded } from "./types.js";
+export class FixedIntervalStrategy {
+    delayMs;
+    constructor(delayMs) {
+        this.delayMs = delayMs;
+    }
+    getDelay() { return this.delayMs; }
+}
+const EXPONENTIAL_BASE = 2;
+export class ExponentialBackoffStrategy {
+    baseMs;
+    capMs;
+    constructor(baseMs, capMs) {
+        this.baseMs = baseMs;
+        this.capMs = capMs;
+    }
+    getDelay(attempt) {
+        return Math.min(this.baseMs * EXPONENTIAL_BASE ** attempt, this.capMs);
+    }
+}
+export function createStrategy(rule) {
+    if (rule.retry_strategy === "fixed")
+        return new FixedIntervalStrategy(rule.retry_delay_ms);
+    return new ExponentialBackoffStrategy(rule.retry_delay_ms, rule.max_delay_ms);
+}
+// ---------- Constants ----------
+const RETRYABLE_THROW_CODES = new Set(["ETIMEDOUT", "ECONNRESET", "ECONNREFUSED"]);
+const HTTP_TOO_MANY_REQUESTS = 429;
+const MS_PER_SECOND = 1000;
+// ---------- Internal helpers ----------
+function isRetryableThrow(err) {
+    if (err && typeof err === "object" && "code" in err) {
+        return RETRYABLE_THROW_CODES.has(err.code ?? "");
+    }
+    return false;
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function parseRetryAfter(headers) {
+    if (!headers)
+        return null;
+    const val = headers["retry-after"] ?? headers["Retry-After"];
+    if (!val)
+        return null;
+    const seconds = parseInt(val, 10);
+    return isNaN(seconds) ? null : seconds * MS_PER_SECOND;
+}
+function extractBody(result) {
+    if ("body" in result)
+        return result.body;
+    return null;
+}
+function extractHeaders(result) {
+    if ("headers" in result)
+        return result.headers;
+    return undefined;
+}
+// ---------- ResilienceLayer ----------
+export class ResilienceLayer {
+    decide(result, state, config) {
+        // stream_abort -> 不可恢复
+        if (result.kind === "stream_abort") {
+            return { action: "abort", reason: "stream_abort" };
+        }
+        // success + statusCode < failoverThreshold -> done
+        if ((result.kind === "success" || result.kind === "stream_success") &&
+            result.statusCode < config.failoverThreshold) {
+            return { action: "done" };
+        }
+        // throw -> 网络异常
+        if (result.kind === "throw") {
+            if (!isRetryableThrow(result.error)) {
+                return { action: "abort", reason: result.error.message };
+            }
+            if (state.attemptCount < config.maxRetries) {
+                return { action: "retry", delayMs: config.baseDelayMs };
+            }
+            return config.isFailover
+                ? { action: "failover", excludeTarget: state.currentTarget }
+                : { action: "abort", reason: "throw exhausted retries" };
+        }
+        // statusCode >= failoverThreshold -> retry or failover
+        if (result.statusCode >= config.failoverThreshold) {
+            const body = extractBody(result);
+            const matchedRule = body && config.ruleMatcher
+                ? config.ruleMatcher.match(result.statusCode, body)
+                : null;
+            const effectiveMaxRetries = Math.min(matchedRule?.max_retries ?? 0, config.maxRetries);
+            if (matchedRule && state.attemptCount < effectiveMaxRetries) {
+                const strategy = createStrategy(matchedRule);
+                const headers = extractHeaders(result);
+                const retryAfterMs = result.statusCode === HTTP_TOO_MANY_REQUESTS
+                    ? parseRetryAfter(headers) : null;
+                const delay = Math.max(strategy.getDelay(state.attemptCount), retryAfterMs ?? 0);
+                return { action: "retry", delayMs: delay };
+            }
+            return config.isFailover
+                ? { action: "failover", excludeTarget: state.currentTarget }
+                : { action: "done" };
+        }
+        // 其他响应（< failoverThreshold 的非成功） -> 仅当 rule 匹配才 retry
+        const body = extractBody(result);
+        if (body && config.ruleMatcher) {
+            const matchedRule = config.ruleMatcher.match(result.statusCode, body);
+            const effectiveMaxRetries = Math.min(matchedRule?.max_retries ?? 0, config.maxRetries);
+            if (matchedRule && state.attemptCount < effectiveMaxRetries) {
+                const strategy = createStrategy(matchedRule);
+                return { action: "retry", delayMs: strategy.getDelay(state.attemptCount) };
+            }
+        }
+        return { action: "done" };
+    }
+    async execute(targets, fn, config) {
+        const allAttempts = [];
+        const excludedTargets = [];
+        const perTargetCounts = new Map();
+        let globalAttemptIndex = 0;
+        let lastResult;
+        const targetKey = (t) => `${t.provider_id}:${t.backend_model}`;
+        const getTargetCount = (t) => perTargetCounts.get(targetKey(t)) ?? 0;
+        const incrementTarget = (t) => {
+            perTargetCounts.set(targetKey(t), (perTargetCounts.get(targetKey(t)) ?? 0) + 1);
+        };
+        while (true) {
+            const available = targets().filter(t => !excludedTargets.some(e => e.backend_model === t.backend_model && e.provider_id === t.provider_id));
+            if (available.length === 0) {
+                return {
+                    result: lastResult ?? { kind: "error", statusCode: 502, body: "All targets exhausted", headers: {}, sentHeaders: {}, sentBody: "" },
+                    attempts: allAttempts,
+                    excludedTargets,
+                };
+            }
+            const currentTarget = available[0];
+            incrementTarget(currentTarget);
+            const start = Date.now();
+            let transportResult;
+            try {
+                transportResult = await fn(currentTarget);
+            }
+            catch (err) {
+                const errMsg = err instanceof Error ? err.message : String(err);
+                transportResult = { kind: "throw", error: err instanceof Error ? err : new Error(errMsg) };
+            }
+            lastResult = transportResult;
+            if (transportResult.kind === "throw") {
+                const throwErr = transportResult.error;
+                allAttempts.push({
+                    target: currentTarget, attemptIndex: globalAttemptIndex,
+                    statusCode: null, error: throwErr instanceof Error ? throwErr.message : String(throwErr),
+                    latencyMs: Date.now() - start, responseBody: null,
+                });
+            }
+            else {
+                allAttempts.push({
+                    target: currentTarget, attemptIndex: globalAttemptIndex,
+                    statusCode: transportResult.statusCode, error: null,
+                    latencyMs: Date.now() - start, responseBody: extractBody(transportResult),
+                });
+            }
+            const state = {
+                attemptCount: getTargetCount(currentTarget) - 1,
+                currentTarget,
+                excludedTargets,
+            };
+            const decision = this.decide(transportResult, state, config);
+            switch (decision.action) {
+                case "done":
+                    return { result: transportResult, attempts: allAttempts, excludedTargets };
+                case "retry":
+                    globalAttemptIndex++;
+                    await sleep(decision.delayMs);
+                    continue;
+                case "failover":
+                    excludedTargets.push(decision.excludeTarget);
+                    globalAttemptIndex++;
+                    // 跨 provider failover 需要切换信号量，抛出异常让上层处理
+                    const nextAvail = targets().filter(t => !excludedTargets.some(e => e.backend_model === t.backend_model && e.provider_id === t.provider_id));
+                    if (nextAvail.length > 0 && nextAvail[0].provider_id !== currentTarget.provider_id) {
+                        throw new ProviderSwitchNeeded(nextAvail[0].provider_id);
+                    }
+                    continue;
+                case "abort":
+                    return { result: transportResult, attempts: allAttempts, excludedTargets };
+            }
+        }
+    }
+}

package/dist/proxy/scope.d.ts

@@ -0,0 +1,18 @@
+import type { ProviderSemaphoreManager } from "./semaphore.js";
+import type { RequestTracker } from "../monitor/request-tracker.js";
+import type { ActiveRequest } from "../monitor/types.js";
+export declare class SemaphoreScope {
+    private manager;
+    constructor(manager: ProviderSemaphoreManager);
+    withSlot<T>(providerId: string, signal: AbortSignal, onQueued: () => void, fn: () => Promise<T>): Promise<T>;
+}
+export declare class TrackerScope {
+    private tracker;
+    constructor(tracker: RequestTracker);
+    track<T>(req: ActiveRequest, fn: () => Promise<T>, extractStatus: (result: T) => {
+        status: "completed" | "failed";
+        statusCode?: number;
+    }): Promise<T>;
+    /** 通知 tracker 请求进入/离开信号量队列，触发前端即时广播 */
+    markQueued(id: string, queued: boolean): void;
+}

package/dist/proxy/scope.js

@@ -0,0 +1,37 @@
+export class SemaphoreScope {
+    manager;
+    constructor(manager) {
+        this.manager = manager;
+    }
+    async withSlot(providerId, signal, onQueued, fn) {
+        const token = await this.manager.acquire(providerId, signal, onQueued);
+        try {
+            return await fn();
+        }
+        finally {
+            this.manager.release(providerId, token);
+        }
+    }
+}
+export class TrackerScope {
+    tracker;
+    constructor(tracker) {
+        this.tracker = tracker;
+    }
+    async track(req, fn, extractStatus) {
+        this.tracker.start(req);
+        try {
+            const result = await fn();
+            this.tracker.complete(req.id, extractStatus(result));
+            return result;
+        }
+        catch (e) {
+            this.tracker.complete(req.id, { status: "failed" });
+            throw e;
+        }
+    }
+    /** 通知 tracker 请求进入/离开信号量队列，触发前端即时广播 */
+    markQueued(id, queued) {
+        this.tracker.update(id, { queued });
+    }
+}

package/dist/proxy/semaphore.d.ts

@@ -12,12 +12,19 @@ interface ConcurrencyConfig {
     queueTimeoutMs: number;
     maxQueueSize: number;
 }
+export interface SemaphoreLogger {
+    debug(obj: Record<string, unknown>, msg: string): void;
+    warn(obj: Record<string, unknown>, msg: string): void;
+}
+export interface AcquireToken {
+    readonly generation: number;
+}
 export declare class ProviderSemaphoreManager {
     private readonly entries;
     private getOrCreate;
     updateConfig(providerId: string, config: ConcurrencyConfig): void;
-    acquire(providerId: string, signal?: AbortSignal, onQueued?: () => void): Promise<void>;
-    release(providerId: string): void;
+    acquire(providerId: string, signal?: AbortSignal, onQueued?: () => void, logger?: SemaphoreLogger): Promise<AcquireToken>;
+    release(providerId: string, token: AcquireToken, logger?: SemaphoreLogger): void;
     getStatus(providerId: string): {
         active: number;
         queued: number;

package/dist/proxy/semaphore.js

@@ -25,6 +25,7 @@ export class ProviderSemaphoreManager {
                 config: { maxConcurrency: 0, queueTimeoutMs: 0, maxQueueSize: 0 },
                 current: 0,
                 queue: [],
+                generation: 0,
             };
             this.entries.set(providerId, entry);
         }
@@ -34,17 +35,19 @@ export class ProviderSemaphoreManager {
         const entry = this.getOrCreate(providerId);
         entry.config = config;
         if (config.maxConcurrency === 0) {
-            // Admin disabled throttling — drain queue without counting, reset current
-            // because no tracking is needed when maxConcurrency=0
             while (entry.queue.length > 0) {
                 const e = entry.queue.shift();
                 if (e.timer)
                     clearTimeout(e.timer);
                 e.resolve();
             }
+            // 递增 generation，使当前所有持有旧 token 的 release() 调用失效
+            entry.generation++;
             entry.current = 0;
             return;
         }
+        if (entry.current < 0)
+            entry.current = 0;
         while (entry.current < config.maxConcurrency &&
             entry.queue.length > 0) {
             entry.current++;
@@ -54,21 +57,35 @@ export class ProviderSemaphoreManager {
             e.resolve();
         }
     }
-    async acquire(providerId, signal, onQueued) {
+    async acquire(providerId, signal, onQueued, logger) {
         const entry = this.getOrCreate(providerId);
         const { maxConcurrency, queueTimeoutMs, maxQueueSize } = entry.config;
         if (maxConcurrency === 0)
-            return;
+            return { generation: entry.generation };
         if (entry.current < maxConcurrency) {
             entry.current++;
-            return;
+            logger?.debug({ providerId, current: entry.current, maxConcurrency, action: "acquire_direct" }, "Semaphore: acquired directly");
+            return { generation: entry.generation };
         }
         if (entry.queue.length >= maxQueueSize) {
+            logger?.debug({ providerId, queueLength: entry.queue.length, maxQueueSize, action: "acquire_rejected" }, "Semaphore: queue full, rejecting");
             throw new SemaphoreQueueFullError(providerId);
         }
+        logger?.debug({ providerId, current: entry.current, maxConcurrency, queueLength: entry.queue.length, action: "acquire_queued" }, "Semaphore: entering wait queue");
         onQueued?.();
         return new Promise((resolve, reject) => {
-            const qe = { resolve, reject, timer: null };
+            const token = { generation: entry.generation };
+            const qe = {
+                resolve: () => {
+                    logger?.debug({ providerId, current: entry.current, maxConcurrency, queueLength: entry.queue.length, action: "acquire_resolved" }, "Semaphore: left wait queue, acquired");
+                    resolve(token);
+                },
+                reject: (err) => {
+                    logger?.debug({ providerId, action: "acquire_rejected_internal", error: err.message }, "Semaphore: wait queue entry rejected");
+                    reject(err);
+                },
+                timer: null,
+            };
             if (queueTimeoutMs > 0) {
                 qe.timer = setTimeout(() => {
                     const idx = entry.queue.indexOf(qe);
@@ -91,18 +108,28 @@ export class ProviderSemaphoreManager {
             entry.queue.push(qe);
         });
     }
-    release(providerId) {
+    release(providerId, token, logger) {
         const entry = this.entries.get(providerId);
         if (!entry)
             return;
+        // maxConcurrency=0 时 acquire 不计数，release 也不应递减
+        if (entry.config.maxConcurrency === 0)
+            return;
+        // generation 不匹配说明此请求在 updateConfig 重置前 acquire，其槽位已被回收
+        if (token.generation !== entry.generation) {
+            logger?.debug({ providerId, tokenGen: token.generation, currentGen: entry.generation, action: "release_stale" }, "Semaphore: stale token, skipping release");
+            return;
+        }
         if (entry.queue.length > 0) {
             const e = entry.queue.shift();
+            logger?.debug({ providerId, current: entry.current, maxConcurrency: entry.config.maxConcurrency, queueRemaining: entry.queue.length, action: "release_dequeue" }, "Semaphore: released, dequeued next waiter");
             if (e.timer)
                 clearTimeout(e.timer);
             e.resolve();
         }
         else {
             entry.current--;
+            logger?.debug({ providerId, current: entry.current, maxConcurrency: entry.config.maxConcurrency, action: "release_decrement" }, "Semaphore: released slot");
         }
     }
     getStatus(providerId) {

package/dist/proxy/stream-proxy.d.ts

@@ -0,0 +1,7 @@
+import type { FastifyReply } from "fastify";
+import type { RawHeaders, TransportResult } from "./types.js";
+import type { SSEMetricsTransform } from "../metrics/sse-metrics-transform.js";
+import { type BuildHeadersFn } from "./transport.js";
+export declare function callStream(backend: {
+    base_url: string;
+}, apiKey: string, body: Record<string, unknown>, clientHeaders: RawHeaders, reply: FastifyReply, timeoutMs: number, upstreamPath: string, buildHeaders: BuildHeadersFn, metricsTransform?: SSEMetricsTransform, checkEarlyError?: (bufferedData: string) => boolean, compatResolve?: (result: TransportResult) => void): Promise<TransportResult>;