@wengine-ai/llms 2.3.2408 → 2.3.2409

package/dist/server.d.ts

@@ -1,4 +1,4 @@
-import { FastifyInstance, FastifyPluginAsync, FastifyPluginCallback, FastifyPluginOptions, FastifyRegisterOptions, preHandlerHookHandler, onRequestHookHandler, preParsingHookHandler, preValidationHookHandler, preSerializationHookHandler, onSendHookHandler, onResponseHookHandler, onTimeoutHookHandler, onErrorHookHandler, onRouteHookHandler, onRegisterHookHandler, onReadyHookHandler, onListenHookHandler, onCloseHookHandler, FastifyServerOptions } from "fastify";
+import { FastifyInstance, FastifyPluginAsync, FastifyPluginCallback, FastifyPluginOptions, FastifyRegisterOptions, preHandlerHookHandler, onRequestHookHandler, preParsingHookHandler, preValidationHookHandler, preSerializationHookHandler, onSendHookHandler, onResponseHookHandler, onRequestAbortHookHandler, onTimeoutHookHandler, onErrorHookHandler, onRouteHookHandler, onRegisterHookHandler, onReadyHookHandler, onListenHookHandler, onCloseHookHandler, FastifyServerOptions } from "fastify";
 import { ConfigService, AppConfig } from "./services/config";
 import { ProviderService } from "./services/provider";
 import { TransformerService } from "./services/transformer";
@@ -47,6 +47,7 @@ declare class Server {
     addHook(hookName: "preSerialization", hookFunction: preSerializationHookHandler): void;
     addHook(hookName: "onSend", hookFunction: onSendHookHandler): void;
     addHook(hookName: "onResponse", hookFunction: onResponseHookHandler): void;
+    addHook(hookName: "onRequestAbort", hookFunction: onRequestAbortHookHandler): void;
     addHook(hookName: "onTimeout", hookFunction: onTimeoutHookHandler): void;
     addHook(hookName: "onError", hookFunction: onErrorHookHandler): void;
     addHook(hookName: "onRoute", hookFunction: onRouteHookHandler): void;

package/dist/utils/concurrency-router.d.ts

@@ -0,0 +1,91 @@
+/**
+ * Concurrency-Priority Routing
+ *
+ * When enabled (`Router.enableConcurrencyPriority`), this layers on top of the
+ * normal routing decision: whatever model the existing router selects (default,
+ * think, longContext, family default, ...) is checked against live in-flight
+ * concurrency:
+ *
+ * - A new (not yet pinned) session overflows to the least-loaded healthy model
+ *   in the scenario-matching fallback list once the selected model's in-flight
+ *   concurrency reaches `Router.concurrencyThreshold` (default 3).
+ * - Each session sticks to the first model it was assigned per routing slot
+ *   (session + model family + scenario), which keeps provider-side prompt
+ *   caches warm and avoids mid-session provider switches.
+ * - A session only changes models when its current model errors and the
+ *   error-fallback path succeeds (routes.ts calls rebindSessionModel).
+ *
+ * All state is in-memory: bindings and in-flight leases are reset on restart,
+ * which is safe because a restart also drops all in-flight requests.
+ */
+export declare const DEFAULT_CONCURRENCY_THRESHOLD = 3;
+interface Lease {
+    id: number;
+    key: string;
+    createdAt: number;
+    released: boolean;
+}
+export interface ConcurrencyPriorityDeps {
+    /**
+     * Resolve a "provider,model" route to its canonical form, or null when the
+     * target is unusable (missing, disabled, unhealthy, quota-exhausted). The
+     * caller injects router.ts's resolveConfiguredModel to avoid a module cycle.
+     */
+    resolve: (modelKey: string) => string | null;
+    /** Ordered overflow candidates (the scenario-matching fallback lists). */
+    fallbackModels: string[];
+    /** Primary-model in-flight concurrency at which new sessions overflow. */
+    threshold: number;
+    log?: (msg: string) => void;
+}
+declare class ConcurrencyTracker {
+    private leases;
+    private nextId;
+    acquire(key: string): Lease;
+    release(lease: Lease | undefined | null): void;
+    private prune;
+    count(key: string): number;
+    clear(): void;
+}
+declare class SessionBindingStore {
+    private bindings;
+    get(sessionId: string): string | undefined;
+    bind(sessionId: string, modelKey: string): void;
+    unbind(sessionId: string): void;
+    clear(): void;
+}
+export declare function getConcurrencyTracker(): ConcurrencyTracker;
+export declare function getSessionBindingStore(): SessionBindingStore;
+/** Test helper: reset all concurrency-priority state. */
+export declare function resetConcurrencyPriorityState(): void;
+/**
+ * Stickiness slot key: bindings are tracked per session AND routing slot
+ * (model family + scenario), so a session can pin one model for its default
+ * traffic and another for think/longContext traffic without interfering.
+ */
+export declare function getRequestSlotKey(req: any): string;
+/**
+ * Attach an in-flight lease for the given model key to the request. If the
+ * request already holds a lease for a different model (e.g. after a fallback
+ * switch), the old lease is released first so counts stay accurate.
+ */
+export declare function acquireRequestLease(req: any, key: string): void;
+/** Release the request's in-flight lease (idempotent). */
+export declare function releaseRequestLease(req: any): void;
+/**
+ * Rebind the request's session to a new model after the error-fallback path
+ * succeeded with that model. No-op for requests that do not participate in
+ * concurrency-priority routing (no lease and no existing binding).
+ */
+export declare function rebindSessionModel(req: any, modelKey: string): void;
+/**
+ * Apply concurrency-priority routing to a resolved default-scenario model.
+ *
+ * `currentModel` is the model the normal routing pipeline resolved for this
+ * request. The returned model is either the sticky bound model, an overflow
+ * fallback model, or `currentModel` unchanged. An in-flight lease for the
+ * returned model is attached to the request and must be released when the
+ * response completes (see request-pipeline.ts onResponse hook).
+ */
+export declare function routeWithConcurrencyPriority(req: any, currentModel: string, deps: ConcurrencyPriorityDeps): string;
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wengine-ai/llms",
-  "version": "2.3.2408",
+  "version": "2.3.2409",
   "description": "A universal LLM API transformation server",
   "main": "dist/cjs/server.cjs",
   "module": "dist/esm/server.mjs",
@@ -64,7 +64,7 @@
     "uuid": "^11.1.0",
     "@fastify/multipart": "^9.0.0",
     "@fastify/static": "^8.2.0",
-    "@wengine-ai/claude-code-router-shared": "^2.3.2408",
+    "@wengine-ai/claude-code-router-shared": "^2.3.2409",
     "adm-zip": "^0.5.16",
     "better-sqlite3": "^12.10.0",
     "rotating-file-stream": "^3.2.7"