@nvae/llmswitch 0.6.0 → 0.8.0

package/dist/gateway/rate-limit.js

@@ -0,0 +1,285 @@
+/**
+ * Cross-process rate limiting.
+ *
+ * Counters live in a small JSON file guarded by an exclusive lock, so limits
+ * survive a daemon restart and hold across several gateway processes sharing one
+ * config directory (the common case being an accidental double start, or a
+ * foreground `serve` alongside a daemon).
+ *
+ * Two deliberate trade-offs:
+ *  - The critical section is a tiny read-modify-write. Upstream LLM calls take
+ *    seconds, so a sub-millisecond file operation per request is irrelevant.
+ *  - If the lock cannot be acquired quickly the request is allowed rather than
+ *    blocked: a slightly loose limit is preferable to stalling live traffic on
+ *    lock contention.
+ */
+import { existsSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { randomBytes } from "node:crypto";
+import { join } from "node:path";
+import { atomicWriteFile, ensureDir } from "../utils/fs.js";
+import { getGatewayDir } from "../utils/paths.js";
+const WINDOW_MS = 60_000;
+const DAY_MS = 86_400_000;
+const LOCK_TIMEOUT_MS = 500;
+const LOCK_STALE_MS = 5_000;
+const LOCK_SPIN_MS = 2;
+export function getRateLimitPath() {
+    return join(getGatewayDir(), "rate-limit.json");
+}
+export function getRateLimitLockPath() {
+    return join(getGatewayDir(), "rate-limit.lock");
+}
+const UNLIMITED = {
+    allowed: true,
+    limit: 0,
+    remaining: -1,
+    resetAt: 0,
+    retryAfterSeconds: 0,
+};
+function sleepSync(ms) {
+    const shared = new Int32Array(new SharedArrayBuffer(4));
+    Atomics.wait(shared, 0, 0, ms);
+}
+function pidAlive(pid) {
+    if (!Number.isInteger(pid) || pid < 1)
+        return false;
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (err) {
+        return err.code === "EPERM";
+    }
+}
+/**
+ * Acquire the counter lock, or return null when it stays busy past the timeout.
+ * A lock whose owner process is gone and which is older than the stale age is
+ * reclaimed.
+ */
+function acquireLock(now) {
+    const path = getRateLimitLockPath();
+    const id = randomBytes(8).toString("hex");
+    const payload = JSON.stringify({ id, pid: process.pid, at: now });
+    const deadline = now + LOCK_TIMEOUT_MS;
+    // The directory may not exist yet on a fresh install; without this the
+    // exclusive create below fails with ENOENT and persistence never engages.
+    try {
+        ensureDir(getGatewayDir());
+    }
+    catch {
+        return null;
+    }
+    for (;;) {
+        try {
+            writeFileSync(path, payload, { encoding: "utf8", flag: "wx", mode: 0o600 });
+            return makeLock(path, id);
+        }
+        catch (err) {
+            if (err.code !== "EEXIST")
+                return null;
+        }
+        if (reclaimable(path)) {
+            try {
+                const tmp = `${path}.${id}.tmp`;
+                writeFileSync(tmp, payload, { encoding: "utf8", mode: 0o600 });
+                renameSync(tmp, path);
+                if (readLockId(path) === id)
+                    return makeLock(path, id);
+            }
+            catch {
+                // Someone else won the race; fall through and retry.
+            }
+        }
+        if (Date.now() >= deadline)
+            return null;
+        sleepSync(LOCK_SPIN_MS);
+    }
+}
+function readLockId(path) {
+    try {
+        const raw = JSON.parse(readFileSync(path, "utf8"));
+        return typeof raw.id === "string" ? raw.id : null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * A lock is reclaimable when its owner is gone. A dead owner can never release
+ * the lock, so that case is reclaimed immediately; when ownership cannot be
+ * determined we fall back to an age check that trusts whichever of the file
+ * mtime or the recorded timestamp looks older.
+ */
+function reclaimable(path) {
+    let record = {};
+    let readable = false;
+    try {
+        record = JSON.parse(readFileSync(path, "utf8"));
+        readable = true;
+    }
+    catch {
+        // Unreadable lock file: fall back to the age check below.
+    }
+    if (readable && typeof record.pid === "number") {
+        return !pidAlive(record.pid);
+    }
+    let fileAge = Number.POSITIVE_INFINITY;
+    try {
+        fileAge = Date.now() - statSync(path).mtimeMs;
+    }
+    catch {
+        return true;
+    }
+    const recordedAge = typeof record.at === "number"
+        ? Date.now() - record.at
+        : Number.NEGATIVE_INFINITY;
+    return Math.max(fileAge, recordedAge) > LOCK_STALE_MS;
+}
+function makeLock(path, id) {
+    return {
+        release() {
+            if (readLockId(path) === id)
+                rmSync(path, { force: true });
+        },
+    };
+}
+function readWindows() {
+    const path = getRateLimitPath();
+    if (!existsSync(path))
+        return {};
+    try {
+        const raw = JSON.parse(readFileSync(path, "utf8"));
+        const windows = raw.windows;
+        if (!windows || typeof windows !== "object")
+            return {};
+        const out = {};
+        for (const [key, value] of Object.entries(windows)) {
+            if (!value || typeof value !== "object")
+                continue;
+            const row = value;
+            if (typeof row.windowStart !== "number" || typeof row.count !== "number") {
+                continue;
+            }
+            out[key] = { windowStart: row.windowStart, count: row.count };
+        }
+        return out;
+    }
+    catch {
+        return {};
+    }
+}
+function writeWindows(windows, now) {
+    // Drop windows that can no longer affect a decision.
+    const pruned = {};
+    for (const [key, record] of Object.entries(windows)) {
+        const span = key.endsWith("#day") ? DAY_MS * 2 : WINDOW_MS * 2;
+        if (now - record.windowStart < span)
+            pruned[key] = record;
+    }
+    ensureDir(getGatewayDir());
+    atomicWriteFile(getRateLimitPath(), JSON.stringify({ version: 1, windows: pruned }, null, 2) + "\n");
+}
+/** In-process fallback used when the lock cannot be taken. */
+const memoryWindows = new Map();
+function decide(record, limit, windowMs, now) {
+    const fresh = !record || now - record.windowStart >= windowMs;
+    const windowStart = fresh ? now : record.windowStart;
+    const used = fresh ? 0 : record.count;
+    const resetAt = Math.ceil((windowStart + windowMs) / 1000);
+    if (used >= limit) {
+        return {
+            next: { windowStart, count: used },
+            decision: {
+                allowed: false,
+                limit,
+                remaining: 0,
+                resetAt,
+                retryAfterSeconds: Math.max(1, Math.ceil((windowStart + windowMs - now) / 1000)),
+            },
+        };
+    }
+    const count = used + 1;
+    return {
+        next: { windowStart, count },
+        decision: {
+            allowed: true,
+            limit,
+            remaining: Math.max(0, limit - count),
+            resetAt,
+            retryAfterSeconds: 0,
+        },
+    };
+}
+/**
+ * Consume one request slot for `keyId`. A limit of 0 or less is unlimited and
+ * records nothing.
+ */
+export function checkRateLimit(keyId, limitPerMinute, now = Date.now()) {
+    return checkWindow(keyId, limitPerMinute, WINDOW_MS, now);
+}
+/** Clear all counters (test seam and `llms gateway ratelimit reset`). */
+export function resetRateLimits() {
+    memoryWindows.clear();
+    try {
+        rmSync(getRateLimitPath(), { force: true });
+        rmSync(getRateLimitLockPath(), { force: true });
+    }
+    catch {
+        // Nothing persisted yet.
+    }
+}
+/**
+ * Consume one request slot from the UTC-day window for `keyId`. A limit of 0
+ * or less means no daily quota and records nothing.
+ */
+export function checkDailyQuota(keyId, limitPerDay, now = Date.now()) {
+    return checkWindow(`${keyId}#day`, limitPerDay, DAY_MS, now);
+}
+/** Read the daily window without consuming a slot. */
+export function peekDailyQuota(keyId, limitPerDay, now = Date.now()) {
+    return peekWindow(`${keyId}#day`, limitPerDay, DAY_MS, now);
+}
+function checkWindow(bucket, limit, windowMs, now) {
+    if (!limit || limit <= 0)
+        return { ...UNLIMITED };
+    const lock = acquireLock(now);
+    if (!lock) {
+        const { next, decision } = decide(memoryWindows.get(bucket), limit, windowMs, now);
+        memoryWindows.set(bucket, next);
+        return decision;
+    }
+    try {
+        const windows = readWindows();
+        const { next, decision } = decide(windows[bucket], limit, windowMs, now);
+        windows[bucket] = next;
+        memoryWindows.set(bucket, next);
+        writeWindows(windows, now);
+        return decision;
+    }
+    catch {
+        // Never fail a request because bookkeeping failed.
+        return { ...UNLIMITED, limit };
+    }
+    finally {
+        lock.release();
+    }
+}
+/** Read the current window without consuming a slot. */
+export function peekRateLimit(keyId, limitPerMinute, now = Date.now()) {
+    return peekWindow(keyId, limitPerMinute, WINDOW_MS, now);
+}
+function peekWindow(bucket, limit, windowMs, now) {
+    if (!limit || limit <= 0)
+        return { ...UNLIMITED };
+    const record = readWindows()[bucket] ?? memoryWindows.get(bucket);
+    const fresh = !record || now - record.windowStart >= windowMs;
+    const windowStart = fresh ? now : record.windowStart;
+    const used = fresh ? 0 : record.count;
+    return {
+        allowed: used < limit,
+        limit,
+        remaining: Math.max(0, limit - used),
+        resetAt: Math.ceil((windowStart + windowMs) / 1000),
+        retryAfterSeconds: 0,
+    };
+}

package/dist/gateway/router.js

@@ -0,0 +1,163 @@
+/**
+ * Model routing and fallback ordering.
+ *
+ * Resolution order for the `model` field a third party sends:
+ *  1. an explicit route alias (`llms gateway route add`)
+ *  2. a qualified reference: `provider/model` or `provider:model`
+ *  3. a bare model id declared by one or more providers
+ *  4. providers that declare no model list (passthrough upstreams)
+ *  5. the configured default provider
+ *
+ * When several providers can serve the same model they are ordered by
+ * `priority` (ascending) and become each other's fallbacks, so ambiguity is
+ * deterministic rather than an error.
+ */
+import { listGatewayProviders, listGatewayRoutes, readGatewayConfig, } from "./store.js";
+export class ModelNotRoutableError extends Error {
+    requested;
+    availableModels;
+    constructor(requested, availableModels) {
+        const hint = availableModels.length
+            ? `可用模型：${availableModels.slice(0, 20).join(", ")}${availableModels.length > 20 ? " …" : ""}`
+            : "尚未配置任何 provider。请执行：llms gateway provider add";
+        super(`模型「${requested}」无法路由。${hint}`);
+        this.requested = requested;
+        this.availableModels = availableModels;
+        this.name = "ModelNotRoutableError";
+    }
+}
+function enabledProviders(providers) {
+    return providers
+        .filter((provider) => provider.enabled && provider.baseUrl)
+        .sort((a, b) => a.priority - b.priority || a.name.localeCompare(b.name));
+}
+function findProvider(providers, name) {
+    const wanted = name.trim().toLowerCase();
+    return providers.find((provider) => provider.name.toLowerCase() === wanted ||
+        provider.displayName.toLowerCase() === wanted);
+}
+function providerHasModel(provider, model) {
+    if (!provider.models.length)
+        return false;
+    const wanted = model.toLowerCase();
+    return provider.models.some((item) => item.toLowerCase() === wanted);
+}
+/** Split `provider/model` or `provider:model`; the model part may contain `/`. */
+function splitQualified(requested) {
+    for (const separator of ["/", ":"]) {
+        const index = requested.indexOf(separator);
+        if (index <= 0)
+            continue;
+        const provider = requested.slice(0, index).trim();
+        const model = requested.slice(index + 1).trim();
+        if (provider && model)
+            return { provider, model };
+    }
+    return null;
+}
+function pushCandidate(out, seen, provider, model, source) {
+    const dedupe = `${provider.name}::${model}`;
+    if (seen.has(dedupe))
+        return;
+    seen.add(dedupe);
+    out.push({ provider, model, source });
+}
+export function resolveModelRoute(requestedRaw, input = {}) {
+    const requested = (requestedRaw || "").trim();
+    const allProviders = input.providers ?? listGatewayProviders();
+    const providers = enabledProviders(allProviders);
+    const routes = input.routes ?? listGatewayRoutes();
+    const config = input.config ?? readGatewayConfig();
+    const candidates = [];
+    const seen = new Set();
+    if (!requested) {
+        throw new ModelNotRoutableError("(empty)", listRoutableModelIds({ providers: allProviders, routes }));
+    }
+    // 1. Explicit alias.
+    const route = routes.find((item) => item.alias.toLowerCase() === requested.toLowerCase());
+    if (route) {
+        const primary = findProvider(providers, route.provider);
+        if (primary) {
+            pushCandidate(candidates, seen, primary, route.model || requested, "route");
+        }
+        for (const fallback of route.fallbacks || []) {
+            const provider = findProvider(providers, fallback.provider);
+            if (provider) {
+                pushCandidate(candidates, seen, provider, fallback.model || route.model || requested, "route");
+            }
+        }
+    }
+    // 2. Qualified provider reference.
+    const qualified = splitQualified(requested);
+    if (qualified) {
+        const provider = findProvider(providers, qualified.provider);
+        if (provider) {
+            pushCandidate(candidates, seen, provider, qualified.model, "qualified");
+        }
+    }
+    // 3. Bare model id declared by providers, ordered by priority.
+    const bare = qualified ? qualified.model : requested;
+    for (const provider of providers) {
+        if (providerHasModel(provider, bare)) {
+            pushCandidate(candidates, seen, provider, bare, "model-list");
+        }
+    }
+    // 4. Passthrough providers (no declared model list).
+    for (const provider of providers) {
+        if (!provider.models.length) {
+            pushCandidate(candidates, seen, provider, bare, "passthrough");
+        }
+    }
+    // 5. Configured default provider.
+    if (config.defaultProvider) {
+        const provider = findProvider(providers, config.defaultProvider);
+        if (provider) {
+            pushCandidate(candidates, seen, provider, bare, "default");
+        }
+    }
+    if (!candidates.length) {
+        throw new ModelNotRoutableError(requested, listRoutableModelIds({ providers: allProviders, routes }));
+    }
+    const maxAttempts = config.fallback.enabled
+        ? Math.max(1, config.fallback.maxAttempts)
+        : 1;
+    return { requested, candidates: candidates.slice(0, maxAttempts) };
+}
+/** Client-visible model catalogue: aliases, bare ids and qualified ids. */
+export function listRoutableModels(input = {}) {
+    const providers = enabledProviders(input.providers ?? listGatewayProviders());
+    const routes = input.routes ?? listGatewayRoutes();
+    const out = [];
+    const seen = new Set();
+    const add = (id, provider, upstreamModel) => {
+        if (!id || seen.has(id))
+            return;
+        seen.add(id);
+        out.push({
+            id,
+            provider: provider.name,
+            upstreamModel,
+            format: provider.apiFormat,
+        });
+    };
+    for (const route of routes) {
+        const provider = providers.find((item) => item.name.toLowerCase() === route.provider.toLowerCase());
+        if (provider)
+            add(route.alias, provider, route.model || route.alias);
+    }
+    for (const provider of providers) {
+        for (const model of provider.models) {
+            add(model, provider, model);
+        }
+    }
+    // Qualified ids are always addressable, listed after the bare ids.
+    for (const provider of providers) {
+        for (const model of provider.models) {
+            add(`${provider.name}/${model}`, provider, model);
+        }
+    }
+    return out;
+}
+export function listRoutableModelIds(input = {}) {
+    return listRoutableModels(input).map((model) => model.id);
+}

package/dist/gateway/runtime.js

@@ -0,0 +1,45 @@
+/**
+ * Gateway listener resolution and exposure guards.
+ *
+ * The gateway is intended for third-party traffic, so binding it to a
+ * non-loopback address requires both an explicit opt-in and at least one active
+ * API key. Otherwise an unauthenticated model proxy would be reachable from the
+ * network.
+ */
+import { advertiseHostForBind, isLoopbackHost, normalizeHost, parseBridgePort, } from "../bridge/runtime.js";
+import { hasAnyActiveKey } from "./keys.js";
+export function parseGatewayPort(value) {
+    return parseBridgePort(value);
+}
+export class GatewayExposureError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "GatewayExposureError";
+    }
+}
+export function assertGatewayListenerAllowed(host, allowRemote, options = {}) {
+    if (isLoopbackHost(host))
+        return;
+    if (!allowRemote) {
+        throw new GatewayExposureError(`非回环监听地址 ${host || "(empty)"} 必须显式传入 --allow-remote`);
+    }
+    const requireKey = options.requireKey !== false;
+    if (requireKey && !hasAnyActiveKey()) {
+        throw new GatewayExposureError("对外暴露前必须至少存在一个有效 API Key。请先执行：llms gateway key create");
+    }
+}
+export function resolveGatewayListener(options) {
+    const bindHost = normalizeHost(options.host);
+    assertGatewayListenerAllowed(bindHost, options.allowRemote, {
+        requireKey: options.requireKey,
+    });
+    return {
+        bindHost,
+        advertiseHost: options.advertiseHost
+            ? normalizeHost(options.advertiseHost)
+            : advertiseHostForBind(bindHost),
+        port: parseGatewayPort(options.port),
+        allowRemote: options.allowRemote,
+    };
+}
+export { isLoopbackHost };