@tokenbuddy/tokenbuddy 1.0.36 → 1.0.38

package/src/prewarm-cache.ts

@@ -1,485 +0,0 @@
-import { createModuleLogger } from "@tokenbuddy/logging";
-
-const logger = createModuleLogger("tb-proxyd:prewarm-cache");
-
-/**
- * Default TTL for a successfully warmed entry. 10 minutes is the v1.2 starting
- * point; see buyer-driven-fallback-design.md §18.13 for the trade-off. The
- * cache constructor accepts an override so tests and the future PR-E config
- * loader can change this without re-architecting.
- */
-export const DEFAULT_PREWARM_TTL_MS = 10 * 60 * 1000;
-
-/**
- * 缓存条目的状态机。
- * - `warming`：调度中，候选尚未稳定
- * - `warm`：上次 commit 成功且 TTL 内
- * - `stale`：TTL 过期或连续 commit 失败
- * - `empty`：commit 返回 0 候选（该 (model, protocol, payment) 在当前 registry 下无 seller）
- */
-export type PrewarmState = "warming" | "warm" | "stale" | "empty";
-
-/**
- * 单个 seller 在某次 prewarm commit 后的健康画像。
- * 由 `PrewarmCache.commitWarm()` 归一化字段（score 0-100、latency 非负）后写入。
- */
-export interface PrewarmCandidate {
-  /** seller 全局 ID */
-  sellerId: string;
-  /** 去掉尾部斜杠后的 seller URL */
-  url: string;
-  /** 综合健康分，0-100，0 表示彻底坏 */
-  healthScore: number;          // 0-100
-  /** 上一次成功的 unix 毫秒时间戳；0 表示尚无成功 */
-  lastSuccessAt: number;
-  /** 上一次失败的 unix 毫秒时间戳；0 表示尚无失败 */
-  lastFailAt: number;
-  /** 平均延迟（毫秒），用于排序时的回退指标 */
-  avgLatencyMs: number;
-  /** health probe 的延迟（毫秒），可选 */
-  healthProbeLatencyMs?: number;
-  /** 首 token 延迟（毫秒），可选；speed 排序的优先指标 */
-  ttftMs?: number;
-  /** 平均推理延迟（毫秒），可选 */
-  avgInferenceMs?: number;
-  /** 最近 10 分钟窗口内的平均输出吞吐（tokens/s），可选 */
-  avgTokensPerSecond?: number;
-  /** 上游状态（与 seller 上报的语义对齐） */
-  upstreamStatus?: "healthy" | "degraded" | "unhealthy" | "unknown";
-  /** 上游错误类名（HTTP status / 错误码），仅在失败时存在 */
-  upstreamErrorClass?: string;
-  /** 临时容量避让截止时间；大于当前时间时不参与路由 */
-  capacityBlockedUntil?: number;
-}
-
-/**
- * 缓存条目：以 `(modelId, protocol, paymentMethod)` 为键，存储一组候选 seller 的健康画像。
- * `warmedAt` 是 TTL 的起点；`consecutiveWarmingFailures` 触发指数退避。
- */
-export interface PrewarmEntry {
-  /** 模型 ID（已归一化） */
-  modelId: string;
-  /** 协议名（已归一化） */
-  protocol: string;
-  /** 支付方式（已归一化） */
-  paymentMethod: string;
-  /** 当前条目状态 */
-  state: PrewarmState;
-  /** 该 (model, protocol, payment) 命中的候选 seller 列表 */
-  candidates: PrewarmCandidate[];
-  /** 本次成功的 commit 时间戳，TTL 起点 */
-  warmedAt: number;
-  /** 本条目的 TTL（毫秒），commit 时可被显式覆盖 */
-  ttlMs: number;
-  /** 连续 warming 失败次数；医生面板据此判断"长期坏" */
-  consecutiveWarmingFailures: number;
-  /** 最近一次进入 warming 的时间戳，调试用 */
-  lastInFlightAt?: number;
-}
-
-/**
- * 调度器在 commit 时传入的"原始"候选数据：所有字段都可空，
- * 由 `toCandidate()` 做归一化（score clamp、latency 截负等）。
- */
-export interface PrewarmCandidateInput {
-  /** seller ID */
-  sellerId: string;
-  /** seller URL */
-  url: string;
-  /** 健康分（可选；缺省在归一化时落到 50） */
-  healthScore?: number;
-  /** 上次成功时间戳（毫秒），可选 */
-  lastSuccessAt?: number;
-  /** 上次失败时间戳（毫秒），可选 */
-  lastFailAt?: number;
-  /** 平均延迟（毫秒），可选 */
-  avgLatencyMs?: number;
-  /** health probe 延迟（毫秒），可选 */
-  healthProbeLatencyMs?: number;
-  /** TTFT（毫秒），可选 */
-  ttftMs?: number;
-  /** 平均推理延迟（毫秒），可选 */
-  avgInferenceMs?: number;
-  /** 最近 10 分钟窗口内的平均输出吞吐（tokens/s），可选 */
-  avgTokensPerSecond?: number;
-  /** 上游状态，可选 */
-  upstreamStatus?: "healthy" | "degraded" | "unhealthy" | "unknown";
-  /** 上游错误类名，可选 */
-  upstreamErrorClass?: string;
-  /** 临时容量避让截止时间；大于当前时间时不参与路由 */
-  capacityBlockedUntil?: number;
-}
-
-/**
- * Build the cache key for a (model, protocol, payment) triple. The colon
- * separator is reserved at the model-id level because `RegistrySeller.models`
- * entries are trimmed but not colon-escaped. v1.2 forbids `:` inside model
- * ids so this format is collision-free.
- */
-export function prewarmKey(modelId: string, protocol: string, paymentMethod: string): string {
-  return `${modelId.trim().toLowerCase()}\u0001${protocol.trim().toLowerCase()}\u0001${paymentMethod.trim().toLowerCase()}`;
-}
-
-function parseKey(key: string): { modelId: string; protocol: string; paymentMethod: string } | undefined {
-  const parts = key.split("\u0001");
-  if (parts.length !== 3) {
-    return undefined;
-  }
-  const [modelId, protocol, paymentMethod] = parts;
-  if (!modelId || !protocol || !paymentMethod) {
-    return undefined;
-  }
-  return { modelId, protocol, paymentMethod };
-}
-
-interface PrewarmCacheOptions {
-  defaultTtlMs?: number;
-  now?: () => number;
-}
-
-/**
- * 进程内的 (model, protocol, payment) → 候选 seller 健康画像缓存。
- * 单线程访问（Node JS 主线程），无内部锁；`commitWarm` 是写路径，
- * `get/freshness` 是热路径读，TTL 由 `warmedAt + ttlMs` 决定。
- */
-export class PrewarmCache {
-  private readonly entries = new Map<string, PrewarmEntry>();
-  private readonly defaultTtlMs: number;
-  private readonly now: () => number;
-
-  constructor(options: PrewarmCacheOptions = {}) {
-    this.defaultTtlMs = options.defaultTtlMs ?? DEFAULT_PREWARM_TTL_MS;
-    this.now = options.now ?? Date.now;
-  }
-
-  /**
-   * Read an entry without mutating state. Returns `undefined` when the key is
-   * unknown; the caller decides whether "absent" should be treated as a miss
-   * (i.e. trigger a fresh prewarm) or as a known empty model.
-   */
-  get(modelId: string, protocol: string, paymentMethod: string): PrewarmEntry | undefined {
-    return this.entries.get(prewarmKey(modelId, protocol, paymentMethod));
-  }
-
-  /**
-   * Look up an entry and return a `Freshness` descriptor. This is the cheap
-   * path used on every inference request to decide whether a prewarm is
-   * still authoritative, expiring soon, or already stale.
-   */
-  freshness(modelId: string, protocol: string, paymentMethod: string): PrewarmFreshness {
-    const entry = this.get(modelId, protocol, paymentMethod);
-    if (!entry) {
-      return { present: false, expired: true, expiringSoon: true, state: "empty" };
-    }
-    const now = this.now();
-    const ageMs = now - entry.warmedAt;
-    const expired = ageMs >= entry.ttlMs;
-    const remainingMs = Math.max(0, entry.ttlMs - ageMs);
-    return {
-      present: true,
-      expired,
-      expiringSoon: !expired && remainingMs <= entry.ttlMs * 0.1,
-      remainingMs,
-      state: expired ? "stale" : entry.state,
-      entry
-    };
-  }
-
-  /**
-   * Mark a (model, protocol, payment) triple as currently being warmed. If an
-   * existing warm entry is present it is kept untouched (the new probe
-   * supersedes it on commit) and the previous state is reported to the
-   * caller via the returned descriptor.
-   */
-  beginWarming(modelId: string, protocol: string, paymentMethod: string, ttlMs?: number): PrewarmBeginResult {
-    const key = prewarmKey(modelId, protocol, paymentMethod);
-    const previous = this.entries.get(key);
-    const now = this.now();
-    const entry: PrewarmEntry = {
-      modelId,
-      protocol,
-      paymentMethod,
-      state: "warming",
-      candidates: previous?.candidates ?? [],
-      warmedAt: previous?.warmedAt ?? now,
-      ttlMs: ttlMs ?? previous?.ttlMs ?? this.defaultTtlMs,
-      consecutiveWarmingFailures: previous?.consecutiveWarmingFailures ?? 0,
-      lastInFlightAt: now
-    };
-    this.entries.set(key, entry);
-    logger.debug("prewarm.cache.warming_started", "prewarm probe in flight", {
-      modelId,
-      protocol,
-      paymentMethod,
-      ttlMs: entry.ttlMs,
-      previousState: previous?.state
-    });
-    return { key, entry, hadPrevious: Boolean(previous) };
-  }
-
-  /**
-   * Commit a successful warm. The entry's `warmedAt` is reset to the current
-   * time so the TTL window starts fresh, and any prior stale candidates are
-   * replaced with the new probe results. The previous candidate set is
-   * returned for caller-side telemetry (e.g. detecting churn).
-   */
-  commitWarm(input: {
-    modelId: string;
-    protocol: string;
-    paymentMethod: string;
-    candidates: PrewarmCandidateInput[];
-    ttlMs?: number;
-  }): PrewarmCommitResult {
-    const key = prewarmKey(input.modelId, input.protocol, input.paymentMethod);
-    const previous = this.entries.get(key);
-    const now = this.now();
-    const next: PrewarmEntry = {
-      modelId: input.modelId,
-      protocol: input.protocol,
-      paymentMethod: input.paymentMethod,
-      state: input.candidates.length > 0 ? "warm" : "empty",
-      candidates: input.candidates.map(toCandidate),
-      warmedAt: now,
-      ttlMs: input.ttlMs ?? previous?.ttlMs ?? this.defaultTtlMs,
-      consecutiveWarmingFailures: 0,
-      lastInFlightAt: now
-    };
-    this.entries.set(key, next);
-
-    if (input.candidates.length === 0) {
-      logger.warn("prewarm.cache.commit_empty", "prewarm commit returned no candidates", {
-        modelId: input.modelId,
-        protocol: input.protocol,
-        paymentMethod: input.paymentMethod
-      });
-    } else {
-      logger.info("prewarm.cache.committed", "prewarm commit updated candidates", {
-        modelId: input.modelId,
-        protocol: input.protocol,
-        paymentMethod: input.paymentMethod,
-        candidateCount: next.candidates.length,
-        ttlMs: next.ttlMs
-      });
-    }
-
-    return {
-      key,
-      entry: next,
-      replacedSellers: previous?.candidates.map((c) => c.sellerId) ?? []
-    };
-  }
-
-  /**
-   * Mark a warm as failed. Consecutive failures are tracked so the scheduler
-   * can apply exponential backoff and so `tb doctor` can surface persistently
-   * broken models.
-   */
-  recordFailure(modelId: string, protocol: string, paymentMethod: string, errorMessage?: string): PrewarmEntry | undefined {
-    const key = prewarmKey(modelId, protocol, paymentMethod);
-    const previous = this.entries.get(key);
-    if (!previous) {
-      return undefined;
-    }
-    const next: PrewarmEntry = {
-      ...previous,
-      state: "stale",
-      consecutiveWarmingFailures: previous.consecutiveWarmingFailures + 1,
-      lastInFlightAt: this.now()
-    };
-    this.entries.set(key, next);
-    logger.warn("prewarm.cache.failure_recorded", "prewarm commit failed; entry marked stale", {
-      modelId,
-      protocol,
-      paymentMethod,
-      consecutiveFailures: next.consecutiveWarmingFailures,
-      errorMessage
-    });
-    return next;
-  }
-
-  /**
-   * Invalidate every entry that references the given seller. Used when the
-   * registry signals a seller is gone (grace period expires) or when a hard
-   * failure (e.g. 5xx storm) should drop the seller from the cache
-   * immediately.
-   */
-  invalidateSeller(sellerId: string): number {
-    let removed = 0;
-    for (const [key, entry] of this.entries.entries()) {
-      const filtered = entry.candidates.filter((candidate) => candidate.sellerId !== sellerId);
-      if (filtered.length !== entry.candidates.length) {
-        removed += 1;
-        this.entries.set(key, {
-          ...entry,
-          candidates: filtered,
-          state: filtered.length > 0 ? entry.state : "empty"
-        });
-      }
-    }
-    if (removed > 0) {
-      logger.info("prewarm.cache.seller_invalidated", "seller dropped from all prewarm entries", {
-        sellerId,
-        entriesAffected: removed
-      });
-    }
-    return removed;
-  }
-
-  /**
-   * Invalidate a specific cache key. Used by `tb doctor --refresh <model>`
-   * and by the registry loop when a model is removed from the focus set.
-   */
-  invalidateKey(modelId: string, protocol: string, paymentMethod: string): boolean {
-    return this.entries.delete(prewarmKey(modelId, protocol, paymentMethod));
-  }
-
-  /**
-   * Drop every entry whose TTL has expired. Returns the number of removed
-   * entries so the caller can log it.
-   */
-  evictExpired(now: number = this.now()): number {
-    let removed = 0;
-    for (const [key, entry] of this.entries.entries()) {
-      if (now - entry.warmedAt >= entry.ttlMs) {
-        this.entries.delete(key);
-        removed += 1;
-      }
-    }
-    if (removed > 0) {
-      logger.info("prewarm.cache.evicted", "expired prewarm entries evicted", { removed });
-    }
-    return removed;
-  }
-
-  /**
-   * Returns `true` when the entry's TTL is within `withinMs` of expiring. The
-   * scheduler uses this to schedule idle-cycle prewarms just-in-time rather
-   * than at fixed wall-clock intervals.
-   */
-  isExpiringSoon(modelId: string, protocol: string, paymentMethod: string, withinMs: number, now: number = this.now()): boolean {
-    const entry = this.get(modelId, protocol, paymentMethod);
-    if (!entry) {
-      return false;
-    }
-    const age = now - entry.warmedAt;
-    return age >= entry.ttlMs - withinMs && age < entry.ttlMs;
-  }
-
-  /**
-   * Snapshot all entries for diagnostics. Returns a deep-copy of the values
-   * so callers can serialize without risking mutation of cache state.
-   */
-  snapshot(): PrewarmEntry[] {
-    return Array.from(this.entries.values()).map((entry) => ({
-      ...entry,
-      candidates: entry.candidates.map((candidate) => ({ ...candidate }))
-    }));
-  }
-
-  /**
-   * List every cached key, decoded back into its (model, protocol, payment)
-   * triple. Used by `tb doctor` to render the prewarm table.
-   */
-  keys(): Array<{ modelId: string; protocol: string; paymentMethod: string }> {
-    const out: Array<{ modelId: string; protocol: string; paymentMethod: string }> = [];
-    for (const key of this.entries.keys()) {
-      const parsed = parseKey(key);
-      if (parsed) {
-        out.push(parsed);
-      }
-    }
-    return out;
-  }
-
-  size(): number {
-    return this.entries.size;
-  }
-
-  clear(): void {
-    this.entries.clear();
-  }
-}
-
-/**
- * `PrewarmCache.freshness()` 的返回：一次"是否还能信任现有 prewarm"的快速判断。
- * 在每条 inference 请求的热路径上使用，避免直接遍历缓存。
- */
-export interface PrewarmFreshness {
-  /** 是否存在对应条目（false 等价于 cold path） */
-  present: boolean;
-  /** 当前是否已超过 TTL */
-  expired: boolean;
-  /** TTL 剩余 ≤ 10% 时视为"即将过期"，调度器用此触发 idle 预热 */
-  expiringSoon: boolean;
-  /** 剩余 TTL（毫秒），过期时省略 */
-  remainingMs?: number;
-  /** 条目当前状态（过期时强制为 `stale`） */
-  state: PrewarmState;
-  /** 关联的缓存条目（如果有） */
-  entry?: PrewarmEntry;
-}
-
-/**
- * `PrewarmCache.beginWarming()` 的返回值：标记一次预热进入 in-flight。
- * `hadPrevious` 用于上层决定是否在 `warming` 期间对外暴露旧候选。
- */
-export interface PrewarmBeginResult {
-  /** 缓存键（与 `prewarmKey()` 的输出等价） */
-  key: string;
-  /** 当前写回缓存的 entry（state=`warming`） */
-  entry: PrewarmEntry;
-  /** 调用前缓存里是否已有 entry，用于上层做"保留旧候选"判断 */
-  hadPrevious: boolean;
-}
-
-/**
- * `PrewarmCache.commitWarm()` 的返回值：成功提交一次候选集合。
- * `replacedSellers` 用于上层做候选 churn 检测。
- */
-export interface PrewarmCommitResult {
-  /** 缓存键 */
-  key: string;
-  /** 提交后的新 entry（state=`warm` 或 `empty`） */
-  entry: PrewarmEntry;
-  /** 上一版 entry 的候选 seller 列表（已不再缓存的 seller ID） */
-  replacedSellers: string[];
-}
-
-function toCandidate(input: PrewarmCandidateInput): PrewarmCandidate {
-  return {
-    sellerId: input.sellerId,
-    url: input.url,
-    healthScore: clampScore(input.healthScore ?? 50),
-    lastSuccessAt: input.lastSuccessAt ?? 0,
-    lastFailAt: input.lastFailAt ?? 0,
-    avgLatencyMs: Math.max(0, input.avgLatencyMs ?? 0),
-    healthProbeLatencyMs: finiteNonNegative(input.healthProbeLatencyMs),
-    ttftMs: finiteNonNegative(input.ttftMs),
-    avgInferenceMs: finiteNonNegative(input.avgInferenceMs),
-    avgTokensPerSecond: finitePositive(input.avgTokensPerSecond),
-    upstreamStatus: input.upstreamStatus,
-    upstreamErrorClass: input.upstreamErrorClass,
-    capacityBlockedUntil: finiteNonNegative(input.capacityBlockedUntil)
-  };
-}
-
-function finiteNonNegative(value: number | undefined): number | undefined {
-  return Number.isFinite(value) ? Math.max(0, value as number) : undefined;
-}
-
-function finitePositive(value: number | undefined): number | undefined {
-  return Number.isFinite(value) && (value as number) > 0 ? value : undefined;
-}
-
-function clampScore(score: number): number {
-  if (!Number.isFinite(score)) {
-    return 50;
-  }
-  if (score < 0) {
-    return 0;
-  }
-  if (score > 100) {
-    return 100;
-  }
-  return score;
-}