@tokenbuddy/tokenbuddy 1.0.36 → 1.0.37

package/src/route-failover.ts

@@ -1,304 +0,0 @@
-import { createModuleLogger } from "@tokenbuddy/logging";
-import type { RegistrySeller } from "./seller-catalog.js";
-import type { SellerPool, FailureKind, PoolEntry } from "./seller-pool.js";
-import type { CreditTracker } from "./credit-tracker.js";
-
-const logger = createModuleLogger("tb-proxyd:route-failover");
-
-/**
- * 路由失败后的下一步动作（控制器收到 `FailoverDecision` 后据此执行）。
- * - `retry_same_seller`：在同 seller 上做 jittered backoff 重试
- * - `failover_next`：从池里取出下一个候选 seller
- * - `fail_fast`：立刻向上抛错，不再尝试其它候选
- * - `abort`：候选已耗尽，放弃本轮
- */
-export type RouteAction = "retry_same_seller" | "failover_next" | "fail_fast" | "abort";
-
-/**
- * 一次失败后 `RouteFailover.decide()` 返回的结构化决策。
- * 控制器根据 `action` 决定是否重试/切换/终止，并把 `wastedCreditMicros`
- * 上报到 doctor 面板。
- */
-export interface FailoverDecision {
-  /** 控制器要执行的下一步动作 */
-  action: RouteAction;
-  /** 仅当 `action === "retry_same_seller"` 时使用，表示 backoff 后多久重试（毫秒） */
-  retryDelayMs?: number;
-  /** 决策理由，结构化但可读，供日志 / doctor 渲染 */
-  reason: string;
-  /** 本次失败后被废弃的 credit（USD micros），供 doctor 统计 */
-  wastedCreditMicros?: number;
-  /** 失败发生在 fresh-purchase 窗口内（刚买不到 N 秒），用于触发软重试保护 */
-  freshPurchase: boolean;
-  /** 在切走这个 seller 之前的最后一次尝试索引（0-based，含本次失败） */
-  retryAttemptsBeforeFailover: number;
-  /** 当次会话是否已超出 auto-purchase 预算；true 时不再触发新一轮 buy */
-  budgetExceeded: boolean;
-}
-
-/**
- * 一次路由选择的最小可执行单元：池里的某个 entry + 关联的 registry 描述。
- * `routeIndex` 仅用于日志排版，控制器自己维护尝试序号。
- */
-export interface RouteCandidate {
-  /** 在本批候选中的序号（0-based），仅作日志可读性使用 */
-  routeIndex: number;
-  /** 池里的 entry（包含 circuit / healthScore 等运行时状态） */
-  entry: PoolEntry;
-  /** registry 里的原始 seller 描述 */
-  registrySeller: RegistrySeller;
-  /** seller 全局 ID（同时也是 token class） */
-  sellerId: string;
-  /** 去掉尾部斜杠后的 seller URL */
-  url: string;
-}
-
-/**
- * 控制器调用 `decide()` 时传入的失败上下文。
- * 描述"这次失败发生在谁身上、是什么错误、第几次尝试"。
- */
-export interface DecideContext {
-  /** 失败时正在调用的 seller ID */
-  sellerId: string;
-  /** HTTP status（如有），用于区分 4xx/5xx */
-  status?: number;
-  /** 错误分类，由调用方在边界层归一化为 `FailureKind` */
-  errorKind: FailureKind;
-  /** 人类可读错误描述（不携带敏感字段），用于日志/doctor */
-  errorMessage?: string;
-  /** 当前是该 seller 的第几次尝试（0-based；日志里同名字段保持该语义） */
-  attempt: number;
-}
-
-/**
- * 构造 `RouteFailover` 所需的依赖与可调参数。
- * 默认值见 `DEFAULTS`：软重试 2 次，jitter 100-400ms。
- */
-export interface RouteFailoverOptions {
-  /** 共享的 seller pool，控制器只读 + 委托 recordSuccess/recordFailure */
-  pool: SellerPool;
-  /** 共享的 credit tracker，用于 wasted 计算与 auto-purchase 预算 */
-  creditTracker: CreditTracker;
-  // v1.1 / v1.2 design defaults; exposed for tests.
-  /** 软重试次数上限，默认 2 */
-  softRetryAttempts?: number;        // default 2
-  /** 软重试 backoff jitter 下界（毫秒），默认 100 */
-  softRetryJitterMinMs?: number;     // default 100
-  /** 软重试 backoff jitter 上界（毫秒），默认 400 */
-  softRetryJitterMaxMs?: number;     // default 400
-  /** 注入的随机源（测试用），默认 `Math.random` */
-  random?: () => number;
-  /** 注入的时钟（测试用），默认 `Date.now` */
-  now?: () => number;
-}
-
-const DEFAULTS = {
-  softRetryAttempts: 2,
-  softRetryJitterMinMs: 100,
-  softRetryJitterMaxMs: 400
-};
-
-/**
- * Decision engine that wraps the v2 `SellerPool` and the
- * `CreditTracker`. v1.2 §17.3 - §17.8 defines the rules: the controller
- * calls `decide()` after a failure to learn whether to retry the same
- * seller, fail over to the next candidate, fail fast, or give up.
- *
- * The controller is intentionally side-effect-light: it delegates
- * bookkeeping (circuit transitions, wasted-credit transfer) to the pool
- * and tracker via `recordFailure`, and only returns a structured
- * decision. This makes the rules unit-testable without a live HTTP server.
- */
-export class RouteFailover {
-  private readonly pool: SellerPool;
-  private readonly creditTracker: CreditTracker;
-  private readonly softRetryAttempts: number;
-  private readonly softRetryJitterMinMs: number;
-  private readonly softRetryJitterMaxMs: number;
-  private readonly random: () => number;
-  private readonly now: () => number;
-
-  constructor(options: RouteFailoverOptions) {
-    this.pool = options.pool;
-    this.creditTracker = options.creditTracker;
-    this.softRetryAttempts = options.softRetryAttempts ?? DEFAULTS.softRetryAttempts;
-    this.softRetryJitterMinMs = options.softRetryJitterMinMs ?? DEFAULTS.softRetryJitterMinMs;
-    this.softRetryJitterMaxMs = options.softRetryJitterMaxMs ?? DEFAULTS.softRetryJitterMaxMs;
-    this.random = options.random ?? Math.random;
-    this.now = options.now ?? Date.now;
-  }
-
-  /**
-   * Pick the next candidate from the pool. Returns `undefined` when the
-   * pool is exhausted, which the caller treats as `abort` (no more routes
-   * to try). The returned `routeIndex` is informational (used for log
-   * lines) and tracks the absolute ordering across attempts; the controller
-   * maintains its own counter.
-   */
-  pickNext(modelId: string, protocol: string, paymentMethod: string, limit: number = 4): RouteCandidate | undefined {
-    const result = this.pool.pick({ modelId, protocol, paymentMethod, limit });
-    if (result.candidates.length === 0) {
-      return undefined;
-    }
-    const first = result.candidates[0];
-    return {
-      routeIndex: 0,
-      entry: first.entry,
-      registrySeller: first.registrySeller,
-      sellerId: first.entry.sellerId,
-      url: first.entry.url
-    };
-  }
-
-  /**
-   * Mark a successful inference against `sellerId` with the latest known
-   * balance. Mirrors `SellerPool.recordSuccess` semantics: resets the
-   * failure counter, closes the circuit, and reports the spend to the
-   * credit tracker.
-   */
-  recordSuccess(sellerId: string, balanceMicros: number): void {
-    this.pool.recordSuccess(sellerId, balanceMicros, this.now());
-  }
-
-  /**
-   * Decide what to do after a failure. Returns a structured
-   * `FailoverDecision`; the controller (daemon.ts) executes the action.
-   * The decision also drives accounting: on a failover, the wasted
-   * balance is reported in `wastedCreditMicros` for the doctor's summary.
-   */
-  decide(context: DecideContext, totalCandidates: number): FailoverDecision {
-    const isHard = context.errorKind === "hard_4xx" || context.errorKind === "auth_invalid" || context.errorKind === "no_compatible";
-    const isSoft = context.errorKind === "soft_5xx" || context.errorKind === "deadline";
-    const isBusyCapacity = context.errorKind === "busy_capacity";
-    const isPurchaseFailure = context.errorKind === "purchase_failed";
-    const info = this.pool.inspect(context.sellerId);
-    const freshPurchase = info.freshPurchase;
-    const budgetExceeded = !this.creditTracker.canAutoPurchase(this.now());
-
-    // Always mark the failure on the pool so circuit thresholds advance.
-    const updated = this.pool.recordFailure(context.sellerId, context.errorKind, {
-      reason: context.errorMessage,
-      now: this.now()
-    });
-
-    if (isBusyCapacity) {
-      return {
-        action: "failover_next",
-        reason: "busy_capacity",
-        freshPurchase,
-        retryAttemptsBeforeFailover: context.attempt,
-        budgetExceeded
-      };
-    }
-
-    if (isPurchaseFailure) {
-      return {
-        action: "failover_next",
-        reason: "purchase_failed",
-        wastedCreditMicros: this.creditTracker.getEntry(context.sellerId)?.currentBalanceMicros,
-        freshPurchase,
-        retryAttemptsBeforeFailover: context.attempt,
-        budgetExceeded
-      };
-    }
-
-    if (isHard) {
-      // Hard failures are not eligible for retry; the seller is wrong
-      // for this request. The pool has already transferred leftover
-      // credit to the wasted bucket.
-      return {
-        action: "failover_next",
-        reason: `hard_failure:${context.errorKind}`,
-        wastedCreditMicros: this.creditTracker.getEntry(context.sellerId)?.leftoverCreditMicros,
-        freshPurchase,
-        retryAttemptsBeforeFailover: context.attempt,
-        budgetExceeded
-      };
-    }
-
-    if (isSoft && freshPurchase && context.attempt < this.softRetryAttempts) {
-      // Soft failure inside the fresh-purchase window: retry the same
-      // seller with jittered backoff. This is the v1.1 protection against
-      // throwing freshly bought credit onto a flaky upstream.
-      const span = Math.max(0, this.softRetryJitterMaxMs - this.softRetryJitterMinMs);
-      const delay = this.softRetryJitterMinMs + Math.floor(this.random() * span);
-      logger.info("route.retry_same_seller.soft", "soft failure in fresh-purchase window; retrying same seller", {
-        sellerId: context.sellerId,
-        attempt: context.attempt,
-        delayMs: delay
-      });
-      return {
-        action: "retry_same_seller",
-        retryDelayMs: delay,
-        reason: "soft_failure_fresh_purchase_window",
-        freshPurchase: true,
-        retryAttemptsBeforeFailover: context.attempt,
-        budgetExceeded
-      };
-    }
-
-    if (isSoft && context.attempt < this.softRetryAttempts) {
-      // Soft failure but not in the fresh-purchase window: still retry
-      // once (caller asked for `attempt < softRetryAttempts`) but log the
-      // reason. This keeps the "give the seller a second chance" window
-      // available for cold sellers without burning any credit.
-      const span = Math.max(0, this.softRetryJitterMaxMs - this.softRetryJitterMinMs);
-      const delay = this.softRetryJitterMinMs + Math.floor(this.random() * span);
-      logger.info("route.retry_same_seller.soft", "soft failure retry outside fresh-purchase window", {
-        sellerId: context.sellerId,
-        attempt: context.attempt,
-        delayMs: delay
-      });
-      return {
-        action: "retry_same_seller",
-        retryDelayMs: delay,
-        reason: "soft_failure_retry",
-        freshPurchase: false,
-        retryAttemptsBeforeFailover: context.attempt,
-        budgetExceeded
-      };
-    }
-
-    // Default: fail over. The wasted credit is whatever current balance
-    // remained on the seller after the failure was recorded (the pool
-    // transfers it to the leftover bucket for hard failures; for soft
-    // failures we surface the *pre-failure* balance so the user sees the
-    // impact of the cut-over).
-    const wasted = updated?.lastFailAt
-      ? this.creditTracker.getEntry(context.sellerId)?.currentBalanceMicros
-      : undefined;
-    return {
-      action: "failover_next",
-      reason: isSoft ? "soft_failure_exhausted_retries" : `${context.errorKind}`,
-      wastedCreditMicros: wasted,
-      freshPurchase,
-      retryAttemptsBeforeFailover: context.attempt,
-      budgetExceeded
-    };
-  }
-
-  /**
-   * Convenience: returns true when the next request should be aborted
-   * outright (no more candidates).
-   */
-  shouldAbort(totalCandidates: number, hasMoreCandidates: boolean): boolean {
-    return !hasMoreCandidates;
-  }
-
-  /**
-   * Inspect the auto-purchase budget. The controller calls this before
-   * triggering a fresh `purchase/create` round-trip so it can refuse to
-   * auto-purchase once the session is at risk.
-   */
-  canAutoPurchase(): boolean {
-    return this.creditTracker.canAutoPurchase(this.now());
-  }
-
-  /**
-   * Surface the wasted credit so far for the route currently failing over.
-   * Returned in micros.
-   */
-  wastedMicrosFor(sellerId: string): number {
-    return this.creditTracker.getEntry(sellerId)?.currentBalanceMicros ?? 0;
-  }
-}