ai-resilient 0.1.0

package/dist/index.cjs

@@ -0,0 +1,645 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  AllModelsExhaustedError: () => AllModelsExhaustedError,
+  LimitTracker: () => LimitTracker,
+  classifyError: () => classifyError,
+  createResilient: () => createResilient,
+  getRetryAfterMs: () => getRetryAfterMs,
+  memoryStore: () => memoryStore,
+  parseRateLimitHeaders: () => parseRateLimitHeaders
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/header-parsers.ts
+function lower(headers) {
+  const out = {};
+  for (const [k, v] of Object.entries(headers)) out[k.toLowerCase()] = v;
+  return out;
+}
+function num(value) {
+  if (value === void 0) return void 0;
+  const n = Number(value);
+  return Number.isFinite(n) ? n : void 0;
+}
+function parseDuration(value) {
+  if (value === void 0) return void 0;
+  const plain = Number(value);
+  if (Number.isFinite(plain)) return plain * 1e3;
+  const pattern = /(\d+(?:\.\d+)?)(ms|h|m|s)/g;
+  let ms = 0;
+  let matched = false;
+  for (const [, amount, unit] of value.matchAll(pattern)) {
+    matched = true;
+    const n = Number(amount);
+    if (unit === "h") ms += n * 36e5;
+    else if (unit === "m") ms += n * 6e4;
+    else if (unit === "s") ms += n * 1e3;
+    else ms += n;
+  }
+  return matched ? Math.round(ms) : void 0;
+}
+function parseResetTimestamp(value) {
+  if (value === void 0) return void 0;
+  const t = Date.parse(value);
+  if (Number.isNaN(t)) return void 0;
+  return Math.max(0, t - Date.now());
+}
+function parseOpenAIStyle(h) {
+  return {
+    requestsRemaining: num(h["x-ratelimit-remaining-requests"]),
+    requestsLimit: num(h["x-ratelimit-limit-requests"]),
+    requestsResetMs: parseDuration(h["x-ratelimit-reset-requests"]),
+    tokensRemaining: num(h["x-ratelimit-remaining-tokens"]),
+    tokensLimit: num(h["x-ratelimit-limit-tokens"]),
+    tokensResetMs: parseDuration(h["x-ratelimit-reset-tokens"])
+  };
+}
+function parseAnthropic(h) {
+  return {
+    requestsRemaining: num(h["anthropic-ratelimit-requests-remaining"]),
+    requestsLimit: num(h["anthropic-ratelimit-requests-limit"]),
+    requestsResetMs: parseResetTimestamp(
+      h["anthropic-ratelimit-requests-reset"]
+    ),
+    tokensRemaining: num(h["anthropic-ratelimit-tokens-remaining"]),
+    tokensLimit: num(h["anthropic-ratelimit-tokens-limit"]),
+    tokensResetMs: parseResetTimestamp(h["anthropic-ratelimit-tokens-reset"])
+  };
+}
+function parseIetfReset(value) {
+  const reset = num(value);
+  if (reset === void 0) return void 0;
+  const YEAR_S = 31536e3;
+  if (reset > YEAR_S) {
+    return Math.max(0, reset * 1e3 - Date.now());
+  }
+  return reset * 1e3;
+}
+function parseGeneric(h) {
+  const openai = parseOpenAIStyle(h);
+  if (Object.values(openai).some((v) => v !== void 0)) return openai;
+  const reset = parseIetfReset(h["ratelimit-reset"]);
+  return {
+    requestsRemaining: num(h["ratelimit-remaining"]),
+    requestsLimit: num(h["ratelimit-limit"]),
+    ...reset !== void 0 ? { requestsResetMs: reset } : {}
+  };
+}
+function parseRateLimitHeaders(provider, headers) {
+  if (headers === void 0) return {};
+  const h = lower(headers);
+  const p = provider.toLowerCase();
+  if (p.startsWith("openai") || p.startsWith("groq")) {
+    return parseOpenAIStyle(h);
+  }
+  if (p.startsWith("anthropic")) {
+    return parseAnthropic(h);
+  }
+  return parseGeneric(h);
+}
+
+// src/limit-tracker.ts
+var MINUTE_MS = 6e4;
+var DAY_MS = 864e5;
+var DEFAULT_SNAPSHOT_TTL_MS = 6e4;
+var MAX_SNAPSHOT_TTL_MS = DAY_MS;
+var LimitTracker = class {
+  store;
+  threshold;
+  cooldown;
+  /** Per-key promise chains serializing usage counter updates. */
+  usageLocks = /* @__PURE__ */ new Map();
+  constructor(options) {
+    this.store = options.store;
+    this.threshold = options.threshold;
+    this.cooldown = options.cooldown;
+  }
+  benchKey(modelKey) {
+    return `ai-resilient:bench:${modelKey}`;
+  }
+  usageKey(modelKey) {
+    return `ai-resilient:usage:${modelKey}`;
+  }
+  headersKey(modelKey) {
+    return `ai-resilient:headers:${modelKey}`;
+  }
+  /**
+   * False if the model is benched (cooldown active) or usage is at or
+   * beyond `(1 - threshold)` of any known limit.
+   */
+  async isAvailable(modelKey, limits) {
+    try {
+      if (await this.isBenched(modelKey)) return false;
+      if (await this.isNearHeaderLimit(modelKey)) return false;
+      if (limits && await this.isNearDeclaredLimit(modelKey, limits)) {
+        return false;
+      }
+      return true;
+    } catch {
+      return true;
+    }
+  }
+  async isBenched(modelKey) {
+    const raw = await this.store.get(this.benchKey(modelKey));
+    if (raw === null) return false;
+    const expiresAt = Number(raw);
+    return Number.isFinite(expiresAt) && Date.now() < expiresAt;
+  }
+  async isNearHeaderLimit(modelKey) {
+    const raw = await this.store.get(this.headersKey(modelKey));
+    if (raw === null) return false;
+    const snapshot = JSON.parse(raw);
+    const now = Date.now();
+    for (const dim of [snapshot.requests, snapshot.tokens]) {
+      if (dim === void 0 || now >= dim.expiresAt) continue;
+      if (this.remainingBelowThreshold(dim.remaining, dim.limit)) return true;
+    }
+    return false;
+  }
+  remainingBelowThreshold(remaining, limit) {
+    if (remaining === void 0) return false;
+    if (remaining <= 0) return true;
+    if (limit === void 0 || limit <= 0) return false;
+    return remaining / limit <= this.threshold;
+  }
+  async isNearDeclaredLimit(modelKey, limits) {
+    const usage = await this.readUsage(modelKey);
+    const now = Date.now();
+    if (limits.requestsPerMinute !== void 0) {
+      const used = usage.requests.filter((t) => now - t < MINUTE_MS).length;
+      if (used >= (1 - this.threshold) * limits.requestsPerMinute) return true;
+    }
+    if (limits.requestsPerDay !== void 0) {
+      const used = usage.requests.filter((t) => now - t < DAY_MS).length;
+      if (used >= (1 - this.threshold) * limits.requestsPerDay) return true;
+    }
+    if (limits.tokensPerMinute !== void 0) {
+      const used = usage.tokens.filter(([t]) => now - t < MINUTE_MS).reduce((sum, [, n]) => sum + n, 0);
+      if (used >= (1 - this.threshold) * limits.tokensPerMinute) return true;
+    }
+    return false;
+  }
+  /**
+   * Record a successful call: parse provider rate-limit headers and
+   * increment sliding-window counters for declared limits.
+   */
+  async recordSuccess(modelKey, input) {
+    try {
+      if (input.headers !== void 0) {
+        await this.recordHeaders(modelKey, input.provider, input.headers);
+      }
+      if (input.limits !== void 0) {
+        await this.recordUsage(modelKey, input.limits, input.totalTokens);
+      }
+    } catch {
+    }
+  }
+  async recordHeaders(modelKey, provider, headers) {
+    const parsed = parseRateLimitHeaders(provider, headers);
+    const now = Date.now();
+    const snapshot = {};
+    if (parsed.requestsRemaining !== void 0) {
+      snapshot.requests = {
+        remaining: parsed.requestsRemaining,
+        limit: parsed.requestsLimit,
+        expiresAt: now + Math.min(
+          parsed.requestsResetMs ?? DEFAULT_SNAPSHOT_TTL_MS,
+          MAX_SNAPSHOT_TTL_MS
+        )
+      };
+    }
+    if (parsed.tokensRemaining !== void 0) {
+      snapshot.tokens = {
+        remaining: parsed.tokensRemaining,
+        limit: parsed.tokensLimit,
+        expiresAt: now + Math.min(
+          parsed.tokensResetMs ?? DEFAULT_SNAPSHOT_TTL_MS,
+          MAX_SNAPSHOT_TTL_MS
+        )
+      };
+    }
+    const ttl = Math.max(
+      (snapshot.requests?.expiresAt ?? 0) - now,
+      (snapshot.tokens?.expiresAt ?? 0) - now
+    );
+    if (ttl <= 0) return;
+    await this.store.set(
+      this.headersKey(modelKey),
+      JSON.stringify(snapshot),
+      ttl
+    );
+  }
+  recordUsage(modelKey, limits, totalTokens) {
+    return this.serializeUsageUpdate(modelKey, async () => {
+      const usage = await this.readUsage(modelKey);
+      const now = Date.now();
+      const trackDay = limits.requestsPerDay !== void 0 ? DAY_MS : MINUTE_MS;
+      usage.requests = usage.requests.filter((t) => now - t < trackDay);
+      usage.requests.push(now);
+      usage.tokens = usage.tokens.filter(([t]) => now - t < MINUTE_MS);
+      if (limits.tokensPerMinute !== void 0 && totalTokens !== void 0) {
+        usage.tokens.push([now, totalTokens]);
+      }
+      await this.store.set(
+        this.usageKey(modelKey),
+        JSON.stringify(usage),
+        trackDay
+      );
+    });
+  }
+  serializeUsageUpdate(modelKey, fn) {
+    const prev = this.usageLocks.get(modelKey) ?? Promise.resolve();
+    const next = prev.then(fn, fn);
+    const tail = next.then(
+      () => void 0,
+      () => void 0
+    );
+    this.usageLocks.set(modelKey, tail);
+    void tail.then(() => {
+      if (this.usageLocks.get(modelKey) === tail) {
+        this.usageLocks.delete(modelKey);
+      }
+    });
+    return next;
+  }
+  async readUsage(modelKey) {
+    const raw = await this.store.get(this.usageKey(modelKey));
+    if (raw === null) return { requests: [], tokens: [] };
+    const parsed = JSON.parse(raw);
+    return {
+      requests: Array.isArray(parsed.requests) ? parsed.requests : [],
+      tokens: Array.isArray(parsed.tokens) ? parsed.tokens : []
+    };
+  }
+  /**
+   * Bench a model after a rate-limit error, for `retryAfterMs` when the
+   * provider sent one, otherwise for the configured cooldown.
+   */
+  async recordRateLimit(modelKey, retryAfterMs) {
+    const benchMs = retryAfterMs ?? this.cooldown;
+    if (benchMs <= 0) return;
+    try {
+      await this.store.set(
+        this.benchKey(modelKey),
+        String(Date.now() + benchMs),
+        benchMs
+      );
+    } catch {
+    }
+  }
+};
+
+// src/classify-error.ts
+function getErrorShape(error) {
+  if (typeof error !== "object" || error === null) {
+    return { message: String(error) };
+  }
+  const e = error;
+  return {
+    statusCode: typeof e["statusCode"] === "number" ? e["statusCode"] : void 0,
+    responseHeaders: typeof e["responseHeaders"] === "object" && e["responseHeaders"] !== null ? e["responseHeaders"] : void 0,
+    isRetryable: typeof e["isRetryable"] === "boolean" ? e["isRetryable"] : void 0,
+    message: typeof e["message"] === "string" ? e["message"] : void 0
+  };
+}
+var RATE_LIMIT_PATTERN = /rate.?limit|too many requests|quota|exhausted|billing|insufficient.?quota/i;
+var TRANSIENT_PATTERN = /overloaded|timeout|timed.?out|econnreset|econnrefused|etimedout|socket hang up|network|fetch failed|service unavailable|internal server error|bad gateway/i;
+function getHeader(headers, name) {
+  if (!headers) return void 0;
+  for (const key of Object.keys(headers)) {
+    if (key.toLowerCase() === name) return headers[key];
+  }
+  return void 0;
+}
+function classifyError(error) {
+  const { statusCode, responseHeaders, isRetryable, message } = getErrorShape(error);
+  const msg = message ?? "";
+  if (statusCode === 429) return "rate-limit";
+  if (statusCode === 400 || statusCode === 401 || statusCode === 403) {
+    return "fatal";
+  }
+  if (statusCode !== void 0 && statusCode >= 500) return "transient";
+  if (getHeader(responseHeaders, "retry-after") !== void 0) {
+    return "rate-limit";
+  }
+  if (RATE_LIMIT_PATTERN.test(msg)) return "rate-limit";
+  if (isRetryable === true) return "transient";
+  if (TRANSIENT_PATTERN.test(msg)) return "transient";
+  return "fatal";
+}
+function getRetryAfterMs(error) {
+  const { responseHeaders } = getErrorShape(error);
+  const raw = getHeader(responseHeaders, "retry-after");
+  if (raw === void 0) return void 0;
+  const seconds = Number(raw);
+  if (Number.isFinite(seconds)) {
+    return Math.max(0, seconds * 1e3);
+  }
+  const date = Date.parse(raw);
+  if (!Number.isNaN(date)) {
+    return Math.max(0, date - Date.now());
+  }
+  return void 0;
+}
+
+// src/errors.ts
+var AllModelsExhaustedError = class extends Error {
+  name = "AllModelsExhaustedError";
+  attempts;
+  constructor(attempts) {
+    const summary = attempts.map((a) => `${a.modelId} (${a.classification})`).join(", ");
+    super(
+      `All models exhausted after ${attempts.length} attempt(s): ${summary}`
+    );
+    this.attempts = attempts;
+  }
+  static isInstance(error) {
+    return error instanceof Error && error.name === "AllModelsExhaustedError";
+  }
+};
+
+// src/resilient-model.ts
+var PRELUDE_PART_TYPES = /* @__PURE__ */ new Set(["stream-start", "response-metadata"]);
+var ResilientLanguageModel = class {
+  specificationVersion = "v2";
+  provider = "ai-resilient";
+  candidates;
+  tracker;
+  onFallback;
+  onError;
+  /**
+   * Bookkeeping writes chained off the response path. buildPlan awaits
+   * this so the next call still sees all prior usage before deciding
+   * availability (read-your-writes), without success responses paying
+   * store-write latency.
+   */
+  pendingRecords = Promise.resolve();
+  constructor(options) {
+    if (options.models.length === 0) {
+      throw new Error("ai-resilient: at least one model is required");
+    }
+    const seen = /* @__PURE__ */ new Map();
+    this.candidates = options.models.map((config) => {
+      const base = `${config.model.provider}:${config.model.modelId}`;
+      const count = seen.get(base) ?? 0;
+      seen.set(base, count + 1);
+      return { config, key: count === 0 ? base : `${base}#${count}` };
+    });
+    this.tracker = options.tracker;
+    this.onFallback = options.onFallback;
+    this.onError = options.onError;
+  }
+  get modelId() {
+    return this.primary.modelId;
+  }
+  get supportedUrls() {
+    return this.primary.supportedUrls;
+  }
+  get primary() {
+    return this.candidates[0].config.model;
+  }
+  queueRecordSuccess(candidate, headers, usage) {
+    this.pendingRecords = this.pendingRecords.then(() => this.recordSuccess(candidate, headers, usage)).catch(() => {
+    });
+  }
+  /**
+   * Order candidates and mark near-limit/benched ones as skipped. If
+   * every model would be skipped, try all of them in configured order
+   * anyway — attempting beats failing without trying.
+   */
+  async buildPlan() {
+    await this.pendingRecords;
+    const plan = [];
+    for (const candidate of this.candidates) {
+      const available = await this.tracker.isAvailable(
+        candidate.key,
+        candidate.config.limits
+      );
+      plan.push({ candidate, skip: !available });
+    }
+    if (plan.every((entry) => entry.skip)) {
+      for (const entry of plan) entry.skip = false;
+    }
+    return plan;
+  }
+  fireFallbacks(pending, to) {
+    for (const { from, reason } of pending) {
+      this.onFallback?.({ from, to, reason });
+    }
+    pending.length = 0;
+  }
+  /**
+   * Handle a failed attempt: notify, classify, bench on rate-limit, and
+   * queue the fallback notification. Rethrows fatal errors.
+   */
+  async handleFailure(error, candidate, attempts, pending) {
+    const modelId = candidate.config.model.modelId;
+    this.onError?.(error, modelId);
+    const classification = classifyError(error);
+    if (classification === "fatal") throw error;
+    attempts.push({ modelId, error, classification });
+    if (classification === "rate-limit") {
+      await this.tracker.recordRateLimit(candidate.key, getRetryAfterMs(error));
+    }
+    pending.push({ from: modelId, reason: classification });
+  }
+  async recordSuccess(candidate, headers, usage) {
+    const totalTokens = usage?.totalTokens ?? (usage?.inputTokens !== void 0 || usage?.outputTokens !== void 0 ? (usage.inputTokens ?? 0) + (usage.outputTokens ?? 0) : void 0);
+    await this.tracker.recordSuccess(candidate.key, {
+      provider: candidate.config.model.provider,
+      headers,
+      totalTokens,
+      limits: candidate.config.limits
+    });
+  }
+  async doGenerate(options) {
+    const plan = await this.buildPlan();
+    const attempts = [];
+    const pending = [];
+    for (const entry of plan) {
+      const { candidate } = entry;
+      const modelId = candidate.config.model.modelId;
+      if (entry.skip) {
+        pending.push({ from: modelId, reason: "proactive" });
+        continue;
+      }
+      this.fireFallbacks(pending, modelId);
+      try {
+        const result = await candidate.config.model.doGenerate(options);
+        this.queueRecordSuccess(
+          candidate,
+          result.response?.headers,
+          result.usage
+        );
+        return result;
+      } catch (error) {
+        await this.handleFailure(error, candidate, attempts, pending);
+      }
+    }
+    throw new AllModelsExhaustedError(attempts);
+  }
+  async doStream(options) {
+    const plan = await this.buildPlan();
+    const attempts = [];
+    const pending = [];
+    let index = 0;
+    const acquire = async () => {
+      while (index < plan.length) {
+        const entry = plan[index++];
+        const { candidate } = entry;
+        const modelId = candidate.config.model.modelId;
+        if (entry.skip) {
+          pending.push({ from: modelId, reason: "proactive" });
+          continue;
+        }
+        this.fireFallbacks(pending, modelId);
+        try {
+          const result = await candidate.config.model.doStream(options);
+          return { candidate, result };
+        } catch (error) {
+          await this.handleFailure(error, candidate, attempts, pending);
+        }
+      }
+      throw new AllModelsExhaustedError(attempts);
+    };
+    for (; ; ) {
+      const { candidate, result } = await acquire();
+      const reader = result.stream.getReader();
+      const prelude = [];
+      let firstContent;
+      let usage;
+      let done = false;
+      try {
+        for (; ; ) {
+          const { done: readDone, value } = await reader.read();
+          if (readDone) {
+            done = true;
+            break;
+          }
+          if (value.type === "error") {
+            throw value.error;
+          }
+          if (value.type === "finish") usage = value.usage;
+          if (PRELUDE_PART_TYPES.has(value.type)) {
+            prelude.push(value);
+            continue;
+          }
+          firstContent = value;
+          break;
+        }
+      } catch (error) {
+        reader.cancel().catch(() => {
+        });
+        await this.handleFailure(error, candidate, attempts, pending);
+        continue;
+      }
+      let preludeIndex = 0;
+      const stream = new ReadableStream({
+        pull: async (controller) => {
+          if (preludeIndex < prelude.length) {
+            controller.enqueue(prelude[preludeIndex++]);
+            return;
+          }
+          if (firstContent !== void 0) {
+            const part = firstContent;
+            firstContent = void 0;
+            controller.enqueue(part);
+            return;
+          }
+          if (!done) {
+            const { done: readDone, value } = await reader.read();
+            if (!readDone) {
+              if (value.type === "finish") usage = value.usage;
+              controller.enqueue(value);
+              return;
+            }
+            done = true;
+          }
+          this.queueRecordSuccess(candidate, result.response?.headers, usage);
+          controller.close();
+        },
+        cancel: (reason) => reader.cancel(reason)
+      });
+      return {
+        stream,
+        ...result.request !== void 0 ? { request: result.request } : {},
+        ...result.response !== void 0 ? { response: result.response } : {}
+      };
+    }
+  }
+};
+
+// src/stores/memory.ts
+function memoryStore() {
+  const map = /* @__PURE__ */ new Map();
+  function evictIfExpired(key) {
+    const entry = map.get(key);
+    if (entry === void 0) return void 0;
+    if (entry.expiresAt !== void 0 && Date.now() >= entry.expiresAt) {
+      map.delete(key);
+      return void 0;
+    }
+    return entry;
+  }
+  return {
+    get(key) {
+      const entry = evictIfExpired(key);
+      return Promise.resolve(entry?.value ?? null);
+    },
+    set(key, value, ttlMs) {
+      map.set(key, {
+        value,
+        ...ttlMs !== void 0 ? { expiresAt: Date.now() + ttlMs } : {}
+      });
+      return Promise.resolve();
+    }
+  };
+}
+
+// src/index.ts
+var DEFAULT_THRESHOLD = 0.1;
+var DEFAULT_COOLDOWN_MS = 6e4;
+function createResilient(options) {
+  const tracker = new LimitTracker({
+    store: options.store ?? memoryStore(),
+    threshold: options.threshold ?? DEFAULT_THRESHOLD,
+    cooldown: options.cooldown ?? DEFAULT_COOLDOWN_MS
+  });
+  return new ResilientLanguageModel({
+    models: options.models,
+    tracker,
+    ...options.onFallback !== void 0 ? { onFallback: options.onFallback } : {},
+    ...options.onError !== void 0 ? { onError: options.onError } : {}
+  });
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  AllModelsExhaustedError,
+  LimitTracker,
+  classifyError,
+  createResilient,
+  getRetryAfterMs,
+  memoryStore,
+  parseRateLimitHeaders
+});
+//# sourceMappingURL=index.cjs.map