@tokentop/ttop 0.5.0 → 0.6.0

package/src/plugins/plugin-host.test.ts

@@ -0,0 +1,197 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import { getPluginHealth, resetAllCircuits, safeInvoke } from "./plugin-host.ts";
+
+afterEach(() => {
+  resetAllCircuits();
+});
+
+// ---------------------------------------------------------------------------
+// Baseline behaviour (no isFailure)
+// ---------------------------------------------------------------------------
+
+describe("safeInvoke — baseline", () => {
+  test("returns ok:true on success", async () => {
+    const result = await safeInvoke("test-plugin", "test", async () => 42);
+    expect(result).toEqual({ ok: true, value: 42 });
+  });
+
+  test("returns ok:false on thrown error", async () => {
+    const result = await safeInvoke("test-plugin", "test", async () => {
+      throw new Error("boom");
+    });
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.error.message).toBe("boom");
+      expect(result.circuitOpen).toBe(false);
+    }
+  });
+
+  test("increments failure count on thrown errors", async () => {
+    for (let i = 0; i < 3; i++) {
+      await safeInvoke("fail-plugin", "test", async () => {
+        throw new Error("oops");
+      });
+    }
+    const health = getPluginHealth("fail-plugin");
+    expect(health.consecutiveFailures).toBe(3);
+    expect(health.totalFailures).toBe(3);
+  });
+
+  test("resets failure count on success", async () => {
+    // Fail twice
+    for (let i = 0; i < 2; i++) {
+      await safeInvoke("reset-plugin", "test", async () => {
+        throw new Error("fail");
+      });
+    }
+    expect(getPluginHealth("reset-plugin").consecutiveFailures).toBe(2);
+
+    // Succeed once — should reset
+    await safeInvoke("reset-plugin", "test", async () => "ok");
+    expect(getPluginHealth("reset-plugin").consecutiveFailures).toBe(0);
+  });
+
+  test("trips circuit after 5 consecutive failures", async () => {
+    for (let i = 0; i < 5; i++) {
+      await safeInvoke("trip-plugin", "test", async () => {
+        throw new Error("fail");
+      });
+    }
+    const health = getPluginHealth("trip-plugin");
+    expect(health.healthy).toBe(false);
+    expect(health.disabledUntil).not.toBeNull();
+
+    // Next call should be short-circuited
+    const result = await safeInvoke("trip-plugin", "test", async () => "ok");
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.circuitOpen).toBe(true);
+    }
+  });
+});
+
+// ---------------------------------------------------------------------------
+// isFailure predicate (the new feature)
+// ---------------------------------------------------------------------------
+
+interface MockUsage {
+  value: number;
+  error?: string;
+  rateLimited?: boolean;
+}
+
+describe("safeInvoke — isFailure predicate", () => {
+  test("counts logical failures when isFailure returns true", async () => {
+    const isFailure = (v: MockUsage) => !!v.error && !v.rateLimited;
+
+    const result = await safeInvoke(
+      "logical-fail",
+      "fetchUsage",
+      async () => ({ value: 0, error: "API error: 500 Internal Server Error" }),
+      { isFailure },
+    );
+
+    // Still returns ok:true so caller can inspect the error details
+    expect(result.ok).toBe(true);
+    if (result.ok) {
+      expect(result.value.error).toContain("500");
+    }
+
+    // But the circuit breaker counted it as a failure
+    const health = getPluginHealth("logical-fail");
+    expect(health.consecutiveFailures).toBe(1);
+    expect(health.totalFailures).toBe(1);
+  });
+
+  test("does NOT count rate-limited responses as failures", async () => {
+    const isFailure = (v: MockUsage) => !!v.error && !v.rateLimited;
+
+    const result = await safeInvoke(
+      "rate-limit",
+      "fetchUsage",
+      async () => ({
+        value: 0,
+        error: "Rate limited. Retry after 300s.",
+        rateLimited: true,
+      }),
+      { isFailure },
+    );
+
+    expect(result.ok).toBe(true);
+    const health = getPluginHealth("rate-limit");
+    expect(health.consecutiveFailures).toBe(0);
+    expect(health.totalFailures).toBe(0);
+  });
+
+  test("resets failures after a true success", async () => {
+    const isFailure = (v: MockUsage) => !!v.error;
+
+    // Fail 3 times logically
+    for (let i = 0; i < 3; i++) {
+      await safeInvoke(
+        "reset-after-logical",
+        "fetchUsage",
+        async () => ({ value: 0, error: "fail" }),
+        { isFailure },
+      );
+    }
+    expect(getPluginHealth("reset-after-logical").consecutiveFailures).toBe(3);
+
+    // Succeed — should reset
+    await safeInvoke("reset-after-logical", "fetchUsage", async () => ({ value: 42 }), {
+      isFailure,
+    });
+    expect(getPluginHealth("reset-after-logical").consecutiveFailures).toBe(0);
+  });
+
+  test("trips circuit after 5 consecutive logical failures", async () => {
+    const isFailure = (v: MockUsage) => !!v.error;
+
+    for (let i = 0; i < 5; i++) {
+      await safeInvoke(
+        "trip-logical",
+        "fetchUsage",
+        async () => ({ value: 0, error: `fail ${i}` }),
+        { isFailure },
+      );
+    }
+
+    const health = getPluginHealth("trip-logical");
+    expect(health.healthy).toBe(false);
+    expect(health.consecutiveFailures).toBe(5);
+    expect(health.disabledUntil).not.toBeNull();
+  });
+
+  test("mixes thrown errors and logical failures for circuit breaker", async () => {
+    const isFailure = (v: MockUsage) => !!v.error;
+
+    // 3 logical failures
+    for (let i = 0; i < 3; i++) {
+      await safeInvoke("mixed-fail", "fetchUsage", async () => ({ value: 0, error: "logical" }), {
+        isFailure,
+      });
+    }
+
+    // 2 thrown errors
+    for (let i = 0; i < 2; i++) {
+      await safeInvoke("mixed-fail", "fetchUsage", async () => {
+        throw new Error("thrown");
+      });
+    }
+
+    // Total: 5 consecutive failures → circuit should trip
+    const health = getPluginHealth("mixed-fail");
+    expect(health.consecutiveFailures).toBe(5);
+    expect(health.healthy).toBe(false);
+  });
+
+  test("does not treat successful value as failure when isFailure returns false", async () => {
+    const result = await safeInvoke("success-check", "fetchUsage", async () => ({ value: 42 }), {
+      isFailure: (v: MockUsage) => !!v.error,
+    });
+
+    expect(result.ok).toBe(true);
+    const health = getPluginHealth("success-check");
+    expect(health.consecutiveFailures).toBe(0);
+  });
+});

package/src/plugins/plugin-host.ts

@@ -52,18 +52,41 @@ export type SafeInvokeResult<T> =
   | { ok: true; value: T }
   | { ok: false; error: Error; circuitOpen: boolean };
 
+/**
+ * Options for `safeInvoke`.
+ *
+ * @template T  The return type of the plugin method.
+ */
+export interface SafeInvokeOptions<T> {
+  /**
+   * Optional predicate that inspects a *successful* return value and decides
+   * whether it should count as a logical failure for the circuit breaker.
+   *
+   * Use this when plugins return error-carrying objects instead of throwing
+   * (e.g. `{ error: "429 Too Many Requests" }`).
+   *
+   * When the predicate returns `true`:
+   *   - The circuit breaker's failure counter is incremented
+   *   - The value is still returned as `{ ok: true, value }` so the caller
+   *     can inspect the error details
+   */
+  isFailure?: (value: T) => boolean;
+}
+
 /**
  * Safely invoke a plugin method with circuit breaker protection.
  *
  * @param pluginId  Unique plugin identifier (for circuit tracking + logging)
  * @param method    Human-readable method name (for logging, e.g. "fetchUsage")
  * @param fn        The actual async function to call
+ * @param options   Optional configuration (e.g. `isFailure` predicate)
  * @returns         `{ ok: true, value }` or `{ ok: false, error, circuitOpen }`
  */
 export async function safeInvoke<T>(
   pluginId: string,
   method: string,
   fn: () => Promise<T>,
+  options?: SafeInvokeOptions<T>,
 ): Promise<SafeInvokeResult<T>> {
   const circuit = getCircuit(pluginId);
   const log = createPluginLogger(pluginId);
@@ -91,7 +114,28 @@ export async function safeInvoke<T>(
   try {
     const value = await fn();
 
-    // Success — reset consecutive failures
+    // Check for logical failures (e.g. provider returned { error: "..." })
+    if (options?.isFailure?.(value)) {
+      circuit.consecutiveFailures++;
+      circuit.totalFailures++;
+      circuit.lastFailureAt = Date.now();
+
+      log.warn(
+        `${method} returned logical failure (${circuit.consecutiveFailures}/${MAX_CONSECUTIVE_FAILURES})`,
+      );
+
+      if (circuit.consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) {
+        circuit.disabledUntil = Date.now() + COOLDOWN_MS;
+        log.error(
+          `Circuit OPEN — plugin disabled for ${COOLDOWN_MS / 1000}s after ${circuit.consecutiveFailures} consecutive failures`,
+        );
+      }
+
+      // Still return the value — caller needs the error details
+      return { ok: true, value };
+    }
+
+    // True success — reset consecutive failures
     if (circuit.consecutiveFailures > 0) {
       log.info(
         `${method} succeeded — resetting circuit breaker (was at ${circuit.consecutiveFailures} failures)`,

package/src/plugins/providers/anthropic.test.ts

@@ -0,0 +1,202 @@
+import { describe, expect, test } from "bun:test";
+import type {
+  Credentials,
+  PluginHttpClient,
+  PluginLogger,
+  ProviderFetchContext,
+} from "@tokentop/plugin-sdk";
+import { anthropicPlugin } from "./anthropic.ts";
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function createMockLogger(): PluginLogger {
+  return {
+    debug: () => {},
+    info: () => {},
+    warn: () => {},
+    error: () => {},
+  };
+}
+
+function createMockHttpClient(response: {
+  status: number;
+  statusText?: string;
+  headers?: Record<string, string>;
+  body?: unknown;
+}): PluginHttpClient {
+  return {
+    fetch: async () =>
+      new Response(JSON.stringify(response.body ?? {}), {
+        status: response.status,
+        statusText: response.statusText ?? "",
+        headers: new Headers(response.headers ?? {}),
+      }),
+  };
+}
+
+function createFetchContext(
+  overrides: { http?: PluginHttpClient; credentials?: Credentials } = {},
+): ProviderFetchContext {
+  return {
+    credentials: overrides.credentials ?? {
+      oauth: {
+        accessToken: "test-token",
+        expiresAt: Date.now() + 3_600_000, // 1 hour from now
+      },
+      source: "external",
+    },
+    http: overrides.http ?? createMockHttpClient({ status: 200, body: {} }),
+    logger: createMockLogger(),
+    config: {},
+    signal: AbortSignal.timeout(5000),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// 429 Rate limit handling
+// ---------------------------------------------------------------------------
+
+describe("anthropic fetchUsage — 429 rate limit", () => {
+  test("returns rateLimited:true with default 300s when retry-after is 0", async () => {
+    const http = createMockHttpClient({
+      status: 429,
+      statusText: "Too Many Requests",
+      headers: { "retry-after": "0" },
+      body: { error: { message: "Rate limited", type: "rate_limit_error" } },
+    });
+
+    const result = await anthropicPlugin.fetchUsage(createFetchContext({ http }));
+
+    expect(result.rateLimited).toBe(true);
+    expect(result.retryAfterMs).toBe(300_000); // 300s default when retry-after is 0
+    expect(result.error).toContain("Rate limited");
+    expect(result.error).toContain("300s");
+  });
+
+  test("returns rateLimited:true with parsed retry-after when positive", async () => {
+    const http = createMockHttpClient({
+      status: 429,
+      statusText: "Too Many Requests",
+      headers: { "retry-after": "60" },
+      body: { error: { message: "Rate limited", type: "rate_limit_error" } },
+    });
+
+    const result = await anthropicPlugin.fetchUsage(createFetchContext({ http }));
+
+    expect(result.rateLimited).toBe(true);
+    expect(result.retryAfterMs).toBe(60_000); // 60s parsed from header
+    expect(result.error).toContain("60s");
+  });
+
+  test("returns rateLimited:true with default 300s when no retry-after header", async () => {
+    const http = createMockHttpClient({
+      status: 429,
+      statusText: "Too Many Requests",
+      body: { error: { message: "Rate limited", type: "rate_limit_error" } },
+    });
+
+    const result = await anthropicPlugin.fetchUsage(createFetchContext({ http }));
+
+    expect(result.rateLimited).toBe(true);
+    expect(result.retryAfterMs).toBe(300_000); // 300s default
+  });
+
+  test("429 response always includes fetchedAt", async () => {
+    const before = Date.now();
+    const http = createMockHttpClient({
+      status: 429,
+      statusText: "Too Many Requests",
+      headers: { "retry-after": "10" },
+    });
+
+    const result = await anthropicPlugin.fetchUsage(createFetchContext({ http }));
+
+    expect(result.fetchedAt).toBeGreaterThanOrEqual(before);
+    expect(result.fetchedAt).toBeLessThanOrEqual(Date.now());
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Other HTTP error codes (regression — should NOT have rateLimited)
+// ---------------------------------------------------------------------------
+
+describe("anthropic fetchUsage — non-429 errors", () => {
+  test("401 returns error without rateLimited", async () => {
+    const http = createMockHttpClient({
+      status: 401,
+      statusText: "Unauthorized",
+    });
+
+    const result = await anthropicPlugin.fetchUsage(createFetchContext({ http }));
+
+    expect(result.error).toContain("expired or invalid");
+    expect(result.rateLimited).toBeUndefined();
+    expect(result.retryAfterMs).toBeUndefined();
+  });
+
+  test("403 returns error without rateLimited", async () => {
+    const http = createMockHttpClient({
+      status: 403,
+      statusText: "Forbidden",
+    });
+
+    const result = await anthropicPlugin.fetchUsage(createFetchContext({ http }));
+
+    expect(result.error).toContain("lacks required scope");
+    expect(result.rateLimited).toBeUndefined();
+  });
+
+  test("500 returns generic error without rateLimited", async () => {
+    const http = createMockHttpClient({
+      status: 500,
+      statusText: "Internal Server Error",
+    });
+
+    const result = await anthropicPlugin.fetchUsage(createFetchContext({ http }));
+
+    expect(result.error).toContain("500");
+    expect(result.rateLimited).toBeUndefined();
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Successful responses
+// ---------------------------------------------------------------------------
+
+describe("anthropic fetchUsage — success", () => {
+  test("parses usage data correctly", async () => {
+    const http = createMockHttpClient({
+      status: 200,
+      body: {
+        five_hour: { utilization: 45.5, resets_at: "2026-03-08T15:00:00Z" },
+        seven_day: { utilization: 72.3, resets_at: "2026-03-10T00:00:00Z" },
+      },
+    });
+
+    const result = await anthropicPlugin.fetchUsage(createFetchContext({ http }));
+
+    expect(result.error).toBeUndefined();
+    expect(result.rateLimited).toBeUndefined();
+    expect(result.limits?.primary?.usedPercent).toBe(46); // rounded from 45.5
+    expect(result.limits?.primary?.label).toBe("5-hour window");
+    expect(result.limits?.secondary?.usedPercent).toBe(72); // rounded from 72.3
+    expect(result.limits?.secondary?.label).toBe("7-day window");
+  });
+
+  test("returns API plan type for apiKey-only credentials", async () => {
+    const result = await anthropicPlugin.fetchUsage(
+      createFetchContext({
+        credentials: {
+          apiKey: "sk-test-key",
+          source: "env",
+        },
+      }),
+    );
+
+    expect(result.planType).toBe("API");
+    expect(result.allowed).toBe(true);
+    expect(result.error).toBeUndefined();
+  });
+});

package/src/plugins/providers/anthropic.ts

@@ -234,6 +234,25 @@ export const anthropicPlugin: ProviderPlugin = {
       if (!response.ok) {
         log.warn("Failed to fetch Anthropic usage", { status: response.status });
 
+        if (response.status === 429) {
+          const retryAfterHeader = response.headers.get("retry-after");
+          const parsedRetryAfter = retryAfterHeader
+            ? Number.parseInt(retryAfterHeader, 10)
+            : Number.NaN;
+          const retrySec =
+            Number.isFinite(parsedRetryAfter) && parsedRetryAfter > 0 ? parsedRetryAfter : 300;
+          log.warn("Rate limited by Anthropic API", {
+            retryAfterHeader,
+            retryAfterSec: retrySec,
+          });
+          return {
+            fetchedAt: Date.now(),
+            error: `Rate limited. Retry after ${retrySec}s.`,
+            rateLimited: true,
+            retryAfterMs: retrySec * 1000,
+          };
+        }
+
         if (response.status === 401) {
           return {
             fetchedAt: Date.now(),

package/src/plugins/providers/antigravity.ts

@@ -241,6 +241,22 @@ export const antigravityPlugin: ProviderPlugin = {
           const errorText = await response.text().catch(() => "");
           lastError = `${response.status} ${response.statusText}: ${errorText.slice(0, 100)}`;
 
+          if (response.status === 429) {
+            const retryAfterHeader = response.headers.get("retry-after");
+            const parsedRetryAfter = retryAfterHeader
+              ? Number.parseInt(retryAfterHeader, 10)
+              : Number.NaN;
+            const retrySec =
+              Number.isFinite(parsedRetryAfter) && parsedRetryAfter > 0 ? parsedRetryAfter : 300;
+            log.warn("Rate limited by Google API", { retryAfterHeader, retryAfterSec: retrySec });
+            return {
+              fetchedAt: Date.now(),
+              error: `Rate limited. Retry after ${retrySec}s.`,
+              rateLimited: true,
+              retryAfterMs: retrySec * 1000,
+            };
+          }
+
           if (response.status === 401) {
             return {
               fetchedAt: Date.now(),

package/src/plugins/providers/chutes.ts

@@ -109,6 +109,22 @@ export const chutesPlugin: ProviderPlugin = {
       if (!response.ok) {
         log.warn("Failed to fetch Chutes usage", { status: response.status });
 
+        if (response.status === 429) {
+          const retryAfterHeader = response.headers.get("retry-after");
+          const parsedRetryAfter = retryAfterHeader
+            ? Number.parseInt(retryAfterHeader, 10)
+            : Number.NaN;
+          const retrySec =
+            Number.isFinite(parsedRetryAfter) && parsedRetryAfter > 0 ? parsedRetryAfter : 300;
+          log.warn("Rate limited by Chutes API", { retryAfterHeader, retryAfterSec: retrySec });
+          return {
+            fetchedAt: Date.now(),
+            error: `Rate limited. Retry after ${retrySec}s.`,
+            rateLimited: true,
+            retryAfterMs: retrySec * 1000,
+          };
+        }
+
         if (response.status === 401 || response.status === 403) {
           return {
             fetchedAt: Date.now(),

package/src/plugins/providers/codex.ts

@@ -178,6 +178,22 @@ export const codexPlugin: ProviderPlugin = {
         const bodyText = await response.text().catch(() => "");
         log.warn("Failed to fetch ChatGPT usage", { status: response.status, body: bodyText });
 
+        if (response.status === 429) {
+          const retryAfterHeader = response.headers.get("retry-after");
+          const parsedRetryAfter = retryAfterHeader
+            ? Number.parseInt(retryAfterHeader, 10)
+            : Number.NaN;
+          const retrySec =
+            Number.isFinite(parsedRetryAfter) && parsedRetryAfter > 0 ? parsedRetryAfter : 300;
+          log.warn("Rate limited by ChatGPT API", { retryAfterHeader, retryAfterSec: retrySec });
+          return {
+            fetchedAt: Date.now(),
+            error: `Rate limited. Retry after ${retrySec}s.`,
+            rateLimited: true,
+            retryAfterMs: retrySec * 1000,
+          };
+        }
+
         if (response.status === 401) {
           return {
             fetchedAt: Date.now(),

package/src/plugins/providers/gemini.ts

@@ -212,6 +212,22 @@ export const geminiPlugin: ProviderPlugin = {
       });
 
       if (!response.ok) {
+        if (response.status === 429) {
+          const retryAfterHeader = response.headers.get("retry-after");
+          const parsedRetryAfter = retryAfterHeader
+            ? Number.parseInt(retryAfterHeader, 10)
+            : Number.NaN;
+          const retrySec =
+            Number.isFinite(parsedRetryAfter) && parsedRetryAfter > 0 ? parsedRetryAfter : 300;
+          log.warn("Rate limited by Google API", { retryAfterHeader, retryAfterSec: retrySec });
+          return {
+            fetchedAt: Date.now(),
+            error: `Rate limited. Retry after ${retrySec}s.`,
+            rateLimited: true,
+            retryAfterMs: retrySec * 1000,
+          };
+        }
+
         const errorText = await response.text().catch(() => "");
         if (response.status === 401) {
           return {

package/src/plugins/providers/github-copilot.ts

@@ -134,6 +134,22 @@ export const githubCopilotPlugin: ProviderPlugin = {
       });
 
       if (!response.ok) {
+        if (response.status === 429) {
+          const retryAfterHeader = response.headers.get("retry-after");
+          const parsedRetryAfter = retryAfterHeader
+            ? Number.parseInt(retryAfterHeader, 10)
+            : Number.NaN;
+          const retrySec =
+            Number.isFinite(parsedRetryAfter) && parsedRetryAfter > 0 ? parsedRetryAfter : 300;
+          log.warn("Rate limited by GitHub API", { retryAfterHeader, retryAfterSec: retrySec });
+          return {
+            fetchedAt: Date.now(),
+            error: `Rate limited. Retry after ${retrySec}s.`,
+            rateLimited: true,
+            retryAfterMs: retrySec * 1000,
+          };
+        }
+
         if (response.status === 404) {
           return {
             planType: "None",

package/src/plugins/providers/minimax.ts

@@ -151,6 +151,22 @@ function createMinimaxPlugin(id: string, name: string): ProviderPlugin {
         if (!response.ok) {
           log.warn("Failed to fetch MiniMax usage", { status: response.status });
 
+          if (response.status === 429) {
+            const retryAfterHeader = response.headers.get("retry-after");
+            const parsedRetryAfter = retryAfterHeader
+              ? Number.parseInt(retryAfterHeader, 10)
+              : Number.NaN;
+            const retrySec =
+              Number.isFinite(parsedRetryAfter) && parsedRetryAfter > 0 ? parsedRetryAfter : 300;
+            log.warn("Rate limited by MiniMax API", { retryAfterHeader, retryAfterSec: retrySec });
+            return {
+              fetchedAt: Date.now(),
+              error: `Rate limited. Retry after ${retrySec}s.`,
+              rateLimited: true,
+              retryAfterMs: retrySec * 1000,
+            };
+          }
+
           if (response.status === 401 || response.status === 403) {
             return {
               fetchedAt: Date.now(),