@listen-fire/anthropic 0.12.0-rc.7 → 0.12.0

package/dist/chat.d.ts

@@ -1,6 +1,10 @@
 import type Anthropic from '@anthropic-ai/sdk';
 import type { ChatModelName, Resolved } from '@listen-fire/engine/public/models';
 export type ChatProvider = {
+    /** A stable, non-reversible id of the API key this provider calls on, when
+     *  it has one: Anthropic's limits are per key, so the limiter counts and
+     *  throttles each key apart. A provider with none shares one module-wide. */
+    readonly credential?: string;
     messages: {
         create(params: Anthropic.MessageCreateParamsNonStreaming, options?: Anthropic.RequestOptions): Promise<Anthropic.Message>;
         stream(params: Anthropic.MessageCreateParamsNonStreaming, options?: Anthropic.RequestOptions): {

package/dist/client.js

@@ -10,55 +10,73 @@ const node_async_hooks_1 = require("node:async_hooks");
 const node_crypto_1 = require("node:crypto");
 const exponential_backoff_1 = require("exponential-backoff");
 const behaviour_json_1 = __importDefault(require("./behaviour.json"));
-const retry_after_1 = require("./retry_after");
 const structured_1 = require("./structured");
 const web_tools_1 = require("./web_tools");
+const keyed_provider_1 = require("./keyed_provider");
 const limiter_1 = require("@listen-fire/engine/public/limiter");
 const log_1 = require("@listen-fire/engine/public/log");
 const models_1 = require("@listen-fire/engine/public/models");
 const types_1 = require("@listen-fire/engine/public/types");
 const SECOND = 1000;
-/** Requests in flight at once: `@lf/anthropic`'s own declared limit, kept by
- *  the limiter slot (services/limiter), as lf's Rust client keeps it. A
- *  request holds its place through its retries. */
-const MODELS_REQUEST = (0, limiter_1.limitRequest)({ module: '@lf/anthropic', team: '' }, [
-    { scope: 'module', concurrent: behaviour_json_1.default.concurrentRequests },
-]);
+function modelsCaller(provider) {
+    return (0, keyed_provider_1.modelsCallerFor)(provider.credential);
+}
+/** Requests in flight at once, per key: `@lf/anthropic`'s own declared limit,
+ *  kept by the limiter slot (services/limiter), as lf's Rust client keeps it.
+ *  A request holds its place through its retries, but not through a vendor's
+ *  rate limit: that is a throttle on the caller, waited out in the queue. */
+function modelsRequest(caller) {
+    return (0, limiter_1.limitRequest)(caller, [{ scope: 'credential', concurrent: behaviour_json_1.default.concurrentRequests }]);
+}
 // behaviour.json, which the Rust lf's client is held to as well.
 const RETRY_LIMIT = behaviour_json_1.default.retry.attempts;
 const INITIAL_DELAY = behaviour_json_1.default.retry.initialDelayMs;
 const TIME_MULTIPLE = behaviour_json_1.default.retry.multiple;
-const RATE_LIMIT_DELAY = behaviour_json_1.default.retry.rateLimitWaitMs;
-async function enqueueQuery(fn, signal) {
+const RATE_LIMIT_WAIT = behaviour_json_1.default.retry.rateLimitWaitMs;
+function rateLimited(e) {
+    return e instanceof Error && e.message.includes('429');
+}
+async function enqueueQuery(provider, fn, signal) {
+    const caller = modelsCaller(provider);
+    const request = modelsRequest(caller);
     // Every request — each turn of a tool loop, each continuation — first asks
     // whether its run has spent its cost cap. Here, in the caller's own async
     // context: the queued job may be started from another job's.
     (0, models_1.assertRunBudget)();
-    return (0, limiter_1.limited)((0, limiter_1.limiterSlot)(), MODELS_REQUEST, async () => {
-        return (0, exponential_backoff_1.backOff)(fn, {
-            jitter: 'none',
-            numOfAttempts: RETRY_LIMIT,
-            startingDelay: INITIAL_DELAY,
-            timeMultiple: TIME_MULTIPLE,
-            retry: async (e, attempt) => {
-                if (signal?.aborted)
-                    return false;
-                if (e instanceof Error && e.message.includes('429')) {
-                    const asked = (0, retry_after_1.rateLimitDelayMs)(e);
-                    const wait = asked ?? RATE_LIMIT_DELAY;
-                    log_1.logger.info(`Hit model vendor rate limit, waiting ${wait / SECOND}s (${asked === undefined ? 'default' : 'from retry-after'})`);
-                    await new Promise((resolve) => setTimeout(resolve, wait));
-                }
-                else if (e instanceof Error && e.message.includes('401')) {
-                    return false;
-                }
-                if (attempt < RETRY_LIMIT) {
-                    log_1.logger.info(`Retrying Anthropic query in ${(INITIAL_DELAY / SECOND) * TIME_MULTIPLE ** (attempt - 1)}s`, { error: e });
-                }
-                return true;
-            },
-        });
-    });
+    for (let rateLimits = 1;; rateLimits++) {
+        try {
+            return await (0, limiter_1.limited)((0, limiter_1.limiterSlot)(), request, async () => {
+                return (0, exponential_backoff_1.backOff)(fn, {
+                    jitter: 'none',
+                    numOfAttempts: RETRY_LIMIT,
+                    startingDelay: INITIAL_DELAY,
+                    timeMultiple: TIME_MULTIPLE,
+                    retry: async (e, attempt) => {
+                        if (signal?.aborted)
+                            return false;
+                        // Out of the permit: the throttle below holds every request
+                        // made as this caller, not just this one.
+                        if (rateLimited(e))
+                            return false;
+                        if (e instanceof Error && e.message.includes('401'))
+                            return false;
+                        if (attempt < RETRY_LIMIT) {
+                            log_1.logger.info(`Retrying Anthropic query in ${(INITIAL_DELAY / SECOND) * TIME_MULTIPLE ** (attempt - 1)}s`, { error: e });
+                        }
+                        return true;
+                    },
+                });
+            });
+        }
+        catch (e) {
+            if (!rateLimited(e) || signal?.aborted || rateLimits >= RETRY_LIMIT)
+                throw e;
+            const asked = (0, limiter_1.vendorWaitMs)(e);
+            const wait = asked ?? RATE_LIMIT_WAIT;
+            log_1.logger.info(`Hit model vendor rate limit, throttling model requests for ${wait / SECOND}s (${asked === undefined ? 'default' : 'from retry-after'})`);
+            await (0, limiter_1.throttleAfter)((0, limiter_1.limiterSlot)(), caller, wait);
+        }
+    }
 }
 function emptyTally() {
     return {
@@ -241,7 +259,7 @@ async function anthropicToolLoop(access, params, toolImpls = {}) {
             throw new Error(`Anthropic tool loop exhausted its turn budget (maxTurns=${maxTurns})${label ? ` in ${label}` : ''} — the agent never finished.`);
         }
         const llmStart = Date.now();
-        const response = await enqueueQuery(async () => {
+        const response = await enqueueQuery(client, async () => {
             return client.messages.create({
                 model: wireModel,
                 max_tokens: max_output_tokens,
@@ -563,7 +581,7 @@ async function anthropicChatDetailed(access, options) {
             // Streamed rather than awaited whole: above ~16K `max_tokens` a
             // non-streaming request outlives the SDK's HTTP timeout. `finalMessage()`
             // reassembles the same `Message`, so nothing downstream changes.
-            response = await enqueueQuery(async () => {
+            response = await enqueueQuery(client, async () => {
                 const stream = client.messages.stream({
                     model: wireModel,
                     max_tokens: maxTokens,
@@ -704,7 +722,7 @@ async function anthropicChatStructured(access, options) {
     const { client, wireModel, resolved } = access.chatCallFor(model);
     const tool = (0, structured_1.buildStructuredTool)({ name: toolName, description: toolDescription, schema });
     const startMs = Date.now();
-    const response = await enqueueQuery(async () => client.messages.create({
+    const response = await enqueueQuery(client, async () => client.messages.create({
         model: wireModel,
         max_tokens: maxTokens,
         system: [{ type: 'text', text: system, cache_control: { type: 'ephemeral' } }],
@@ -932,7 +950,7 @@ async function anthropicWebChat(access, options) {
         const settleWatch = watchSlowCall({ ...callFields, turn });
         let response;
         try {
-            response = await enqueueQuery(async () => {
+            response = await enqueueQuery(client, async () => {
                 const stream = client.messages.stream({
                     model: wireModel,
                     max_tokens: maxTokens,

package/dist/keyed_provider.d.ts

@@ -1,7 +1,15 @@
+import { type Caller } from '@listen-fire/engine/public/limiter';
 import type { ChatProvider } from './chat';
 export interface AnthropicKey {
     apiKey: string;
     /** Another endpoint speaking the same Messages API (a proxy, a test fake). */
     baseUrl?: string;
 }
+/** What the limiter knows a key by: a hash, so neither the key nor any prefix
+ *  of it reaches a throttle key or a shared limiter's store. */
+export declare function credentialId(apiKey: string): string;
+/** Who a model request is made as: the API key it calls on (Anthropic's
+ *  limits are per key), so a 429 on one key leaves the others open. A provider
+ *  with no key of its own is one caller, module-wide. */
+export declare function modelsCallerFor(credential: string | undefined): Caller;
 export declare function anthropicKeyedChatProvider({ apiKey, baseUrl }: AnthropicKey): ChatProvider;

package/dist/keyed_provider.js

@@ -5,16 +5,48 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.credentialId = credentialId;
+exports.modelsCallerFor = modelsCallerFor;
 exports.anthropicKeyedChatProvider = anthropicKeyedChatProvider;
+const node_crypto_1 = require("node:crypto");
 const sdk_1 = __importDefault(require("@anthropic-ai/sdk"));
+const limiter_1 = require("@listen-fire/engine/public/limiter");
 // One per key rather than one per process: a unit that pays with its own key
 // (knowledge's `KNOWLEDGE_LLM_API_KEY`) must not share the platform's client.
 const keyedClients = new Map();
+/** What the limiter knows a key by: a hash, so neither the key nor any prefix
+ *  of it reaches a throttle key or a shared limiter's store. */
+function credentialId(apiKey) {
+    return `sha256:${(0, node_crypto_1.createHash)('sha256').update(apiKey).digest('hex').slice(0, 16)}`;
+}
+/** Who a model request is made as: the API key it calls on (Anthropic's
+ *  limits are per key), so a 429 on one key leaves the others open. A provider
+ *  with no key of its own is one caller, module-wide. */
+function modelsCallerFor(credential) {
+    return { module: '@lf/anthropic', team: '', ...(credential !== undefined ? { credential } : {}) };
+}
+/** Every response the key's client receives reports the key's remaining
+ *  budget; a spent one closes the key before the 429 would. A failed
+ *  response is the handler's (it reads the 429's own Retry-After). */
+function fetchReportingBudget(credential) {
+    return async (input, init) => {
+        const response = await fetch(input, init);
+        if (response.ok)
+            await (0, limiter_1.budgetAfter)((0, limiter_1.limiterSlot)(), modelsCallerFor(credential), (0, limiter_1.vendorBudget)(response.headers));
+        return response;
+    };
+}
 function anthropicKeyedChatProvider({ apiKey, baseUrl }) {
     const cacheKey = `${baseUrl ?? ''}\n${apiKey}`;
     let keyed = keyedClients.get(cacheKey);
     if (!keyed) {
-        keyed = new sdk_1.default({ apiKey, ...(baseUrl !== undefined ? { baseURL: baseUrl } : {}) });
+        // No SDK retries: a 429 must reach the module's handler at once, which
+        // releases the permit and throttles the key for everyone sharing the
+        // limiter. The SDK would sleep on it (twice) inside the held permit.
+        // The client's own backoff retries the other transient failures.
+        const credential = credentialId(apiKey);
+        const sdk = new sdk_1.default({ apiKey, maxRetries: 0, fetch: fetchReportingBudget(credential), ...(baseUrl !== undefined ? { baseURL: baseUrl } : {}) });
+        keyed = { credential, messages: sdk.messages };
         keyedClients.set(cacheKey, keyed);
     }
     return keyed;

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@listen-fire/anthropic",
   "license": "Apache-2.0",
-  "version": "0.12.0-rc.7",
+  "version": "0.12.0",
   "description": "The models slot's module for Claude (`@lf/anthropic`): the client every model call goes through (chat, structured output, the tool loop, web chat with Anthropic's hosted search and fetch), the chat seam a provider stands behind, Anthropic's API on a key the caller holds, the price table every call is billed from (prices.json, which the Rust lf embeds too), and the extraction and `AI()` clients the engine's models slot is filled with.",
   "engines": {
-    "node": "22.20.0"
+    "node": ">=22.23.4 <23"
   },
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -14,7 +14,7 @@
     "exponential-backoff": "3.1.1",
     "jsonrepair": "3.8.0",
     "zod": "4.1.12",
-    "@listen-fire/engine": "^0.12.0-rc.7"
+    "@listen-fire/engine": "^0.12.0"
   },
   "devDependencies": {
     "@types/jest": "29.5.14",
@@ -30,8 +30,7 @@
     "dist"
   ],
   "publishConfig": {
-    "access": "public",
-    "tag": "next"
+    "access": "public"
   },
   "scripts": {
     "code:type-check": "tsc --pretty --noEmit && node ../../scripts/check-module-boundary.cjs",

package/dist/retry_after.d.ts

@@ -1,6 +0,0 @@
-/**
- * How long the vendor asked a rate-limited caller to wait, or undefined when
- * it did not say. Clamped to [1s, 120s] so a hostile or absurd value can
- * neither spin nor stall a queue slot.
- */
-export declare function rateLimitDelayMs(e: unknown, now?: number): number | undefined;

package/dist/retry_after.js

@@ -1,57 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.rateLimitDelayMs = rateLimitDelayMs;
-const SECOND = 1000;
-const MIN_DELAY = 1 * SECOND;
-const MAX_DELAY = 120 * SECOND;
-function clamp(ms) {
-    return Math.min(MAX_DELAY, Math.max(MIN_DELAY, Math.round(ms)));
-}
-/** `Retry-After` is either whole seconds or an HTTP date. */
-function fromHeaderValue(value, now) {
-    const trimmed = value.trim();
-    if (/^\d+(\.\d+)?$/.test(trimmed))
-        return Number(trimmed) * SECOND;
-    const date = Date.parse(trimmed);
-    return Number.isNaN(date) ? undefined : date - now;
-}
-function readHeader(headers, name) {
-    if (!headers || typeof headers !== 'object')
-        return undefined;
-    if ('get' in headers && typeof headers.get === 'function') {
-        const value = headers.get(name);
-        return typeof value === 'string' ? value : undefined;
-    }
-    for (const [key, value] of Object.entries(headers)) {
-        if (key.toLowerCase() === name && typeof value === 'string')
-            return value;
-    }
-    return undefined;
-}
-/**
- * Google's `google.rpc.RetryInfo` detail: `"retryDelay": "17s"` (or
- * `"17.5s"`) on the wire, `{ seconds: 17 }` in a decoded form. The genai SDK's
- * `ApiError` carries the JSON body only as text in its message.
- */
-function fromRetryInfo(text) {
-    const quoted = /"retryDelay"\s*:\s*"(\d+(?:\.\d+)?)s"/.exec(text);
-    if (quoted)
-        return Number(quoted[1]) * SECOND;
-    const object = /"retryDelay"\s*:\s*\{\s*"seconds"\s*:\s*"?(\d+)"?/.exec(text);
-    return object ? Number(object[1]) * SECOND : undefined;
-}
-/**
- * How long the vendor asked a rate-limited caller to wait, or undefined when
- * it did not say. Clamped to [1s, 120s] so a hostile or absurd value can
- * neither spin nor stall a queue slot.
- */
-function rateLimitDelayMs(e, now = Date.now()) {
-    const headers = typeof e === 'object' && e !== null && 'headers' in e ? e.headers : undefined;
-    const header = readHeader(headers, 'retry-after');
-    const fromHeader = header === undefined ? undefined : fromHeaderValue(header, now);
-    if (fromHeader !== undefined)
-        return clamp(fromHeader);
-    const text = e instanceof Error ? e.message : undefined;
-    const fromBody = text === undefined ? undefined : fromRetryInfo(text);
-    return fromBody === undefined ? undefined : clamp(fromBody);
-}