@listen-fire/anthropic 0.12.0-rc.8 → 0.12.1

package/dist/chat.d.ts

@@ -1,6 +1,10 @@
 import type Anthropic from '@anthropic-ai/sdk';
 import type { ChatModelName, Resolved } from '@listen-fire/engine/public/models';
 export type ChatProvider = {
+    /** A stable, non-reversible id of the API key this provider calls on, when
+     *  it has one: Anthropic's limits are per key, so the limiter counts and
+     *  throttles each key apart. A provider with none shares one module-wide. */
+    readonly credential?: string;
     messages: {
         create(params: Anthropic.MessageCreateParamsNonStreaming, options?: Anthropic.RequestOptions): Promise<Anthropic.Message>;
         stream(params: Anthropic.MessageCreateParamsNonStreaming, options?: Anthropic.RequestOptions): {

package/dist/client.js

@@ -12,21 +12,22 @@ const exponential_backoff_1 = require("exponential-backoff");
 const behaviour_json_1 = __importDefault(require("./behaviour.json"));
 const structured_1 = require("./structured");
 const web_tools_1 = require("./web_tools");
+const keyed_provider_1 = require("./keyed_provider");
 const limiter_1 = require("@listen-fire/engine/public/limiter");
 const log_1 = require("@listen-fire/engine/public/log");
 const models_1 = require("@listen-fire/engine/public/models");
 const types_1 = require("@listen-fire/engine/public/types");
 const SECOND = 1000;
-/** Who every model request is made as: where the vendor's rate limit lands
- *  (its throttle key) and what `@lf/anthropic`'s declared limit counts. */
-const MODELS_CALLER = { module: '@lf/anthropic', team: '' };
-/** Requests in flight at once: `@lf/anthropic`'s own declared limit, kept by
- *  the limiter slot (services/limiter), as lf's Rust client keeps it. A
- *  request holds its place through its retries, but not through a vendor's
+function modelsCaller(provider) {
+    return (0, keyed_provider_1.modelsCallerFor)(provider.credential);
+}
+/** Requests in flight at once, per key: `@lf/anthropic`'s own declared limit,
+ *  kept by the limiter slot (services/limiter), as lf's Rust client keeps it.
+ *  A request holds its place through its retries, but not through a vendor's
  *  rate limit: that is a throttle on the caller, waited out in the queue. */
-const MODELS_REQUEST = (0, limiter_1.limitRequest)(MODELS_CALLER, [
-    { scope: 'module', concurrent: behaviour_json_1.default.concurrentRequests },
-]);
+function modelsRequest(caller) {
+    return (0, limiter_1.limitRequest)(caller, [{ scope: 'credential', concurrent: behaviour_json_1.default.concurrentRequests }]);
+}
 // behaviour.json, which the Rust lf's client is held to as well.
 const RETRY_LIMIT = behaviour_json_1.default.retry.attempts;
 const INITIAL_DELAY = behaviour_json_1.default.retry.initialDelayMs;
@@ -35,14 +36,16 @@ const RATE_LIMIT_WAIT = behaviour_json_1.default.retry.rateLimitWaitMs;
 function rateLimited(e) {
     return e instanceof Error && e.message.includes('429');
 }
-async function enqueueQuery(fn, signal) {
+async function enqueueQuery(provider, fn, signal) {
+    const caller = modelsCaller(provider);
+    const request = modelsRequest(caller);
     // Every request — each turn of a tool loop, each continuation — first asks
     // whether its run has spent its cost cap. Here, in the caller's own async
     // context: the queued job may be started from another job's.
     (0, models_1.assertRunBudget)();
     for (let rateLimits = 1;; rateLimits++) {
         try {
-            return await (0, limiter_1.limited)((0, limiter_1.limiterSlot)(), MODELS_REQUEST, async () => {
+            return await (0, limiter_1.limited)((0, limiter_1.limiterSlot)(), request, async () => {
                 return (0, exponential_backoff_1.backOff)(fn, {
                     jitter: 'none',
                     numOfAttempts: RETRY_LIMIT,
@@ -71,7 +74,7 @@ async function enqueueQuery(fn, signal) {
             const asked = (0, limiter_1.vendorWaitMs)(e);
             const wait = asked ?? RATE_LIMIT_WAIT;
             log_1.logger.info(`Hit model vendor rate limit, throttling model requests for ${wait / SECOND}s (${asked === undefined ? 'default' : 'from retry-after'})`);
-            await (0, limiter_1.throttleAfter)((0, limiter_1.limiterSlot)(), MODELS_CALLER, wait);
+            await (0, limiter_1.throttleAfter)((0, limiter_1.limiterSlot)(), caller, wait);
         }
     }
 }
@@ -256,7 +259,7 @@ async function anthropicToolLoop(access, params, toolImpls = {}) {
             throw new Error(`Anthropic tool loop exhausted its turn budget (maxTurns=${maxTurns})${label ? ` in ${label}` : ''} — the agent never finished.`);
         }
         const llmStart = Date.now();
-        const response = await enqueueQuery(async () => {
+        const response = await enqueueQuery(client, async () => {
             return client.messages.create({
                 model: wireModel,
                 max_tokens: max_output_tokens,
@@ -578,7 +581,7 @@ async function anthropicChatDetailed(access, options) {
             // Streamed rather than awaited whole: above ~16K `max_tokens` a
             // non-streaming request outlives the SDK's HTTP timeout. `finalMessage()`
             // reassembles the same `Message`, so nothing downstream changes.
-            response = await enqueueQuery(async () => {
+            response = await enqueueQuery(client, async () => {
                 const stream = client.messages.stream({
                     model: wireModel,
                     max_tokens: maxTokens,
@@ -719,7 +722,7 @@ async function anthropicChatStructured(access, options) {
     const { client, wireModel, resolved } = access.chatCallFor(model);
     const tool = (0, structured_1.buildStructuredTool)({ name: toolName, description: toolDescription, schema });
     const startMs = Date.now();
-    const response = await enqueueQuery(async () => client.messages.create({
+    const response = await enqueueQuery(client, async () => client.messages.create({
         model: wireModel,
         max_tokens: maxTokens,
         system: [{ type: 'text', text: system, cache_control: { type: 'ephemeral' } }],
@@ -947,7 +950,7 @@ async function anthropicWebChat(access, options) {
         const settleWatch = watchSlowCall({ ...callFields, turn });
         let response;
         try {
-            response = await enqueueQuery(async () => {
+            response = await enqueueQuery(client, async () => {
                 const stream = client.messages.stream({
                     model: wireModel,
                     max_tokens: maxTokens,

package/dist/keyed_provider.d.ts

@@ -1,7 +1,15 @@
+import { type Caller } from '@listen-fire/engine/public/limiter';
 import type { ChatProvider } from './chat';
 export interface AnthropicKey {
     apiKey: string;
     /** Another endpoint speaking the same Messages API (a proxy, a test fake). */
     baseUrl?: string;
 }
+/** What the limiter knows a key by: a hash, so neither the key nor any prefix
+ *  of it reaches a throttle key or a shared limiter's store. */
+export declare function credentialId(apiKey: string): string;
+/** Who a model request is made as: the API key it calls on (Anthropic's
+ *  limits are per key), so a 429 on one key leaves the others open. A provider
+ *  with no key of its own is one caller, module-wide. */
+export declare function modelsCallerFor(credential: string | undefined): Caller;
 export declare function anthropicKeyedChatProvider({ apiKey, baseUrl }: AnthropicKey): ChatProvider;

package/dist/keyed_provider.js

@@ -5,16 +5,48 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.credentialId = credentialId;
+exports.modelsCallerFor = modelsCallerFor;
 exports.anthropicKeyedChatProvider = anthropicKeyedChatProvider;
+const node_crypto_1 = require("node:crypto");
 const sdk_1 = __importDefault(require("@anthropic-ai/sdk"));
+const limiter_1 = require("@listen-fire/engine/public/limiter");
 // One per key rather than one per process: a unit that pays with its own key
 // (knowledge's `KNOWLEDGE_LLM_API_KEY`) must not share the platform's client.
 const keyedClients = new Map();
+/** What the limiter knows a key by: a hash, so neither the key nor any prefix
+ *  of it reaches a throttle key or a shared limiter's store. */
+function credentialId(apiKey) {
+    return `sha256:${(0, node_crypto_1.createHash)('sha256').update(apiKey).digest('hex').slice(0, 16)}`;
+}
+/** Who a model request is made as: the API key it calls on (Anthropic's
+ *  limits are per key), so a 429 on one key leaves the others open. A provider
+ *  with no key of its own is one caller, module-wide. */
+function modelsCallerFor(credential) {
+    return { module: '@lf/anthropic', team: '', ...(credential !== undefined ? { credential } : {}) };
+}
+/** Every response the key's client receives reports the key's remaining
+ *  budget; a spent one closes the key before the 429 would. A failed
+ *  response is the handler's (it reads the 429's own Retry-After). */
+function fetchReportingBudget(credential) {
+    return async (input, init) => {
+        const response = await fetch(input, init);
+        if (response.ok)
+            await (0, limiter_1.budgetAfter)((0, limiter_1.limiterSlot)(), modelsCallerFor(credential), (0, limiter_1.vendorBudget)(response.headers));
+        return response;
+    };
+}
 function anthropicKeyedChatProvider({ apiKey, baseUrl }) {
     const cacheKey = `${baseUrl ?? ''}\n${apiKey}`;
     let keyed = keyedClients.get(cacheKey);
     if (!keyed) {
-        keyed = new sdk_1.default({ apiKey, ...(baseUrl !== undefined ? { baseURL: baseUrl } : {}) });
+        // No SDK retries: a 429 must reach the module's handler at once, which
+        // releases the permit and throttles the key for everyone sharing the
+        // limiter. The SDK would sleep on it (twice) inside the held permit.
+        // The client's own backoff retries the other transient failures.
+        const credential = credentialId(apiKey);
+        const sdk = new sdk_1.default({ apiKey, maxRetries: 0, fetch: fetchReportingBudget(credential), ...(baseUrl !== undefined ? { baseURL: baseUrl } : {}) });
+        keyed = { credential, messages: sdk.messages };
         keyedClients.set(cacheKey, keyed);
     }
     return keyed;

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@listen-fire/anthropic",
   "license": "Apache-2.0",
-  "version": "0.12.0-rc.8",
+  "version": "0.12.1",
   "description": "The models slot's module for Claude (`@lf/anthropic`): the client every model call goes through (chat, structured output, the tool loop, web chat with Anthropic's hosted search and fetch), the chat seam a provider stands behind, Anthropic's API on a key the caller holds, the price table every call is billed from (prices.json, which the Rust lf embeds too), and the extraction and `AI()` clients the engine's models slot is filled with.",
   "engines": {
-    "node": "22.20.0"
+    "node": ">=22.23.4 <23"
   },
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -14,7 +14,7 @@
     "exponential-backoff": "3.1.1",
     "jsonrepair": "3.8.0",
     "zod": "4.1.12",
-    "@listen-fire/engine": "^0.12.0-rc.8"
+    "@listen-fire/engine": "^0.12.1"
   },
   "devDependencies": {
     "@types/jest": "29.5.14",
@@ -30,8 +30,7 @@
     "dist"
   ],
   "publishConfig": {
-    "access": "public",
-    "tag": "next"
+    "access": "public"
   },
   "scripts": {
     "code:type-check": "tsc --pretty --noEmit && node ../../scripts/check-module-boundary.cjs",