ag-ai 0.2.1 → 0.3.1

package/README.md

@@ -28,6 +28,33 @@ runtime imports only Node builtins.
 - Codex is not used. The executor is OpenCode; the image adapter is separate
   because OpenCode has no image-generation output.
 
+## Gemini Fallback
+
+Gemini is always the primary provider when no `endpoint` is selected. A client
+can opt into one explicit cluster fallback after every eligible Gemini
+model/key combination is quota- or capacity-exhausted, or is still in its
+cooldown window:
+
+```ts
+const client = createAIClient({
+  fallback: {
+    endpoint: { protocol: 'ag-ai', url: clusterURL, token: clusterToken },
+    model: 'gpt-5.6-luna',
+    reasoningEffort: 'xhigh',
+  },
+});
+```
+
+The fallback endpoint is never inferred. It is used only after aggregate
+Gemini exhaustion, not for validation, authentication, transport, or other
+nonquota failures. The Gemini model id and reasoning settings are not copied
+to the cluster request; `inputs`, `outputSchema`, and other request fields are
+preserved while the configured fallback model and reasoning effort are used.
+`onFallback` receives an explicit exhaustion event, and `onGenerated` reports
+the actual selected model and provider (`google` or `cluster`). Cluster
+network operations and retry waits for this fallback are individually capped
+at 3000 ms; model generation time is not included in that cap.
+
 ## Tooling
 
 Formatting and linting use `e7-oxfmt` / `e7-oxlint` from

package/dist/client/adapters/cluster.d.ts

@@ -5,6 +5,8 @@ export type ClusterAdapterConfig = {
     timeoutMs?: number;
     pollIntervalMs?: number;
     maxPollIntervalMs?: number;
+    /** Maximum time for each queue/status network operation; generation is not capped by this. */
+    networkTimeoutMs?: number;
 };
 /** Validate and normalize the direct private LAN endpoint used by the ag-ai cluster. */
 export declare const normalizeClusterEndpoint: (endpoint: AIEndpoint) => AIEndpoint;

package/dist/client/adapters/cluster.js

@@ -168,19 +168,43 @@ const responseJSON = async (response, signal) => {
         return undefined;
     }
 };
-const fetchJSON = async ({ fetch, url, headers, method, body, signal, }) => {
-    const response = await awaitWithSignal(fetch(url, {
-        method,
-        headers,
-        ...(body === undefined ? {} : { body }),
-        redirect: 'manual',
-        ...(signal === undefined ? {} : { signal }),
-    }), signal);
-    const payload = await responseJSON(response, signal);
-    if (!response.ok) {
-        throw new ClusterHTTPError(response.status, getErrorMessage(payload, `AI endpoint returned HTTP ${response.status}`));
-    }
-    return payload;
+const fetchJSON = async ({ fetch, url, headers, method, body, signal, networkTimeoutMs, }) => {
+    const controller = networkTimeoutMs === undefined ? undefined : new AbortController();
+    const timeout = controller === undefined ? undefined : setTimeout(() => controller.abort(), networkTimeoutMs);
+    const onAbort = controller === undefined ? undefined : () => controller.abort();
+    if (controller !== undefined && onAbort !== undefined) {
+        signal?.addEventListener('abort', onAbort, { once: true });
+    }
+    const requestSignal = controller?.signal ?? signal;
+    try {
+        const response = await awaitWithSignal(fetch(url, {
+            method,
+            headers,
+            ...(body === undefined ? {} : { body }),
+            redirect: 'manual',
+            ...(requestSignal === undefined ? {} : { signal: requestSignal }),
+        }), requestSignal);
+        const payload = await responseJSON(response, requestSignal);
+        if (!response.ok) {
+            throw new ClusterHTTPError(response.status, getErrorMessage(payload, `AI endpoint returned HTTP ${response.status}`));
+        }
+        return payload;
+    }
+    catch (error) {
+        if (signal?.aborted)
+            throw createAbortError();
+        if (controller?.signal.aborted) {
+            throw new Error(`AI endpoint network request timed out after ${networkTimeoutMs}ms`);
+        }
+        throw error;
+    }
+    finally {
+        if (timeout !== undefined)
+            clearTimeout(timeout);
+        if (controller !== undefined && onAbort !== undefined) {
+            signal?.removeEventListener('abort', onAbort);
+        }
+    }
 };
 const isRetryable = (error) => {
     if (error instanceof ClusterHTTPError)
@@ -390,6 +414,7 @@ const generateCluster = async (request, config) => {
             method: 'POST',
             body: JSON.stringify(body),
             signal: operationAbort.signal,
+            networkTimeoutMs: config.networkTimeoutMs,
         }), dependencies.sleep, operationAbort.signal));
         jobId = getString(created, 'id');
         if (!jobId)
@@ -430,6 +455,7 @@ const generateCluster = async (request, config) => {
                     headers: endpointHeaders(endpoint),
                     method: 'GET',
                     signal: operationAbort.signal,
+                    networkTimeoutMs: config.networkTimeoutMs,
                 }));
             }
             catch (error) {

package/dist/client/adapters/google.d.ts

@@ -5,6 +5,15 @@ export type GoogleAdapterConfig = {
     dependencies?: AIClientDependencies;
     createAI?: (apiKey: string) => GoogleGenAI;
 };
+/** Every eligible Gemini combination was exhausted by quota/capacity or cooldown. */
+export declare class GeminiPoolExhaustedError extends Error {
+    readonly causes: readonly unknown[];
+    readonly models: readonly string[];
+    constructor(options?: {
+        causes?: readonly unknown[];
+        models?: readonly string[];
+    });
+}
 export declare const generateGoogle: (request: AIRequest, config?: GoogleAdapterConfig) => Promise<GenerationResult>;
 export declare const listGoogleModels: (dependencies?: AIClientDependencies) => Promise<AIModel[]>;
 export { resetModelBackoff };

package/dist/client/adapters/google.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.resetGoogleAdapterForTests = exports.resetModelBackoff = exports.listGoogleModels = exports.generateGoogle = void 0;
+exports.resetGoogleAdapterForTests = exports.resetModelBackoff = exports.listGoogleModels = exports.generateGoogle = exports.GeminiPoolExhaustedError = void 0;
 const genai_1 = require("@google/genai");
 const apikey_1 = require("../../internal/apikey");
 const log_1 = require("../../internal/log");
@@ -25,6 +25,18 @@ const NON_TEXT_MODEL_CAPABILITIES = [
     'robotics',
     'tts',
 ];
+/** Every eligible Gemini combination was exhausted by quota/capacity or cooldown. */
+class GeminiPoolExhaustedError extends Error {
+    causes;
+    models;
+    constructor(options = {}) {
+        super('Gemini model pool exhausted; all eligible combinations are unavailable');
+        this.name = 'GeminiPoolExhaustedError';
+        this.causes = options.causes ?? [];
+        this.models = options.models ?? [];
+    }
+}
+exports.GeminiPoolExhaustedError = GeminiPoolExhaustedError;
 let genAIs = new Map();
 const geminiModelsCache = new node_cache_1.TypedNodeCache({ stdTTL: 86400 });
 const geminiModelsCacheKey = 'gemini-models-v3';
@@ -212,15 +224,27 @@ const getCombinations = async (model, prefer, output, signal, config) => {
     if (models.length === 0)
         throw new Error('No text generation models are available');
     const combinations = [];
+    let cooldownOnly = false;
+    let nonCooldownBlock = false;
     for (const [key, ai] of clients) {
         for (const selectedModel of models) {
-            const available = (0, apikey_1.getAvailableCombinations)(`gemini-${selectedModel}`, false);
+            const service = `gemini-${selectedModel}`;
+            const status = (0, quota_1.getModelKeyBackoffStatus)(selectedModel, key, config.dependencies?.now?.() ?? Date.now());
+            if (status === 'expired')
+                (0, apikey_1.clearKeyServiceBlock)(key, service);
+            const available = (0, apikey_1.getAvailableCombinations)(service, false);
             if (available.some((entry) => entry.key === key)) {
                 combinations.push([key, ai, selectedModel]);
             }
+            else if (status === 'active') {
+                cooldownOnly = true;
+            }
+            else {
+                nonCooldownBlock = true;
+            }
         }
     }
-    return combinations;
+    return { combinations, models, cooldownOnly: cooldownOnly && !nonCooldownBlock, nonCooldownBlock };
 };
 const mediaType = (mimeType) => {
     const normalized = mimeType.toLowerCase();
@@ -379,22 +403,37 @@ const generateGoogle = async (request, config = {}) => {
                 setTimeout(resolve, ms);
             }));
         const result = await (0, retryOnError_1.retryOnError)(`generation:${request.ident ?? 'unknown'}`, async () => {
-            const combinations = await getCombinations(request.model, request.prefer, request.output, controller.signal, config);
-            if (combinations.length === 0)
+            const combinationSet = await getCombinations(request.model, request.prefer, request.output, controller.signal, config);
+            const combinations = combinationSet.combinations;
+            if (combinations.length === 0) {
+                if (combinationSet.cooldownOnly) {
+                    throw new GeminiPoolExhaustedError({
+                        models: combinationSet.models,
+                    });
+                }
                 throw new Error('No available model for this request');
+            }
             let lastFailure;
+            const exhaustionCauses = [];
+            const exhaustedModels = new Set();
+            let attemptedCombination = false;
+            let nonExhaustionFailure;
+            let skippedForNonCooldown = false;
             for (const [key, ai, selectedModel] of combinations) {
-                if (!(0, apikey_1.getAvailableCombinations)(`gemini-${selectedModel}`, false).some((entry) => entry.key === key)) {
+                const service = `gemini-${selectedModel}`;
+                const nowMs = config.dependencies?.now?.() ?? Date.now();
+                const keyBackoffStatus = (0, quota_1.getModelKeyBackoffStatus)(selectedModel, key, nowMs);
+                if (keyBackoffStatus === 'active') {
+                    (0, log_1.debug)(`[${selectedModel}] skipping rate-limited model/key`, {
+                        remainingMs: (0, quota_1.modelKeyBackoffRemainingMs)(selectedModel, key, nowMs),
+                    });
                     continue;
                 }
-                // Reactive per-model backoff: a 429 skips the model until its window
-                // elapses while every other model keeps serving.
-                if ((0, quota_1.isModelBackedOff)(selectedModel)) {
-                    (0, log_1.debug)(`[${selectedModel}] skipping rate-limited model`, {
-                        remainingMs: (0, quota_1.modelBackoffRemainingMs)(selectedModel),
-                    });
+                if (!(0, apikey_1.getAvailableCombinations)(service, false).some((entry) => entry.key === key)) {
+                    skippedForNonCooldown = true;
                     continue;
                 }
+                attemptedCombination = true;
                 const selectedRequest = {
                     ...requestConfig,
                     model: selectedModel,
@@ -423,22 +462,39 @@ const generateGoogle = async (request, config = {}) => {
                     const unavailableModel = failureStatus === 404 && message.includes(selectedModel);
                     // Fallback only; per-model backoff below is keyed by rate limiting.
                     if (rateLimited || overloadedModel || unavailableModel) {
-                        (0, log_1.warn)(`[${selectedModel}] generation attempt failed; trying next available combination`, {
+                        (0, log_1.debug)(`[${selectedModel}] generation attempt failed; trying next available combination`, {
                             status: failureStatus,
                         });
                         (0, apikey_1.blockKeyService)(key, `gemini-${selectedModel}`);
                         if (rateLimited || overloadedModel) {
-                            (0, quota_1.recordModelRateLimit)(selectedModel, {
+                            (0, quota_1.recordModelKeyRateLimit)(selectedModel, key, {
                                 cause: error,
                                 nowMs: config.dependencies?.now?.(),
                             });
                         }
+                        if (unavailableModel)
+                            nonExhaustionFailure = error;
+                        if (rateLimited || overloadedModel)
+                            exhaustionCauses.push(error);
+                        if (!unavailableModel)
+                            exhaustedModels.add(selectedModel);
                         lastFailure = error;
                         continue;
                     }
                     throw error;
                 }
             }
+            if (nonExhaustionFailure !== undefined)
+                throw nonExhaustionFailure;
+            if (skippedForNonCooldown || combinationSet.nonCooldownBlock) {
+                throw new Error('No available model for this request');
+            }
+            if (!attemptedCombination || exhaustionCauses.length > 0) {
+                throw new GeminiPoolExhaustedError({
+                    causes: exhaustionCauses,
+                    models: [...exhaustedModels],
+                });
+            }
             throw lastFailure ?? new Error('No available model for this request');
         }, {
             retries: 1,
@@ -448,6 +504,7 @@ const generateGoogle = async (request, config = {}) => {
             sleepFn: async (ms) => {
                 await sleep(ms);
             },
+            errorCheck: (error) => !(error instanceof GeminiPoolExhaustedError) && (0, retryOnError_1.isRetryableApiError)(error),
         });
         return result;
     }

package/dist/client/client.d.ts

@@ -1,4 +1,5 @@
-import type { AIClient, AIClientDefaults, AIClientDependencies, AIEndpoint, AIInput, AIModel, AIModelPreference, AIPart, AIRequest, BinaryMedia, GenerateTextOptions, GenerationResult, PromptDirectOptions, PromptImageOptions } from './types';
+import type { AIClient, AIClientDefaults, AIClientDependencies, AIEndpoint, AIInput, AIModel, AIModelPreference, AIProvider, AIPart, AIRequest, BinaryMedia, GenerateTextOptions, GenerationResult, PromptDirectOptions, PromptImageOptions } from './types';
+export { GeminiPoolExhaustedError } from './adapters/google';
 export declare const createAIClient: (defaults?: AIClientDefaults) => AIClient;
 export declare const generate: (request: AIRequest) => Promise<GenerationResult>;
 export declare function generateText(prompt: string, options?: GenerateTextOptions): Promise<string>;
@@ -8,4 +9,4 @@ export declare const promptImage: (options: PromptImageOptions) => Promise<strin
 export declare const models: (options?: {
     endpoint?: AIEndpoint;
 }) => Promise<AIModel[]>;
-export type { AIClient, AIClientDefaults, AIClientDependencies, AIEndpoint, AIInput, AIModel, AIModelPreference, AIPart, AIRequest, BinaryMedia, GenerateTextOptions, GenerationResult, PromptDirectOptions, PromptImageOptions, };
+export type { AIClient, AIClientDefaults, AIClientDependencies, AIEndpoint, AIInput, AIProvider, AIModel, AIModelPreference, AIPart, AIRequest, BinaryMedia, GenerateTextOptions, GenerationResult, PromptDirectOptions, PromptImageOptions, };

package/dist/client/client.js

@@ -1,11 +1,14 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.models = exports.promptImage = exports.promptDirect = exports.generate = exports.createAIClient = void 0;
+exports.models = exports.promptImage = exports.promptDirect = exports.generate = exports.createAIClient = exports.GeminiPoolExhaustedError = void 0;
 exports.generateText = generateText;
 const cluster_1 = require("./adapters/cluster");
 const google_1 = require("./adapters/google");
+var google_2 = require("./adapters/google");
+Object.defineProperty(exports, "GeminiPoolExhaustedError", { enumerable: true, get: function () { return google_2.GeminiPoolExhaustedError; } });
 const MAX_DOWNLOAD_BYTES = 25 * 1024 * 1024;
 const DOWNLOAD_TIMEOUT_MS = 30 * 1000;
+const FALLBACK_NETWORK_TIMEOUT_MS = 3000;
 const defaultSleep = (milliseconds) => new Promise((resolve) => {
     setTimeout(resolve, milliseconds);
 });
@@ -112,6 +115,7 @@ const mergeRequest = (defaults, request) => ({
     outputSchema: request.outputSchema ?? defaults.outputSchema,
     webSearch: request.webSearch ?? defaults.webSearch,
     onGenerated: request.onGenerated ?? defaults.onGenerated,
+    onFallback: request.onFallback ?? defaults.onFallback,
     signal: request.signal ?? defaults.signal,
     timeoutMs: request.timeoutMs ?? defaults.timeoutMs,
     staggerMs: request.staggerMs ?? defaults.staggerMs,
@@ -133,16 +137,55 @@ const createAIClient = (defaults = {}) => {
     const dependencies = getDependencies(defaults);
     const generate = async (request) => {
         const merged = mergeRequest(defaults, request);
-        const result = merged.endpoint === undefined
-            ? await (0, google_1.generateGoogle)(merged, { dependencies })
-            : await (0, cluster_1.generateCluster)(merged, {
+        let result;
+        let provider;
+        if (merged.endpoint === undefined) {
+            try {
+                result = await (0, google_1.generateGoogle)(merged, { dependencies });
+                provider = 'google';
+            }
+            catch (error) {
+                if (!(error instanceof google_1.GeminiPoolExhaustedError) || defaults.fallback === undefined) {
+                    throw error;
+                }
+                const fallback = defaults.fallback;
+                const fallbackPollIntervalMs = Math.min(defaults.pollIntervalMs ?? 250, 3000);
+                const fallbackMaxPollIntervalMs = Math.min(Math.max(fallbackPollIntervalMs, defaults.maxPollIntervalMs ?? 2000), FALLBACK_NETWORK_TIMEOUT_MS);
+                const fallbackRequest = {
+                    ...merged,
+                    endpoint: fallback.endpoint,
+                    model: fallback.model,
+                    reasoningEffort: fallback.reasoningEffort,
+                };
+                const fallbackEvent = {
+                    from: { provider: 'google' },
+                    to: { provider: 'cluster', model: fallback.model },
+                    reason: 'gemini-pool-exhausted',
+                    cause: error,
+                };
+                merged.onFallback?.(fallbackEvent);
+                result = await (0, cluster_1.generateCluster)(fallbackRequest, {
+                    endpoint: fallback.endpoint,
+                    dependencies,
+                    timeoutMs: merged.timeoutMs,
+                    pollIntervalMs: fallbackPollIntervalMs,
+                    maxPollIntervalMs: fallbackMaxPollIntervalMs,
+                    networkTimeoutMs: FALLBACK_NETWORK_TIMEOUT_MS,
+                });
+                provider = 'cluster';
+            }
+        }
+        else {
+            result = await (0, cluster_1.generateCluster)(merged, {
                 endpoint: merged.endpoint,
                 dependencies,
-                timeoutMs: defaults.timeoutMs,
+                timeoutMs: merged.timeoutMs,
                 pollIntervalMs: defaults.pollIntervalMs,
                 maxPollIntervalMs: defaults.maxPollIntervalMs,
             });
-        merged.onGenerated?.({ model: result.model, generatedAt: result.generatedAt });
+            provider = 'cluster';
+        }
+        merged.onGenerated?.({ model: result.model, generatedAt: result.generatedAt, provider });
         return result;
     };
     async function generateText(promptOrOptions, options) {

package/dist/client/quota.d.ts

@@ -16,7 +16,7 @@ export declare class QuotaDeferredError extends Error {
         cause?: unknown;
     });
 }
-/** True when any message in the cause chain signals model quota/capacity exhaustion. */
+/** True when a cause chain signals model quota/capacity exhaustion. */
 export declare const isQuotaExhaustedError: (cause: unknown) => boolean;
 /**
  * Honor the provider's retry delay when present (observed 9-57s). Returns undefined
@@ -34,6 +34,21 @@ export declare const MODEL_RATE_LIMIT_BASE_DELAY_MS = 60000;
 export declare const MODEL_RATE_LIMIT_MAX_DELAY_MS: number;
 /** Small jitter so backed-off models do not resurface in lockstep. */
 export declare const MODEL_RATE_LIMIT_JITTER_MS = 1000;
+export type ModelKeyBackoffStatus = 'active' | 'expired' | 'none';
+/** Inspect a model/key cooldown without exposing the credential in state or logs. */
+export declare const getModelKeyBackoffStatus: (model: string, apiKey: string, nowMs?: number) => ModelKeyBackoffStatus;
+/** Record a quota/capacity response for one model/key combination. */
+export declare const recordModelKeyRateLimit: (model: string, apiKey: string, opt?: {
+    cause?: unknown;
+    nowMs?: number;
+    random?: () => number;
+}) => number;
+/** True while one model/key combination remains inside its cooldown window. */
+export declare const isModelKeyBackedOff: (model: string, apiKey: string, nowMs?: number) => boolean;
+/** Remaining cooldown for one model/key combination, or 0 when eligible. */
+export declare const modelKeyBackoffRemainingMs: (model: string, apiKey: string, nowMs?: number) => number;
+/** Mark one model/key combination eligible again. */
+export declare const clearModelKeyBackoff: (model: string, apiKey: string) => void;
 /**
  * Record a rate-limit response for a model. The model is skipped until the
  * backoff window elapses: the provider's own retry delay when present,

package/dist/client/quota.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.resetModelBackoff = exports.clearModelBackoff = exports.modelBackoffRemainingMs = exports.isModelBackedOff = exports.recordModelRateLimit = exports.MODEL_RATE_LIMIT_JITTER_MS = exports.MODEL_RATE_LIMIT_MAX_DELAY_MS = exports.MODEL_RATE_LIMIT_BASE_DELAY_MS = exports.quotaBackoffDelayMs = exports.getQuotaRetryDelayMs = exports.isQuotaExhaustedError = exports.QuotaDeferredError = exports.AI_QUOTA_EXHAUSTED_MESSAGE = void 0;
+exports.resetModelBackoff = exports.clearModelBackoff = exports.modelBackoffRemainingMs = exports.isModelBackedOff = exports.recordModelRateLimit = exports.clearModelKeyBackoff = exports.modelKeyBackoffRemainingMs = exports.isModelKeyBackedOff = exports.recordModelKeyRateLimit = exports.getModelKeyBackoffStatus = exports.MODEL_RATE_LIMIT_JITTER_MS = exports.MODEL_RATE_LIMIT_MAX_DELAY_MS = exports.MODEL_RATE_LIMIT_BASE_DELAY_MS = exports.quotaBackoffDelayMs = exports.getQuotaRetryDelayMs = exports.isQuotaExhaustedError = exports.QuotaDeferredError = exports.AI_QUOTA_EXHAUSTED_MESSAGE = void 0;
+const node_crypto_1 = require("node:crypto");
 const retryOnError_1 = require("../internal/retryOnError");
 /**
  * Reactive per-model rate-limit backoff.
@@ -26,8 +27,20 @@ class QuotaDeferredError extends Error {
     }
 }
 exports.QuotaDeferredError = QuotaDeferredError;
-/** True when any message in the cause chain signals model quota/capacity exhaustion. */
-const isQuotaExhaustedError = (cause) => (0, retryOnError_1.isOverloadedApiKeyError)(cause);
+const hasRateLimitStatus = (cause, seen = new Set()) => {
+    if (cause === null || cause === undefined || seen.has(cause))
+        return false;
+    if (typeof cause !== 'object')
+        return false;
+    seen.add(cause);
+    const record = cause;
+    const status = record.status ?? record.statusCode ?? record.code;
+    if (status === 429 || status === '429')
+        return true;
+    return [record.cause, record.error, record.response].some((nested) => hasRateLimitStatus(nested, seen));
+};
+/** True when a cause chain signals model quota/capacity exhaustion. */
+const isQuotaExhaustedError = (cause) => hasRateLimitStatus(cause) || (0, retryOnError_1.isOverloadedApiKeyError)(cause);
 exports.isQuotaExhaustedError = isQuotaExhaustedError;
 const RETRY_DELAY_PATTERNS = [
     /"retryDelay"\s*:\s*"(\d+(?:\.\d+)?)s"/i,
@@ -95,6 +108,54 @@ exports.MODEL_RATE_LIMIT_MAX_DELAY_MS = 15 * 60_000;
 exports.MODEL_RATE_LIMIT_JITTER_MS = 1_000;
 const modelBackoffUntilMs = new Map();
 const modelBackoffHits = new Map();
+const modelKeyBackoffUntilMs = new Map();
+const modelKeyBackoffHits = new Map();
+const modelKeyBackoffId = (model, apiKey) => `${model}:${(0, node_crypto_1.createHash)('sha256').update(apiKey).digest('hex').slice(0, 16)}`;
+/** Inspect a model/key cooldown without exposing the credential in state or logs. */
+const getModelKeyBackoffStatus = (model, apiKey, nowMs = Date.now()) => {
+    const id = modelKeyBackoffId(model, apiKey);
+    const until = modelKeyBackoffUntilMs.get(id);
+    if (until === undefined)
+        return 'none';
+    if (nowMs >= until) {
+        modelKeyBackoffUntilMs.delete(id);
+        modelKeyBackoffHits.delete(id);
+        return 'expired';
+    }
+    return 'active';
+};
+exports.getModelKeyBackoffStatus = getModelKeyBackoffStatus;
+/** Record a quota/capacity response for one model/key combination. */
+const recordModelKeyRateLimit = (model, apiKey, opt) => {
+    const now = opt?.nowMs ?? Date.now();
+    const id = modelKeyBackoffId(model, apiKey);
+    const serverDelay = opt?.cause === undefined ? undefined : (0, exports.getQuotaRetryDelayMs)(opt.cause);
+    const hits = (modelKeyBackoffHits.get(id) ?? 0) + 1;
+    modelKeyBackoffHits.set(id, hits);
+    const exponential = Math.min(exports.MODEL_RATE_LIMIT_BASE_DELAY_MS * 2 ** (hits - 1), exports.MODEL_RATE_LIMIT_MAX_DELAY_MS);
+    const jitter = Math.floor((opt?.random ?? Math.random)() * exports.MODEL_RATE_LIMIT_JITTER_MS);
+    const until = now + Math.max(serverDelay ?? 0, exponential) + jitter;
+    modelKeyBackoffUntilMs.set(id, until);
+    return until;
+};
+exports.recordModelKeyRateLimit = recordModelKeyRateLimit;
+/** True while one model/key combination remains inside its cooldown window. */
+const isModelKeyBackedOff = (model, apiKey, nowMs) => (0, exports.getModelKeyBackoffStatus)(model, apiKey, nowMs) === 'active';
+exports.isModelKeyBackedOff = isModelKeyBackedOff;
+/** Remaining cooldown for one model/key combination, or 0 when eligible. */
+const modelKeyBackoffRemainingMs = (model, apiKey, nowMs = Date.now()) => {
+    if ((0, exports.getModelKeyBackoffStatus)(model, apiKey, nowMs) !== 'active')
+        return 0;
+    return Math.max(0, (modelKeyBackoffUntilMs.get(modelKeyBackoffId(model, apiKey)) ?? nowMs) - nowMs);
+};
+exports.modelKeyBackoffRemainingMs = modelKeyBackoffRemainingMs;
+/** Mark one model/key combination eligible again. */
+const clearModelKeyBackoff = (model, apiKey) => {
+    const id = modelKeyBackoffId(model, apiKey);
+    modelKeyBackoffUntilMs.delete(id);
+    modelKeyBackoffHits.delete(id);
+};
+exports.clearModelKeyBackoff = clearModelKeyBackoff;
 /**
  * Record a rate-limit response for a model. The model is skipped until the
  * backoff window elapses: the provider's own retry delay when present,
@@ -144,5 +205,7 @@ exports.clearModelBackoff = clearModelBackoff;
 const resetModelBackoff = () => {
     modelBackoffUntilMs.clear();
     modelBackoffHits.clear();
+    modelKeyBackoffUntilMs.clear();
+    modelKeyBackoffHits.clear();
 };
 exports.resetModelBackoff = resetModelBackoff;

package/dist/client/types.d.ts

@@ -1,5 +1,6 @@
 export type AIOutputType = 'text' | 'image' | 'audio' | 'video' | 'file';
 export type AIModelPreference = 'fast' | 'quality';
+export type AIProvider = 'google' | 'cluster';
 export type AIInputRole = 'user' | 'assistant' | 'system' | 'developer';
 export type AITextPart = {
     type: 'text';
@@ -29,6 +30,8 @@ export type AIOutputSchema = {
 export type GenerationMetadata = {
     model: string;
     generatedAt: number;
+    /** Present on client provenance callbacks; raw adapters may omit it. */
+    provider?: AIProvider;
 };
 export type GenerationUsage = {
     inputTokens: number;
@@ -67,6 +70,7 @@ export type AIRequest = {
     outputSchema?: AIOutputSchema;
     webSearch?: boolean;
     onGenerated?: (metadata: GenerationMetadata) => void;
+    onFallback?: (event: AIFallbackEvent) => void;
     signal?: AbortSignal;
     timeoutMs?: number;
     /**
@@ -76,6 +80,22 @@ export type AIRequest = {
      */
     staggerMs?: number;
 };
+export type AIFallback = {
+    endpoint: AIEndpoint;
+    model: string;
+    reasoningEffort: string;
+};
+export type AIFallbackEvent = {
+    from: {
+        provider: 'google';
+    };
+    to: {
+        provider: 'cluster';
+        model: string;
+    };
+    reason: 'gemini-pool-exhausted';
+    cause: unknown;
+};
 export type BinaryMedia = {
     arraybuffer: ArrayBuffer;
     type: string;
@@ -107,6 +127,7 @@ export type AIClientDependencies = {
 };
 export type AIClientDefaults = AIRequestOptions & AIClientDependencies & {
     dependencies?: AIClientDependencies;
+    fallback?: AIFallback;
     pollIntervalMs?: number;
     maxPollIntervalMs?: number;
 };

package/dist/internal/apikey.d.ts

@@ -3,3 +3,5 @@ export declare const getAvailableCombinations: (service: string, resetIfExhauste
     key: string;
 }[];
 export declare const blockKeyService: (apiKey: string, service: string) => void;
+/** Remove only one key+service block, preserving every unrelated blocklist entry. */
+export declare const clearKeyServiceBlock: (apiKey: string, service: string) => void;

package/dist/internal/apikey.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.blockKeyService = exports.getAvailableCombinations = void 0;
+exports.clearKeyServiceBlock = exports.blockKeyService = exports.getAvailableCombinations = void 0;
 const array_1 = require("./array");
 const log_1 = require("./log");
 const node_cache_1 = require("./node-cache");
@@ -33,7 +33,7 @@ const getAvailableCombinations = (service, resetIfExhausted = true) => {
     }
     // If all combinations are blocklisted, clear the blocklist and try again
     if (combinations.length === 0 && resetIfExhausted) {
-        (0, log_1.warn)(`[${service}] all API key + service combinations were blocklisted; clearing blocklist and trying again.`);
+        (0, log_1.debug)(`[${service}] all API key + service combinations were blocklisted; clearing blocklist and trying again.`);
         blocklist.flushAll();
         for (const key of keys) {
             combinations.push({ service, key });
@@ -45,7 +45,12 @@ const getAvailableCombinations = (service, resetIfExhausted = true) => {
 exports.getAvailableCombinations = getAvailableCombinations;
 // Block a key+service combination
 const blockKeyService = (apiKey, service) => {
-    (0, log_1.warn)(`[${service}] key+service blocklisted`);
+    (0, log_1.debug)(`[${service}] key+service blocklisted`);
     addToBlocklist(apiKey, service);
 };
 exports.blockKeyService = blockKeyService;
+/** Remove only one key+service block, preserving every unrelated blocklist entry. */
+const clearKeyServiceBlock = (apiKey, service) => {
+    blocklist.del(getBlocklistKey(apiKey, service));
+};
+exports.clearKeyServiceBlock = clearKeyServiceBlock;

package/dist/k8s/opencode.js

@@ -40,7 +40,22 @@ const opencodeStack = (context) => {
             (0, common_1.env)('XDG_CACHE_HOME', '/data/opencode/cache'),
             // The unrestricted /provider response processes thousands of models from
             // unconfigured providers on every request, exceeding the runner's budget.
-            (0, common_1.env)('OPENCODE_CONFIG_CONTENT', JSON.stringify({ enabled_providers: ['openai', 'google'] })),
+            // Each advertised model carries the reasoning effort its consumer asks
+            // for: sol at high for draft copy, 5.5 at high for draft images, and luna
+            // at max for the client's cluster fallback. OpenCode merges these entries
+            // with its model catalogue; it does not replace it.
+            (0, common_1.env)('OPENCODE_CONFIG_CONTENT', JSON.stringify({
+                enabled_providers: ['openai', 'google'],
+                provider: {
+                    openai: {
+                        models: {
+                            'gpt-5.6-sol': { options: { reasoningEffort: 'high' } },
+                            'gpt-5.6-luna': { options: { reasoningEffort: 'max' } },
+                            'gpt-5.5': { options: { reasoningEffort: 'high' } },
+                        },
+                    },
+                },
+            })),
         ],
         startupProbe: { ...probe, failureThreshold: 30 },
         readinessProbe: { ...(0, common_1.openCodeHealthProbe)(common_1.OPENCODE_PORT, 'models'), initialDelaySeconds: 5 },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ag-ai",
-  "version": "0.2.1",
+  "version": "0.3.1",
   "description": "Provider-neutral AI client and cluster runtime for the rpi-mega cluster.",
   "license": "ISC",
   "author": "admin@gec.dev",

package/runtime/executor/opencode.mjs

@@ -191,6 +191,26 @@ const usageFromTokens = (tokens) => {
   return { inputTokens, outputTokens, totalTokens: inputTokens + outputTokens };
 };
 
+/**
+ * Read the reasoning efforts a model declares in the OpenCode provider catalog.
+ * OpenCode stores the effort on the model's `options`, and newer builds may also
+ * expose an explicit list. Missing metadata falls back to the executor default
+ * rather than inventing capabilities.
+ */
+const reasoningEffortsFromModel = (model) => {
+  const efforts = [];
+  const configured = asObject(model?.options)?.reasoningEffort;
+  if (typeof configured === 'string' && configured.length > 0) efforts.push(configured);
+  for (const source of [model?.supportedReasoningEfforts, model?.reasoningEfforts]) {
+    if (!Array.isArray(source)) continue;
+    for (const item of source) {
+      const value = typeof item === 'string' ? item : asObject(item)?.reasoningEffort;
+      if (typeof value === 'string' && value.length > 0) efforts.push(value);
+    }
+  }
+  return [...new Set(efforts)];
+};
+
 /**
  * Create an executor backed by an OpenCode HTTP server. The server URL comes
  * from `AG_AI_OPENCODE_URL`; the basic-auth password comes from
@@ -397,12 +417,15 @@ const createOpenCodeExecutor = async ({
         const modelId = typeof model?.id === 'string' && model.id.length > 0 ? model.id : key;
         const inputModalities = ['text'];
         if (model?.capabilities?.input?.image === true) inputModalities.push('image');
+        const declaredEfforts = reasoningEffortsFromModel(model);
+        const supportedReasoningEfforts =
+          declaredEfforts.length > 0 ? declaredEfforts : [DEFAULT_REASONING_EFFORT];
         models.push({
           id: `${providerId}/${modelId}`,
           inputModalities,
           outputModalities: imageAdapter !== undefined ? ['text', 'image'] : ['text'],
-          supportedReasoningEfforts: [DEFAULT_REASONING_EFFORT],
-          defaultReasoningEffort: DEFAULT_REASONING_EFFORT,
+          supportedReasoningEfforts,
+          defaultReasoningEffort: supportedReasoningEfforts[0],
           webSearch: false,
           prefer: ['quality'],
           isDefault: defaults[providerId] === modelId,

package/runtime/protocol.mjs

@@ -117,6 +117,16 @@ const normalizeModelCatalog = (models) => {
 const unsupported = (what) =>
   new GenerationProtocolError(422, 'unsupported', `${what} is not supported by the selected model`);
 
+/**
+ * Resolve an unqualified model id (for example `gpt-5.6-luna`) to the single
+ * catalog entry that exposes it under a provider prefix. An ambiguous or absent
+ * match stays unresolved so the caller gets an explicit unsupported error.
+ */
+const uniqueSuffixModel = (catalog, requestedModel) => {
+  const matches = catalog.filter((candidate) => candidate.id.endsWith(`/${requestedModel}`));
+  return matches.length === 1 ? matches[0] : undefined;
+};
+
 const normalizePart = (part, capabilities, total) => {
   const object = asObject(part);
   const type = object?.type;
@@ -262,7 +272,9 @@ const normalizeGenerationRequest = (value, models) => {
       catalog.find((candidate) => candidate.prefer.includes('quality'))?.id ??
       catalog[0].id;
   }
-  const model = catalog.find((candidate) => candidate.id === requestedModel);
+  const model =
+    catalog.find((candidate) => candidate.id === requestedModel) ??
+    (requestedModel.includes('/') ? undefined : uniqueSuffixModel(catalog, requestedModel));
   if (!model) throw unsupported(`model ${requestedModel}`);
   const capabilities = normalizeCapabilities(model);
   const instructions =
@@ -306,7 +318,7 @@ const normalizeGenerationRequest = (value, models) => {
     instructions,
     input,
     output,
-    model: requestedModel,
+    model: model.id,
     ...(prefer === undefined ? {} : { prefer }),
     reasoningEffort,
     ...(outputSchema === undefined ? {} : { outputSchema }),