@juspay/neurolink 12.14.21 → 12.15.1

package/dist/rag/ragIntegration.js

@@ -3,8 +3,11 @@ import { extname, resolve } from "path";
 import { z } from "zod";
 import { SpanSerializer, SpanType, SpanStatus, getMetricsAggregator, } from "../observability/index.js";
 import { logger } from "../utils/logger.js";
+import { redactUrlForError } from "../utils/logSanitize.js";
 import { createChunker } from "./ChunkerFactory.js";
 import { createVectorQueryTool, InMemoryVectorStore, } from "./retrieval/vectorQueryTool.js";
+import { ImageLoader } from "./document/imageLoader.js";
+import { IMAGE_EXTENSIONS } from "../processors/config/index.js";
 import { withSpan } from "../telemetry/withSpan.js";
 import { tracers } from "../telemetry/tracers.js";
 /**
@@ -43,6 +46,8 @@ function detectStrategy(filePath) {
     const ext = extname(filePath).toLowerCase();
     return EXTENSION_TO_STRATEGY[ext] || "recursive";
 }
+/** Embedding dimension used by {@link generateSimpleEmbedding}. */
+const EMBEDDING_DIMENSION = 128;
 /**
  * Simple hash function for strings (FNV-1a variant).
  * Maps a word to a bucket index deterministically.
@@ -177,14 +182,62 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
     if (!files || files.length === 0) {
         throw new Error("RAG config requires at least one file path in 'files'");
     }
-    // 1. Load files
+    // 1. Load files — separate text and image files
     const fileContents = [];
+    const imageFiles = [];
     for (const filePath of files) {
+        // HTTP(S) sources are handled by ImageLoader directly and must not go
+        // through resolve()/existsSync(), which would mangle the URL into a
+        // bogus local path and always report it as missing.
+        if (/^https?:\/\//i.test(filePath)) {
+            // `new URL()` throws on a string the guarding regex still accepts —
+            // "https://" is the minimal case. Uncaught, it leaves this loop, is
+            // rethrown by prepareRAGTool, and disables RAG for the WHOLE call, so
+            // one bad entry discards every file already loaded. Skip the source
+            // instead, matching how an unreadable local file is handled below.
+            let ext;
+            try {
+                ext = extname(new URL(filePath).pathname).toLowerCase();
+            }
+            catch {
+                logger.warn(`[RAG] Malformed URL source, skipping: ${redactUrlForError(filePath)}`);
+                continue;
+            }
+            if (ext === ".svg") {
+                // IMAGE_EXTENSIONS includes .svg, but every consumer downstream of
+                // here treats an image as raster bytes: ImageLoader returns the file
+                // verbatim, captioning base64s it into a vision call, and the Bedrock
+                // embed path ships it as-is — Nova rejects the format outright and
+                // Titan accepts markup it cannot embed meaningfully. Skipping is
+                // honest; supporting it needs a rasterize-or-sanitize step in
+                // ImageLoader, not a classification change here.
+                logger.warn(`[RAG] SVG is not supported as a RAG image source, skipping: ${redactUrlForError(filePath)}`);
+                continue;
+            }
+            if (IMAGE_EXTENSIONS.includes(ext)) {
+                imageFiles.push({ path: filePath, ext });
+            }
+            else {
+                logger.warn(`[RAG] Unsupported URL source, skipping: ${redactUrlForError(filePath)}`);
+            }
+            continue;
+        }
         const resolvedPath = resolve(filePath);
         if (!existsSync(resolvedPath)) {
             logger.warn(`[RAG] File not found, skipping: ${resolvedPath}`);
             continue;
         }
+        // Detect image files by extension
+        const ext = extname(resolvedPath).toLowerCase();
+        if (ext === ".svg") {
+            // See the URL branch above for why SVG is excluded from the raster path.
+            logger.warn(`[RAG] SVG is not supported as a RAG image source, skipping: ${resolvedPath}`);
+            continue;
+        }
+        if (IMAGE_EXTENSIONS.includes(ext)) {
+            imageFiles.push({ path: resolvedPath, ext });
+            continue;
+        }
         try {
             const content = readFileSync(resolvedPath, "utf-8");
             const strategy = userStrategy || detectStrategy(resolvedPath);
@@ -195,11 +248,12 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
         }
     }
     // Auto-increase topK for multi-file scenarios to ensure coverage
-    // (computed after loading so it reflects only files that actually exist)
-    const topK = fileContents.length > 1
-        ? Math.max(userTopK, fileContents.length * 3)
-        : userTopK;
-    if (fileContents.length === 0) {
+    // (computed after loading so it reflects only files that actually exist).
+    // Image files count toward the file total so image-only RAG gets a topK
+    // that covers every source.
+    const totalSources = fileContents.length + imageFiles.length;
+    const topK = totalSources > 1 ? Math.max(userTopK, totalSources * 3) : userTopK;
+    if (fileContents.length === 0 && imageFiles.length === 0) {
         throw new Error("RAG: No files could be loaded. Check that file paths exist and are readable.");
     }
     logger.info(`[RAG] Loaded ${fileContents.length} files for indexing`);
@@ -230,9 +284,44 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
             });
         }
     }
-    logger.info(`[RAG] Created ${allChunks.length} chunks from ${fileContents.length} files`);
+    logger.info(`[RAG] Created ${allChunks.length} chunks from ${fileContents.length} text files`);
+    // 2b. Load image files (multi-modal RAG).
+    //
+    // Deliberately NOT embedded here. Images are vectorised below through the
+    // same `embedFn` as the text chunks, because index-wide a single embedding
+    // space is the whole precondition for similarity to mean anything. Embedding
+    // them here — as this path originally did, with a hash of the caption —
+    // produced a 128-dimension image vector sitting in an index whose text
+    // vectors came from a provider at 768 or 1536, which is not a worse ranking
+    // but an incomparable one.
+    const imageChunks = [];
+    if (imageFiles.length > 0) {
+        const imageLoader = new ImageLoader();
+        for (const { path: imgPath } of imageFiles) {
+            try {
+                const imageDoc = await imageLoader.load(imgPath);
+                // Use filename-based text representation for simple embedding
+                const imageText = imageDoc.text;
+                imageChunks.push({
+                    id: `img-chunk-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
+                    text: imageText,
+                    metadata: {
+                        text: imageText,
+                        source: imgPath,
+                        hasImage: true,
+                        mimeType: imageDoc.mimeType,
+                        imageWidth: imageDoc.metadata.width,
+                        imageHeight: imageDoc.metadata.height,
+                    },
+                });
+            }
+            catch (error) {
+                logger.warn(`[RAG] Failed to load image: ${imgPath}: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        logger.info(`[RAG] Loaded ${imageChunks.length} image files for multi-modal RAG`);
+    }
     // 3. Generate embeddings and store in vector store
-    const EMBEDDING_DIMENSION = 128;
     const vectorStore = new InMemoryVectorStore();
     const indexName = "rag-index";
     // When the caller configured an embedding provider/model, embed BOTH the
@@ -262,9 +351,19 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
             logger.warn("[RAG] Failed to create embedding provider; falling back to hash embeddings", { error: error instanceof Error ? error.message : String(error) });
         }
     }
+    // Text and image captions embed together, in one call and under one
+    // fallback. Splitting them into two try/catch blocks would let the text side
+    // succeed on the provider while the image side fell back to the hash, which
+    // is the mixed-space index this is trying to prevent.
     let chunkVectors;
+    let imageVectors;
     try {
-        chunkVectors = await Promise.all(allChunks.map((chunk) => embedFn(chunk.text)));
+        const allVectors = await Promise.all([
+            ...allChunks.map((chunk) => embedFn(chunk.text)),
+            ...imageChunks.map((chunk) => embedFn(chunk.text)),
+        ]);
+        chunkVectors = allVectors.slice(0, allChunks.length);
+        imageVectors = allVectors.slice(allChunks.length);
     }
     catch (error) {
         // One failed chunk must not leave a mixed-space index — flip the whole
@@ -272,17 +371,36 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
         logger.warn("[RAG] Provider embedding failed mid-index; falling back to hash embeddings for index and queries", { error: error instanceof Error ? error.message : String(error) });
         embedFn = (text) => Promise.resolve(generateSimpleEmbedding(text, EMBEDDING_DIMENSION));
         chunkVectors = allChunks.map((chunk) => generateSimpleEmbedding(chunk.text, EMBEDDING_DIMENSION));
+        imageVectors = imageChunks.map((chunk) => generateSimpleEmbedding(chunk.text, EMBEDDING_DIMENSION));
+    }
+    const items = [
+        ...allChunks.map((chunk, i) => ({
+            id: `rag-chunk-${i}`,
+            vector: chunkVectors[i],
+            metadata: {
+                text: chunk.text,
+                ...chunk.metadata,
+            },
+        })),
+        ...imageChunks.map((chunk, i) => ({
+            id: chunk.id,
+            vector: imageVectors[i],
+            metadata: chunk.metadata,
+        })),
+    ];
+    // The guard near the top of this function fires when no source was found
+    // at all. This one covers the other way to end up with an empty index:
+    // sources WERE found and every one of them failed to load — an image over
+    // the size ceiling, a corrupt file, an unreachable URL. Each of those is
+    // caught and warned individually, so without this the call returns
+    // `chunksIndexed: 0` alongside a positive `filesLoaded` and reports
+    // success, and the caller gets a silently empty knowledge base.
+    if (items.length === 0) {
+        throw new Error("RAG: every source failed to load, so nothing was indexed. " +
+            "Check the warnings above for the per-source reason.");
     }
-    const items = allChunks.map((chunk, i) => ({
-        id: `rag-chunk-${i}`,
-        vector: chunkVectors[i],
-        metadata: {
-            text: chunk.text,
-            ...chunk.metadata,
-        },
-    }));
     await vectorStore.upsert(indexName, items);
-    logger.info(`[RAG] Indexed ${items.length} chunks in vector store`);
+    logger.info(`[RAG] Indexed ${items.length} chunks (${allChunks.length} text + ${imageChunks.length} images) in vector store`);
     // 4. Create the search tool
     // Determine embedding provider/model for the query tool
     const provider = embeddingProvider || fallbackProvider || "vertex";
@@ -315,15 +433,16 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
                 // Query through the same embedding space the index was built in —
                 // provider embeddings when configured (and healthy), else the hash.
                 const queryEmbedding = await embedFn(query).catch(() => generateSimpleEmbedding(query, EMBEDDING_DIMENSION));
-                // Fetch more candidates than needed so diversity can select across files
-                const fetchK = fileContents.length > 1 ? topK * 3 : topK;
+                // Fetch more candidates than needed so diversity can select across files and images
+                const loadedSources = fileContents.length + imageChunks.length;
+                const fetchK = loadedSources > 1 ? topK * 3 : topK;
                 const rawResults = await vectorStore.query({
                     indexName,
                     queryVector: queryEmbedding,
                     topK: fetchK,
                 });
-                // Apply source-file diversity for multi-file RAG
-                const results = fileContents.length > 1
+                // Apply source-file diversity for multi-file/multi-image RAG
+                const results = loadedSources > 1
                     ? diversifyResults(rawResults, topK)
                     : rawResults.slice(0, topK);
                 if (results.length === 0) {
@@ -335,7 +454,8 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
                     };
                 }
                 const relevantContext = results
-                    .map((r, i) => `[${i + 1}] ${r.metadata?.text || r.text || ""}`)
+                    .map((r, i) => `[${i + 1}] ${r.metadata?.text || r.text || ""}` +
+                    (r.metadata?.hasImage ? " [image]" : ""))
                     .join("\n\n");
                 span.setAttribute("rag.results_count", results.length);
                 return {
@@ -345,6 +465,7 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
                         score: r.score,
                         source: r.metadata?.source,
                         text: (r.metadata?.text || r.text || "").slice(0, 200),
+                        hasImage: r.metadata?.hasImage ?? false,
                     })),
                     totalResults: results.length,
                 };
@@ -354,8 +475,8 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
     return {
         tool: aiTool,
         toolName,
-        chunksIndexed: allChunks.length,
-        filesLoaded: fileContents.length,
+        chunksIndexed: allChunks.length + imageChunks.length,
+        filesLoaded: fileContents.length + imageFiles.length,
     };
 }
 /** @internal Exported for testing only */

package/dist/server/routes/claudeProxyRoutes.js

@@ -3693,6 +3693,25 @@ async function executeClaudeCodexFallback(args) {
     registerProxyResponseObserver(ctx.metadata, {
         onTerminal: () => childAccounting,
     });
+    const recordCodexUsage = (result) => {
+        ctx.metadata.codexFallbackModel = model;
+        try {
+            tracer?.setModelSubstitution(body.model, model, "openai");
+            tracer?.setServedAccount(codexHeaders["x-neurolink-account"] ?? "unknown", "codex-oauth");
+            if (result?.usage) {
+                tracer?.setUsage({
+                    inputTokens: result.usage.input,
+                    outputTokens: result.usage.output,
+                    cacheReadTokens: result.usage.cacheReadTokens ?? 0,
+                    cacheCreationTokens: result.usage.cacheCreationTokens ?? 0,
+                    reasoningTokens: result.usage.reasoning,
+                });
+            }
+        }
+        catch {
+            // Instrumentation must never turn completed output into a failed stream.
+        }
+    };
     const codexHeaders = { ...(codexCtx.responseHeaders ?? {}) };
     if (body.stream) {
         const bridge = await createCodexFallbackStream(codexResponse, body.model);
@@ -3712,12 +3731,14 @@ async function executeClaudeCodexFallback(args) {
             if (status >= 400) {
                 ctx.metadata.terminalErrorType = errorType;
             }
+            recordCodexUsage(result);
             tracer?.end(status, Date.now() - requestStartTime);
             logFinalRequest(status, account, accountType, errorType, message, {
                 inputTokens: result?.usage?.input,
                 outputTokens: result?.usage?.output,
                 cacheCreationTokens: result?.usage?.cacheCreationTokens,
                 cacheReadTokens: result?.usage?.cacheReadTokens,
+                reasoningTokens: result?.usage?.reasoning,
             });
             recordFallbackAttempt({
                 provider: "codex",
@@ -3862,6 +3883,7 @@ async function executeClaudeCodexFallback(args) {
         ...(parsed.usage ? { usage: parsed.usage } : {}),
         toolCalls: parsed.toolCalls,
     };
+    recordCodexUsage(parsed);
     tracer?.end(200, Date.now() - requestStartTime);
     const clientResponse = serializeClaudeResponse(internal, body.model);
     observeClaudeJsonOutput(ctx.metadata, clientResponse);
@@ -3870,6 +3892,7 @@ async function executeClaudeCodexFallback(args) {
         outputTokens: parsed.usage?.output,
         cacheCreationTokens: parsed.usage?.cacheCreationTokens,
         cacheReadTokens: parsed.usage?.cacheReadTokens,
+        reasoningTokens: parsed.usage?.reasoning,
     });
     const clientResponseText = JSON.stringify(clientResponse);
     logProxyBody({
@@ -6009,7 +6032,15 @@ function createClaudeRequestRuntimeContext(args) {
             requestId: ctx.requestId,
             method: ctx.method,
             path: ctx.path,
-            model: body.model,
+            model: typeof ctx.metadata.codexFallbackModel === "string"
+                ? ctx.metadata.codexFallbackModel
+                : body.model,
+            ...(typeof ctx.metadata.codexFallbackModel === "string"
+                ? {
+                    requestedModel: body.model,
+                    inputIncludesCachedTokens: false,
+                }
+                : {}),
             stream: !!body.stream,
             toolCount: Array.isArray(body.tools) ? body.tools.length : 0,
             account: finalAccountLabel ?? "",
@@ -6042,6 +6073,9 @@ function createClaudeRequestRuntimeContext(args) {
             ...(extra?.cacheReadTokens !== undefined
                 ? { cacheReadTokens: extra.cacheReadTokens }
                 : {}),
+            ...(extra?.reasoningTokens !== undefined
+                ? { reasoningTokens: extra.reasoningTokens }
+                : {}),
             ...(traceCtx
                 ? { traceId: traceCtx.traceId, spanId: traceCtx.spanId }
                 : {}),

package/dist/server/routes/codexProxyRoutes.d.ts

@@ -35,19 +35,15 @@ declare function refreshCodexTokenOnce(key: string, refreshToken: string): Promi
  */
 declare function loadCodexProxyAccounts(): Promise<CodexRuntimeAccount[]>;
 /**
- * Order accounts fill-first by quota: eligible (not cooling) first, then least
- * session utilization, treating unknown quota as "probe first" so a fresh
- * account gets observed rather than starved.
+ * Order non-cooling accounts ahead of cooling ones, then defer rejected quota.
+ * Within each group, probe unknown quota first and otherwise fill the least-used
+ * session so fresh accounts get observed rather than starved.
  */
 declare function orderCodexAccounts(accounts: CodexRuntimeAccount[], now: number): CodexRuntimeAccount[];
 /** Build the upstream request headers, replacing client auth with the account's. */
 declare function buildCodexUpstreamHeaders(clientHeaders: Record<string, string>, account: CodexRuntimeAccount): Record<string, string>;
-/**
- * Classify a 429 into a cooldown plan. Codex reports primary (session) and
- * secondary (weekly) windows; a rejected window cools until its reset, a plain
- * burst limit cools briefly and clamps to 15 min.
- */
-declare function planCodexCooldown(quota: AccountQuota | null, retryAfterMs: number, now: number): {
+/** Honor known quota resets while bounding unscoped or transient account cooldowns. */
+declare function planCodexCooldown(quota: AccountQuota | null, retryAfterMs: number, now: number, errorText?: string): {
     coolingUntil: number;
     reason: RateLimitCoolingReason;
 };

package/dist/server/routes/codexProxyRoutes.js

@@ -188,9 +188,9 @@ async function loadCodexProxyAccounts() {
     return accounts;
 }
 /**
- * Order accounts fill-first by quota: eligible (not cooling) first, then least
- * session utilization, treating unknown quota as "probe first" so a fresh
- * account gets observed rather than starved.
+ * Order non-cooling accounts ahead of cooling ones, then defer rejected quota.
+ * Within each group, probe unknown quota first and otherwise fill the least-used
+ * session so fresh accounts get observed rather than starved.
  */
 function orderCodexAccounts(accounts, now) {
     return [...accounts].sort((a, b) => {
@@ -199,8 +199,13 @@ function orderCodexAccounts(accounts, now) {
         if (aCooling !== bCooling) {
             return aCooling ? 1 : -1;
         }
-        const aUsed = a.quota ? a.quota.sessionUsed : -1;
-        const bUsed = b.quota ? b.quota.sessionUsed : -1;
+        const aRejected = a.quota?.unifiedStatus === "rejected";
+        const bRejected = b.quota?.unifiedStatus === "rejected";
+        if (aRejected !== bRejected) {
+            return aRejected ? 1 : -1;
+        }
+        const aUsed = a.quota && a.quota.sessionStatus !== "unknown" ? a.quota.sessionUsed : -1;
+        const bUsed = b.quota && b.quota.sessionStatus !== "unknown" ? b.quota.sessionUsed : -1;
         return aUsed - bUsed;
     });
 }
@@ -231,12 +236,51 @@ function buildCodexUpstreamHeaders(clientHeaders, account) {
     }
     return headers;
 }
-/**
- * Classify a 429 into a cooldown plan. Codex reports primary (session) and
- * secondary (weekly) windows; a rejected window cools until its reset, a plain
- * burst limit cools briefly and clamps to 15 min.
- */
-function planCodexCooldown(quota, retryAfterMs, now) {
+/** Extract explicit plan-exhaustion evidence without guessing account-wide scope. */
+function parseCodexQuotaError(errorText, now) {
+    try {
+        const payload = JSON.parse(errorText);
+        const error = payload?.error;
+        if (!error ||
+            typeof error !== "object" ||
+            (error.type !== "usage_limit_reached" &&
+                error.code !== "usage_limit_reached")) {
+            return null;
+        }
+        const absolute = error.resets_at ?? error.reset_at;
+        const relative = error.resets_in_seconds ?? error.reset_after_seconds;
+        const resetAt = typeof absolute === "number" && Number.isFinite(absolute) && absolute > 0
+            ? absolute > 4_102_444_800
+                ? absolute
+                : absolute * 1000
+            : typeof relative === "number" &&
+                Number.isFinite(relative) &&
+                relative > 0
+                ? now + relative * 1000
+                : 0;
+        // Only explicit account-wide window evidence permits a long cooldown.
+        // A model/surface-scoped or unscoped rejection must not park the entire pool.
+        const window = error.limit_type;
+        const scoped = error.model || (error.scope !== undefined && error.scope !== "account");
+        const scope = scoped
+            ? "unknown"
+            : window === "primary" || window === "session"
+                ? "session"
+                : window === "secondary" || window === "weekly"
+                    ? "weekly"
+                    : "unknown";
+        return {
+            errorCode: "usage_limit_reached",
+            resetAt: Number.isFinite(resetAt) ? resetAt : 0,
+            scope,
+        };
+    }
+    catch {
+        return null;
+    }
+}
+/** Honor known quota resets while bounding unscoped or transient account cooldowns. */
+function planCodexCooldown(quota, retryAfterMs, now, errorText = "") {
     // A reported reset can already be in the past — a stale header, or a clock
     // skew. Taken literally the account is eligible again immediately and the
     // pool re-sends to something the provider just rejected.
@@ -255,8 +299,25 @@ function planCodexCooldown(quota, retryAfterMs, now) {
             };
         }
     }
+    const bodyQuota = parseCodexQuotaError(errorText, now);
+    if (bodyQuota) {
+        if (bodyQuota.scope !== "unknown" && bodyQuota.resetAt > 0) {
+            return {
+                coolingUntil: Math.max(bodyQuota.resetAt, floor),
+                reason: bodyQuota.scope,
+            };
+        }
+        return {
+            coolingUntil: now +
+                Math.min(MAX_TRANSIENT_COOLDOWN_MS, Math.max(bodyQuota.resetAt - now, retryAfterMs, DEFAULT_TRANSIENT_COOLDOWN_MS)),
+            reason: "unified",
+        };
+    }
     const delay = Math.min(MAX_TRANSIENT_COOLDOWN_MS, Math.max(retryAfterMs, DEFAULT_TRANSIENT_COOLDOWN_MS));
-    return { coolingUntil: now + delay, reason: "transient" };
+    return {
+        coolingUntil: now + delay,
+        reason: quota?.unifiedStatus === "rejected" ? "unified" : "transient",
+    };
 }
 /** Set the x-neurolink-* attribution headers on the context. */
 function publishCodexHeaders(ctx, account, attempt, quota) {
@@ -268,9 +329,14 @@ function publishCodexHeaders(ctx, account, attempt, quota) {
     ctx.responseHeaders["x-neurolink-served-by"] = "codex";
     ctx.responseHeaders["x-neurolink-attempt"] = String(attempt);
     ctx.responseHeaders["x-neurolink-quota-source"] = quota ? "live" : "none";
-    if (quota) {
+    if (quota && quota.sessionStatus !== "unknown") {
         ctx.responseHeaders["x-neurolink-quota-session-left-pct"] = String(Math.round((1 - quota.sessionUsed) * 100));
-        ctx.responseHeaders["x-neurolink-weekly-left-pct"] = String(Math.round((1 - quota.weeklyUsed) * 100));
+    }
+    if (quota && quota.weeklyStatus !== "unknown") {
+        const remaining = String(Math.round((1 - quota.weeklyUsed) * 100));
+        ctx.responseHeaders["x-neurolink-quota-weekly-left-pct"] = remaining;
+        // Preserve the original Codex header for existing clients.
+        ctx.responseHeaders["x-neurolink-weekly-left-pct"] = remaining;
     }
 }
 /** Core pooled handler for POST /backend-api/codex/responses. */
@@ -389,6 +455,7 @@ async function executeCodexResponsesRequest(ctx) {
         // This is the cost provider. accountKey and the response header identify
         // the actual Codex pool that supplied the credential.
         provider: "openai",
+        inputIncludesCachedTokens: true,
         ...tracer?.getTraceContext(),
         ...(reasoningEffort ? { reasoningEffort } : {}),
         firstUsefulOutputStatus: extra.firstUsefulOutputMs !== undefined ? "observed" : "not_observed",
@@ -679,11 +746,20 @@ async function executeCodexResponsesRequest(ctx) {
                                         outputTokens: usage.outputTokens,
                                         cacheReadTokens: usage.cacheReadTokens,
                                         cacheCreationTokens: usage.cacheCreationTokens,
+                                        reasoningTokens: usage.reasoningTokensObserved
+                                            ? usage.reasoningTokens
+                                            : undefined,
                                     }
                                     : {};
                                 if (usage) {
                                     try {
-                                        tracer?.setUsage(usage);
+                                        tracer?.setUsage({
+                                            ...usage,
+                                            reasoningTokens: usage.reasoningTokensObserved
+                                                ? usage.reasoningTokens
+                                                : undefined,
+                                            inputIncludesCachedTokens: true,
+                                        });
                                     }
                                     catch {
                                         // Pricing/metrics must never change the stream outcome.
@@ -825,7 +901,9 @@ async function executeCodexResponsesRequest(ctx) {
                         saveAccountQuota(account.key, quota).catch(() => undefined);
                     }
                     const retryAfterMs = parseRetryAfterMs(upstream.headers.get("retry-after"));
-                    const plan = planCodexCooldown(quota, retryAfterMs, Date.now());
+                    const now = Date.now();
+                    const bodyQuota = parseCodexQuotaError(errText, now);
+                    const plan = planCodexCooldown(quota, retryAfterMs, now, errText);
                     await saveAccountCooldown(account.key, plan.coolingUntil, plan.reason).catch(() => undefined);
                     const rateLimitKind = plan.reason === "transient" ? "transient" : "quota";
                     const errorMessage = summarizeCodexUpstreamError(errText, "Codex account rate-limited");
@@ -833,11 +911,21 @@ async function executeCodexResponsesRequest(ctx) {
                     writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
                         errorType: "rate_limit_error",
                         errorMessage,
-                        retryable: true,
+                        retryable: rateLimitKind === "transient",
                         rateLimitKind,
                         cooldownReason: plan.reason,
+                        ...(bodyQuota
+                            ? {
+                                errorCode: bodyQuota.errorCode,
+                                quotaResetAt: bodyQuota.resetAt || undefined,
+                                quotaScope: bodyQuota.scope,
+                            }
+                            : {}),
                     });
-                    lastFailure = { errorType: "rate_limit_error" };
+                    lastFailure = {
+                        errorType: "rate_limit_error",
+                        errorCode: bodyQuota?.errorCode,
+                    };
                     lastErrorStatus = 429;
                     lastErrorMessage = "Codex account rate-limited";
                     break; // rotate

package/dist/types/codex.d.ts

@@ -64,6 +64,8 @@ export type CodexRateLimitWindow = {
 };
 /** Codex rate-limit block: a primary (short) and secondary (long) window. */
 export type CodexRateLimits = {
+    allowed?: boolean;
+    limit_reached?: boolean;
     primary?: CodexRateLimitWindow | null;
     secondary?: CodexRateLimitWindow | null;
 };
@@ -73,6 +75,8 @@ export type CodexUsageResponse = {
     rate_limits?: CodexRateLimits | null;
     /** Current ChatGPT WHAM account-usage payload. */
     rate_limit?: {
+        allowed?: boolean;
+        limit_reached?: boolean;
         primary_window?: CodexRateLimitWindow | null;
         secondary_window?: CodexRateLimitWindow | null;
     } | null;
@@ -86,6 +90,12 @@ export type CodexUsageFetchResult = {
     ok: false;
     reason: "not_oauth" | "auth" | "rate_limited" | "http" | "network" | "parse";
 };
+/** Explicit plan-exhaustion evidence from a Codex error response. */
+export type CodexQuotaError = {
+    errorCode: string;
+    resetAt: number;
+    scope: "session" | "weekly" | "unknown";
+};
 /** A Codex account with its runtime cooldown/quota state hydrated from disk. */
 export type CodexRuntimeAccount = {
     key: string;

package/dist/types/providers.d.ts

@@ -28,6 +28,18 @@ export type SupportedModelName = BedrockModels | DeepSeekModels | OpenAIModels |
  * Extract provider names from enum
  */
 export type ProviderName = (typeof AIProviderName)[keyof typeof AIProviderName];
+/**
+ * Multi-modal embedding input — accepts text, image, or both.
+ * Used by providers that support multi-modal embeddings (e.g. Bedrock Titan Image, Nova Multimodal).
+ */
+export type EmbedInput = {
+    /** Text content to embed */
+    text?: string;
+    /** Image data as Buffer or base64 string */
+    image?: Buffer | string;
+    /** MIME type of the image (e.g. "image/png", "image/jpeg") */
+    mimeType?: string;
+};
 /**
  * Provider status information
  */
@@ -741,7 +753,15 @@ export type AIProvider = {
     stream(optionsOrPrompt: StreamOptions | string, analysisSchema?: ValidationSchema): Promise<StreamResult>;
     generate(optionsOrPrompt: TextGenerationOptions | string, analysisSchema?: ValidationSchema): Promise<EnhancedGenerateResult | null>;
     gen(optionsOrPrompt: TextGenerationOptions | string, analysisSchema?: ValidationSchema): Promise<EnhancedGenerateResult | null>;
-    embed(text: string, modelName?: string): Promise<number[]>;
+    /**
+     * Generate an embedding vector for text or multi-modal input.
+     * Accepts either a plain string (text-only) or an EmbedInput object
+     * for multi-modal embeddings (text + image).
+     */
+    embed(input: string | EmbedInput, modelName?: string): Promise<number[]>;
+    /**
+     * Generate embedding vectors for multiple text inputs in batch.
+     */
     embedMany(texts: string[], modelName?: string): Promise<number[][]>;
     setupToolExecutor(sdk: {
         customTools: Map<string, unknown>;