@juspay/neurolink 12.14.21 → 12.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/CHANGELOG.md +2 -6
  2. package/dist/browser/neurolink.min.js +397 -397
  3. package/dist/cli/commands/auth.js +6 -2
  4. package/dist/cli/commands/proxy.d.ts +4 -0
  5. package/dist/cli/commands/proxy.js +119 -39
  6. package/dist/cli/index.js +8 -18
  7. package/dist/cli/lifecycle.d.ts +8 -0
  8. package/dist/cli/lifecycle.js +90 -0
  9. package/dist/core/baseProvider.d.ts +6 -3
  10. package/dist/core/baseProvider.js +7 -3
  11. package/dist/providers/amazonBedrock/client.d.ts +7 -5
  12. package/dist/providers/amazonBedrock/client.js +125 -16
  13. package/dist/providers/cohere.d.ts +2 -2
  14. package/dist/providers/cohere.js +5 -1
  15. package/dist/providers/googleAiStudio/client.d.ts +3 -3
  16. package/dist/providers/googleAiStudio/client.js +6 -2
  17. package/dist/providers/googleVertex/client.d.ts +2 -2
  18. package/dist/providers/googleVertex/client.js +5 -1
  19. package/dist/providers/jina.d.ts +2 -2
  20. package/dist/providers/jina.js +5 -1
  21. package/dist/providers/litellm/client.d.ts +2 -2
  22. package/dist/providers/litellm/client.js +5 -1
  23. package/dist/providers/ollama/client.d.ts +2 -2
  24. package/dist/providers/ollama/client.js +5 -1
  25. package/dist/providers/openAI/client.d.ts +3 -3
  26. package/dist/providers/openAI/client.js +6 -2
  27. package/dist/providers/voyage.d.ts +2 -2
  28. package/dist/providers/voyage.js +5 -1
  29. package/dist/proxy/codexAccountUsage.js +18 -6
  30. package/dist/proxy/codexFallback.js +17 -10
  31. package/dist/proxy/codexUsage.js +3 -0
  32. package/dist/proxy/proxyAnalysis.js +14 -11
  33. package/dist/proxy/proxyTokenUsage.d.ts +7 -0
  34. package/dist/proxy/proxyTokenUsage.js +21 -0
  35. package/dist/proxy/proxyTracer.d.ts +2 -0
  36. package/dist/proxy/proxyTracer.js +21 -40
  37. package/dist/proxy/quotaHeaders.js +7 -3
  38. package/dist/rag/document/MDocument.d.ts +19 -1
  39. package/dist/rag/document/MDocument.js +30 -3
  40. package/dist/rag/document/imageLoader.d.ts +55 -0
  41. package/dist/rag/document/imageLoader.js +183 -0
  42. package/dist/rag/index.d.ts +1 -0
  43. package/dist/rag/index.js +2 -0
  44. package/dist/rag/pipeline/RAGPipeline.d.ts +54 -1
  45. package/dist/rag/pipeline/RAGPipeline.js +358 -1
  46. package/dist/rag/ragIntegration.js +146 -25
  47. package/dist/server/routes/claudeProxyRoutes.js +35 -1
  48. package/dist/server/routes/codexProxyRoutes.d.ts +5 -9
  49. package/dist/server/routes/codexProxyRoutes.js +106 -18
  50. package/dist/types/codex.d.ts +10 -0
  51. package/dist/types/providers.d.ts +21 -1
  52. package/dist/types/proxy.d.ts +24 -6
  53. package/dist/types/rag.d.ts +144 -0
  54. package/dist/types/rag.js +12 -1
  55. package/dist/utils/pricing.js +4 -2
  56. package/docs-site/static/search-index.json +1 -1
  57. package/package.json +2 -1
@@ -10,6 +10,7 @@ import { createBedrockLoopAdapter } from "./loopAdapter.js";
10
10
  import { AuthenticationError, ProviderError, RateLimitError, } from "../../types/index.js";
11
11
  import { classifyProviderError } from "../../utils/errorClassifier.js";
12
12
  import { isAbortError, withTimeout } from "../../utils/errorHandling.js";
13
+ import { ImageProcessor } from "../../utils/imageProcessor.js";
13
14
  import { logger } from "../../utils/logger.js";
14
15
  import { resolveSamplingParams } from "../../models/modelRegistry.js";
15
16
  import { buildMultimodalMessagesArray } from "../../utils/messageBuilder.js";
@@ -18,6 +19,36 @@ import { convertZodToJsonSchema } from "../../utils/schemaConversion.js";
18
19
  import { SpanKind, SpanStatusCode } from "@opentelemetry/api";
19
20
  import { bedrockTracer } from "./constants.js";
20
21
  import { loadBedrockControl } from "./utils.js";
22
+ /**
23
+ * The only image formats Nova Multimodal Embeddings accepts.
24
+ *
25
+ * An allowlist rather than a derivation from the mimetype, because every
26
+ * shortcut here produces a value Nova rejects for some ordinary input:
27
+ * splitting on "/" yields `svg+xml` for an SVG and `jpeg; charset=binary`
28
+ * when the mimetype carries parameters, and normalising `jpeg` to `jpg` —
29
+ * the natural-looking direction — turns the single most common input into
30
+ * the one spelling that is not on this list.
31
+ */
32
+ const NOVA_IMAGE_FORMATS = {
33
+ "image/png": "png",
34
+ "image/jpeg": "jpeg",
35
+ "image/jpg": "jpeg",
36
+ "image/gif": "gif",
37
+ "image/webp": "webp",
38
+ };
39
+ /**
40
+ * Nova's format name for `mimeType`, or undefined when Nova has no name for it.
41
+ *
42
+ * Deliberately not defaulting to png. The RAG image path accepts bmp, tiff and
43
+ * avif, none of which Nova names, so a default would hand AWS a BMP labelled as
44
+ * a PNG — bytes and declared format disagreeing, with no error and no way for
45
+ * the caller to tell it happened. An unnamed format is surfaced to the caller
46
+ * instead; see the throw at the call site.
47
+ */
48
+ function novaImageFormat(mimeType) {
49
+ const normalized = (mimeType ?? "").split(";").at(0)?.trim().toLowerCase();
50
+ return NOVA_IMAGE_FORMATS[normalized ?? ""];
51
+ }
21
52
  export class AmazonBedrockProvider extends BaseProvider {
22
53
  bedrockClient;
23
54
  conversationHistory = [];
@@ -1162,24 +1193,100 @@ export class AmazonBedrockProvider extends BaseProvider {
1162
1193
  return classifyProviderError(error, rules, this.providerName, this.modelName);
1163
1194
  }
1164
1195
  /**
1165
- * Generate embeddings for text using Amazon Bedrock embedding models
1166
- * Uses the native AWS SDK InvokeModel command for Titan embeddings
1167
- * @param text - The text to embed
1196
+ * Generate embeddings for text or multi-modal input using Amazon Bedrock.
1197
+ * Supports both text-only models (Titan Embed Text) and multi-modal models
1198
+ * (Titan Embed Image, Nova Multimodal Embeddings).
1199
+ *
1200
+ * @param input - Text string or EmbedInput with text/image/mimeType
1168
1201
  * @param modelName - The embedding model to use (default: amazon.titan-embed-text-v2:0)
1169
1202
  * @returns Promise resolving to the embedding vector
1170
1203
  */
1171
- async embed(text, modelName) {
1172
- const embeddingModelName = modelName || "amazon.titan-embed-text-v2:0";
1204
+ async embed(input, modelName) {
1205
+ // Normalize input to EmbedInput shape
1206
+ const embedInput = typeof input === "string" ? { text: input } : input;
1207
+ const embeddingModelName = modelName || this.getDefaultEmbeddingModel();
1208
+ const isNovaModel = embeddingModelName.includes("nova") &&
1209
+ embeddingModelName.includes("multimodal");
1210
+ const isMultiModalModel = embeddingModelName.includes("image") ||
1211
+ embeddingModelName.includes("multimodal") ||
1212
+ embeddingModelName === "amazon.titan-embed-image-v1" ||
1213
+ isNovaModel;
1173
1214
  logger.debug("Generating embedding", {
1174
1215
  provider: this.providerName,
1175
1216
  model: embeddingModelName,
1176
- textLength: text.length,
1217
+ hasText: typeof embedInput.text === "string",
1218
+ hasImage: embedInput.image !== undefined,
1219
+ isMultiModalModel,
1177
1220
  });
1178
1221
  try {
1179
- // Titan Embed models expect a specific input format
1180
- const requestBody = JSON.stringify({
1181
- inputText: text,
1182
- });
1222
+ // Build request body based on model type
1223
+ let requestBody;
1224
+ if (embedInput.image && !isMultiModalModel) {
1225
+ throw new ProviderError(`${this.providerName} model ${embeddingModelName} does not support image embeddings`, this.providerName);
1226
+ }
1227
+ if (isNovaModel) {
1228
+ // Nova Multimodal Embeddings uses SINGLE_EMBEDDING + singleEmbeddingParams
1229
+ // for BOTH text-only and image requests. Nova requires exactly one
1230
+ // modality per request — combined image+text is rejected explicitly
1231
+ // rather than silently dropping one of them.
1232
+ if (embedInput.image && embedInput.text) {
1233
+ throw new ProviderError(`${this.providerName} model ${embeddingModelName} does not support combined image+text embeddings; provide exactly one of image or text`, this.providerName);
1234
+ }
1235
+ const singleEmbeddingParams = {
1236
+ embeddingPurpose: "GENERIC_RETRIEVAL",
1237
+ };
1238
+ if (embedInput.image) {
1239
+ const imageBuffer = Buffer.isBuffer(embedInput.image)
1240
+ ? embedInput.image
1241
+ : Buffer.from(embedInput.image, "base64");
1242
+ const imageFormat = novaImageFormat(embedInput.mimeType);
1243
+ if (imageFormat === undefined) {
1244
+ throw new ProviderError(`${this.providerName} model ${embeddingModelName} does not support image format ` +
1245
+ `'${embedInput.mimeType ?? "unknown"}'; supported formats are ` +
1246
+ `${Object.keys(NOVA_IMAGE_FORMATS).join(", ")}`, this.providerName);
1247
+ }
1248
+ singleEmbeddingParams.image = {
1249
+ detailLevel: "STANDARD_IMAGE",
1250
+ format: imageFormat,
1251
+ source: {
1252
+ // Bounded on purpose. `Buffer.toString("base64")` allocates ~4/3
1253
+ // the buffer in one go, so an oversized image exhausts memory
1254
+ // before the request is ever built. The chat-image path already
1255
+ // routes through this guard (messageBuilder.ts); the embed paths
1256
+ // never adopted it, even though ImageLoader enforces the same
1257
+ // ceiling for images it loads itself.
1258
+ bytes: ImageProcessor.safeBase64Convert(imageBuffer, "bedrock nova embed image"),
1259
+ },
1260
+ };
1261
+ }
1262
+ if (embedInput.text) {
1263
+ singleEmbeddingParams.text = {
1264
+ value: embedInput.text,
1265
+ truncationMode: "NONE",
1266
+ };
1267
+ }
1268
+ requestBody = JSON.stringify({
1269
+ taskType: "SINGLE_EMBEDDING",
1270
+ singleEmbeddingParams,
1271
+ });
1272
+ }
1273
+ else if (embedInput.image) {
1274
+ // Titan Multimodal Embeddings expects inputImage as a base64 string
1275
+ const imageBuffer = Buffer.isBuffer(embedInput.image)
1276
+ ? embedInput.image
1277
+ : Buffer.from(embedInput.image, "base64");
1278
+ requestBody = JSON.stringify({
1279
+ inputText: embedInput.text ?? "",
1280
+ // Same ceiling as the Nova branch above.
1281
+ inputImage: ImageProcessor.safeBase64Convert(imageBuffer, "bedrock titan embed image"),
1282
+ });
1283
+ }
1284
+ else {
1285
+ // Text-only embedding (Titan Embed Text)
1286
+ requestBody = JSON.stringify({
1287
+ inputText: embedInput.text ?? "",
1288
+ });
1289
+ }
1183
1290
  const command = new InvokeModelCommand({
1184
1291
  modelId: embeddingModelName,
1185
1292
  contentType: "application/json",
@@ -1187,23 +1294,25 @@ export class AmazonBedrockProvider extends BaseProvider {
1187
1294
  body: requestBody,
1188
1295
  });
1189
1296
  const response = await withTimeout(this.bedrockClient.send(command), 60_000, new Error("Bedrock embedding API call timed out"));
1190
- // Parse the response
1297
+ // Parse the response — Titan returns `embedding`, Nova returns `embeddings[0].embedding`
1191
1298
  const responseBody = JSON.parse(new TextDecoder().decode(response.body));
1192
- if (!responseBody.embedding || !Array.isArray(responseBody.embedding)) {
1299
+ const embedding = isNovaModel
1300
+ ? responseBody.embeddings?.at(0)?.embedding
1301
+ : responseBody.embedding;
1302
+ if (!Array.isArray(embedding)) {
1193
1303
  throw new Error("Invalid embedding response from Bedrock");
1194
1304
  }
1195
1305
  logger.debug("Embedding generated successfully", {
1196
1306
  provider: this.providerName,
1197
1307
  model: embeddingModelName,
1198
- embeddingDimension: responseBody.embedding.length,
1308
+ embeddingDimension: embedding.length,
1199
1309
  });
1200
- return responseBody.embedding;
1310
+ return embedding;
1201
1311
  }
1202
1312
  catch (error) {
1203
1313
  logger.error("Embedding generation failed", {
1204
1314
  error: error instanceof Error ? error.message : String(error),
1205
1315
  model: embeddingModelName,
1206
- textLength: text.length,
1207
1316
  });
1208
1317
  throw this.handleProviderError(error);
1209
1318
  }
@@ -1215,7 +1324,7 @@ export class AmazonBedrockProvider extends BaseProvider {
1215
1324
  * @returns Promise resolving to an array of embedding vectors
1216
1325
  */
1217
1326
  async embedMany(texts, modelName) {
1218
- const embeddingModelName = modelName || "amazon.titan-embed-text-v2:0";
1327
+ const embeddingModelName = modelName || this.getDefaultEmbeddingModel();
1219
1328
  logger.debug("Generating batch embeddings", {
1220
1329
  provider: this.providerName,
1221
1330
  model: embeddingModelName,
@@ -1,5 +1,5 @@
1
1
  import type { AIProviderName } from "../constants/enums.js";
2
- import type { NeurolinkCredentials } from "../types/index.js";
2
+ import type { EmbedInput, NeurolinkCredentials } from "../types/index.js";
3
3
  import { OpenAIChatCompletionsProvider } from "./openaiChatCompletionsBase.js";
4
4
  /**
5
5
  * Cohere Provider — direct HTTP, no AI SDK.
@@ -41,7 +41,7 @@ export declare class CohereProvider extends OpenAIChatCompletionsProvider {
41
41
  * lives on the native API (POST /v2/embed). Documented at
42
42
  * https://docs.cohere.com/reference/embed.
43
43
  */
44
- embed(text: string, modelName?: string): Promise<number[]>;
44
+ embed(input: string | EmbedInput, modelName?: string): Promise<number[]>;
45
45
  /**
46
46
  * Batch embedding via Cohere's native /v2/embed endpoint. Cohere caps at
47
47
  * 96 inputs per request; larger batches are chunked.
@@ -115,7 +115,11 @@ export class CohereProvider extends OpenAIChatCompletionsProvider {
115
115
  * lives on the native API (POST /v2/embed). Documented at
116
116
  * https://docs.cohere.com/reference/embed.
117
117
  */
118
- async embed(text, modelName) {
118
+ async embed(input, modelName) {
119
+ if (typeof input !== "string" && input.image) {
120
+ throw new ProviderError(`${this.providerName} does not support image embeddings; provide text input`, this.providerName);
121
+ }
122
+ const text = typeof input === "string" ? input : (input.text ?? "");
119
123
  const vectors = await this.embedMany([text], modelName);
120
124
  if (!vectors[0]) {
121
125
  throw new ProviderError("Cohere /v2/embed returned no embeddings.", "cohere");
@@ -1,6 +1,6 @@
1
1
  import { type AIProviderName } from "../../constants/enums.js";
2
2
  import { BaseProvider } from "../../core/baseProvider.js";
3
- import type { ZodUnknownSchema, EnhancedGenerateResult, TextGenerationOptions, StreamOptions, StreamResult } from "../../types/index.js";
3
+ import type { EmbedInput, ZodUnknownSchema, EnhancedGenerateResult, TextGenerationOptions, StreamOptions, StreamResult } from "../../types/index.js";
4
4
  import type { LanguageModel, Schema } from "../../types/index.js";
5
5
  export declare class GoogleAIStudioProvider extends BaseProvider {
6
6
  private credentials?;
@@ -74,11 +74,11 @@ export declare class GoogleAIStudioProvider extends BaseProvider {
74
74
  protected getDefaultEmbeddingModel(): string;
75
75
  /**
76
76
  * Generate embeddings for text using Google AI Studio embedding models
77
- * @param text - The text to embed
77
+ * @param input - The text to embed (string or EmbedInput)
78
78
  * @param modelName - The embedding model to use (default: gemini-embedding-001)
79
79
  * @returns Promise resolving to the embedding vector
80
80
  */
81
- embed(text: string, modelName?: string): Promise<number[]>;
81
+ embed(input: string | EmbedInput, modelName?: string): Promise<number[]>;
82
82
  /**
83
83
  * Generate embeddings for multiple texts in a single batch
84
84
  * @param texts - The texts to embed
@@ -1571,11 +1571,15 @@ export class GoogleAIStudioProvider extends BaseProvider {
1571
1571
  }
1572
1572
  /**
1573
1573
  * Generate embeddings for text using Google AI Studio embedding models
1574
- * @param text - The text to embed
1574
+ * @param input - The text to embed (string or EmbedInput)
1575
1575
  * @param modelName - The embedding model to use (default: gemini-embedding-001)
1576
1576
  * @returns Promise resolving to the embedding vector
1577
1577
  */
1578
- async embed(text, modelName) {
1578
+ async embed(input, modelName) {
1579
+ if (typeof input !== "string" && input.image) {
1580
+ throw new ProviderError(`${this.providerName} does not support image embeddings; provide text input`, this.providerName);
1581
+ }
1582
+ const text = typeof input === "string" ? input : (input.text ?? "");
1579
1583
  const embeddingModelName = modelName || this.getDefaultEmbeddingModel() || "gemini-embedding-001";
1580
1584
  logger.debug("Generating embedding", {
1581
1585
  provider: this.providerName,
@@ -1,7 +1,7 @@
1
1
  import type { ZodType } from "zod";
2
2
  import { AIProviderName } from "../../constants/enums.js";
3
3
  import { BaseProvider } from "../../core/baseProvider.js";
4
- import type { EnhancedGenerateResult, TextGenerationOptions, StreamOptions, StreamResult, VertexAnthropicMessage, ChatMessage, MinimalChatMessage } from "../../types/index.js";
4
+ import type { EmbedInput, EnhancedGenerateResult, TextGenerationOptions, StreamOptions, StreamResult, VertexAnthropicMessage, ChatMessage, MinimalChatMessage } from "../../types/index.js";
5
5
  import type { Schema, LanguageModel } from "../../types/index.js";
6
6
  export declare function buildAnthropicHistoryMessages(conversationMessages: Array<ChatMessage | MinimalChatMessage>): VertexAnthropicMessage[];
7
7
  export declare function appendUserMessage(messages: VertexAnthropicMessage[], content: VertexAnthropicMessage["content"]): void;
@@ -424,7 +424,7 @@ export declare class GoogleVertexProvider extends BaseProvider {
424
424
  * provider", and `neurolink rag index --provider=vertex` fails even
425
425
  * though the SDK conceptually supports it.
426
426
  */
427
- embed(text: string, modelName?: string): Promise<number[]>;
427
+ embed(input: string | EmbedInput, modelName?: string): Promise<number[]>;
428
428
  /**
429
429
  * Batch-embed an array of strings via Vertex @google/genai.
430
430
  * Mirrors {@link embed} but returns one vector per input string.
@@ -6500,7 +6500,11 @@ export class GoogleVertexProvider extends BaseProvider {
6500
6500
  * provider", and `neurolink rag index --provider=vertex` fails even
6501
6501
  * though the SDK conceptually supports it.
6502
6502
  */
6503
- async embed(text, modelName) {
6503
+ async embed(input, modelName) {
6504
+ if (typeof input !== "string" && input.image) {
6505
+ throw new ProviderError(`${this.providerName} does not support image embeddings; provide text input`, this.providerName);
6506
+ }
6507
+ const text = typeof input === "string" ? input : (input.text ?? "");
6504
6508
  const embeddingModelName = modelName || this.getDefaultEmbeddingModel() || "text-embedding-004";
6505
6509
  logger.debug("Generating embedding", {
6506
6510
  provider: this.providerName,
@@ -1,6 +1,6 @@
1
1
  import type { AIProviderName } from "../constants/enums.js";
2
2
  import { BaseProvider } from "../core/baseProvider.js";
3
- import type { NeurolinkCredentials, StreamOptions, StreamResult, ValidationSchema } from "../types/index.js";
3
+ import type { EmbedInput, NeurolinkCredentials, StreamOptions, StreamResult, ValidationSchema } from "../types/index.js";
4
4
  import type { LanguageModel } from "../types/index.js";
5
5
  /**
6
6
  * Jina AI Provider — embeddings + reranking.
@@ -23,7 +23,7 @@ export declare class JinaProvider extends BaseProvider {
23
23
  protected getAISDKModel(): LanguageModel;
24
24
  protected executeStream(_options: StreamOptions, _analysisSchema?: ValidationSchema): Promise<StreamResult>;
25
25
  protected formatProviderError(error: unknown): Error;
26
- embed(text: string, modelName?: string): Promise<number[]>;
26
+ embed(input: string | EmbedInput, modelName?: string): Promise<number[]>;
27
27
  embedMany(texts: string[], modelName?: string): Promise<number[][]>;
28
28
  /**
29
29
  * Rerank a list of documents against a query.
@@ -76,7 +76,11 @@ export class JinaProvider extends BaseProvider {
76
76
  }
77
77
  return new ProviderError(`Jina AI error: ${message}`, "jina");
78
78
  }
79
- async embed(text, modelName) {
79
+ async embed(input, modelName) {
80
+ if (typeof input !== "string" && input.image) {
81
+ throw new ProviderError(`${this.providerName} does not support image embeddings; provide text input`, this.providerName);
82
+ }
83
+ const text = typeof input === "string" ? input : (input.text ?? "");
80
84
  const vectors = await this.callEmbeddings([text], modelName);
81
85
  if (!vectors[0]) {
82
86
  throw new Error("Jina AI returned no embedding for the provided text");
@@ -1,5 +1,5 @@
1
1
  import type { AIProviderName } from "../../constants/enums.js";
2
- import type { OpenAICompatBuildBodyArgs, OpenAICompatStreamLifecycleListeners } from "../../types/index.js";
2
+ import type { EmbedInput, OpenAICompatBuildBodyArgs, OpenAICompatStreamLifecycleListeners } from "../../types/index.js";
3
3
  import { OpenAIChatCompletionsProvider } from "../openaiChatCompletionsBase.js";
4
4
  /** Test hook: reset the discovery caches (state is module-global). */
5
5
  export declare function clearLiteLLMModelLimitsCache(): void;
@@ -87,7 +87,7 @@ export declare class LiteLLMProvider extends OpenAIChatCompletionsProvider {
87
87
  /**
88
88
  * Generate an embedding for a single text input via native /v1/embeddings.
89
89
  */
90
- embed(text: string, modelName?: string): Promise<number[]>;
90
+ embed(input: string | EmbedInput, modelName?: string): Promise<number[]>;
91
91
  /**
92
92
  * Generate embeddings for multiple text inputs via native /v1/embeddings.
93
93
  */
@@ -466,7 +466,11 @@ export class LiteLLMProvider extends OpenAIChatCompletionsProvider {
466
466
  /**
467
467
  * Generate an embedding for a single text input via native /v1/embeddings.
468
468
  */
469
- async embed(text, modelName) {
469
+ async embed(input, modelName) {
470
+ if (typeof input !== "string" && input.image) {
471
+ throw new ProviderError(`${this.providerName} does not support image embeddings; provide text input`, this.providerName);
472
+ }
473
+ const text = typeof input === "string" ? input : (input.text ?? "");
470
474
  const embeddingModelName = modelName ||
471
475
  process.env.LITELLM_EMBEDDING_MODEL ||
472
476
  "gemini-embedding-001";
@@ -1,5 +1,5 @@
1
1
  import type { AIProviderName } from "../../constants/enums.js";
2
- import type { NeurolinkCredentials, StreamOptions, TextGenerationOptions } from "../../types/index.js";
2
+ import type { EmbedInput, NeurolinkCredentials, StreamOptions, TextGenerationOptions } from "../../types/index.js";
3
3
  import { OpenAIChatCompletionsProvider } from "../openaiChatCompletionsBase.js";
4
4
  /**
5
5
  * Ollama Provider — direct HTTP, no AI SDK.
@@ -62,7 +62,7 @@ export declare class OllamaProvider extends OpenAIChatCompletionsProvider {
62
62
  * Uses `OLLAMA_EMBEDDING_MODEL` (default `nomic-embed-text`); the embedding
63
63
  * model must be pulled locally (`ollama pull nomic-embed-text`).
64
64
  */
65
- embed(text: string, modelName?: string): Promise<number[]>;
65
+ embed(input: string | EmbedInput, modelName?: string): Promise<number[]>;
66
66
  /**
67
67
  * Generate embeddings for multiple text inputs via native /v1/embeddings.
68
68
  */
@@ -214,7 +214,11 @@ export class OllamaProvider extends OpenAIChatCompletionsProvider {
214
214
  * Uses `OLLAMA_EMBEDDING_MODEL` (default `nomic-embed-text`); the embedding
215
215
  * model must be pulled locally (`ollama pull nomic-embed-text`).
216
216
  */
217
- async embed(text, modelName) {
217
+ async embed(input, modelName) {
218
+ if (typeof input !== "string" && input.image) {
219
+ throw new ProviderError(`${this.providerName} does not support image embeddings; provide text input`, this.providerName);
220
+ }
221
+ const text = typeof input === "string" ? input : (input.text ?? "");
218
222
  const embeddingModel = modelName ||
219
223
  process.env.OLLAMA_EMBEDDING_MODEL ||
220
224
  DEFAULT_EMBEDDING_MODEL;
@@ -1,5 +1,5 @@
1
1
  import type { AIProviderName } from "../../constants/enums.js";
2
- import type { EnhancedGenerateResult, NeurolinkCredentials, OpenAICompatStreamLifecycleListeners, TextGenerationOptions } from "../../types/index.js";
2
+ import type { EmbedInput, EnhancedGenerateResult, NeurolinkCredentials, OpenAICompatStreamLifecycleListeners, TextGenerationOptions } from "../../types/index.js";
3
3
  import { OpenAIChatCompletionsProvider } from "../openaiChatCompletionsBase.js";
4
4
  /**
5
5
  * OpenAI Provider — direct HTTP, no AI SDK.
@@ -38,11 +38,11 @@ export declare class OpenAIProvider extends OpenAIChatCompletionsProvider {
38
38
  /**
39
39
  * Generate an embedding for a single text input via native /v1/embeddings.
40
40
  *
41
- * @param text - The text to embed
41
+ * @param input - The text to embed (string or EmbedInput)
42
42
  * @param modelName - The embedding model to use (default: text-embedding-3-small)
43
43
  * @returns Promise resolving to the embedding vector
44
44
  */
45
- embed(text: string, modelName?: string): Promise<number[]>;
45
+ embed(input: string | EmbedInput, modelName?: string): Promise<number[]>;
46
46
  /**
47
47
  * Generate embeddings for multiple texts in a single batch via native /v1/embeddings.
48
48
  *
@@ -220,11 +220,15 @@ export class OpenAIProvider extends OpenAIChatCompletionsProvider {
220
220
  /**
221
221
  * Generate an embedding for a single text input via native /v1/embeddings.
222
222
  *
223
- * @param text - The text to embed
223
+ * @param input - The text to embed (string or EmbedInput)
224
224
  * @param modelName - The embedding model to use (default: text-embedding-3-small)
225
225
  * @returns Promise resolving to the embedding vector
226
226
  */
227
- async embed(text, modelName) {
227
+ async embed(input, modelName) {
228
+ if (typeof input !== "string" && input.image) {
229
+ throw new ProviderError(`${this.providerName} does not support image embeddings; provide text input`, this.providerName);
230
+ }
231
+ const text = typeof input === "string" ? input : (input.text ?? "");
228
232
  const embeddingModelName = modelName || this.getDefaultEmbeddingModel();
229
233
  logger.debug("Generating embedding", {
230
234
  provider: this.providerName,
@@ -1,6 +1,6 @@
1
1
  import type { AIProviderName } from "../constants/enums.js";
2
2
  import { BaseProvider } from "../core/baseProvider.js";
3
- import type { NeurolinkCredentials, StreamOptions, StreamResult, ValidationSchema } from "../types/index.js";
3
+ import type { EmbedInput, NeurolinkCredentials, StreamOptions, StreamResult, ValidationSchema } from "../types/index.js";
4
4
  import type { LanguageModel } from "../types/index.js";
5
5
  /**
6
6
  * Voyage AI Provider — embedding-only.
@@ -29,7 +29,7 @@ export declare class VoyageProvider extends BaseProvider {
29
29
  protected getAISDKModel(): LanguageModel;
30
30
  protected executeStream(_options: StreamOptions, _analysisSchema?: ValidationSchema): Promise<StreamResult>;
31
31
  protected formatProviderError(error: unknown): Error;
32
- embed(text: string, modelName?: string): Promise<number[]>;
32
+ embed(input: string | EmbedInput, modelName?: string): Promise<number[]>;
33
33
  embedMany(texts: string[], modelName?: string): Promise<number[][]>;
34
34
  /**
35
35
  * POST /embeddings — Voyage accepts up to 128 inputs per request.
@@ -86,7 +86,11 @@ export class VoyageProvider extends BaseProvider {
86
86
  return new ProviderError(`Voyage AI error: ${message}`, "voyage");
87
87
  }
88
88
  // ===== Embedding implementations =====
89
- async embed(text, modelName) {
89
+ async embed(input, modelName) {
90
+ if (typeof input !== "string" && input.image) {
91
+ throw new ProviderError(`${this.providerName} does not support image embeddings; provide text input`, this.providerName);
92
+ }
93
+ const text = typeof input === "string" ? input : (input.text ?? "");
90
94
  const vectors = await this.callEmbeddings([text], modelName);
91
95
  if (!vectors[0]) {
92
96
  throw new ProviderError("Voyage AI returned no embedding for the provided text", "voyage");
@@ -103,9 +103,14 @@ function windowResetEpochSeconds(window, nowSeconds) {
103
103
  }
104
104
  return 0;
105
105
  }
106
- function deriveWindowStatus(usedFraction) {
107
- return usedFraction >= 1 ? "rejected" : "allowed";
106
+ /** Classify only finite, nonnegative utilization; invalid measurements stay unknown. */
107
+ function deriveWindowStatus(percent) {
108
+ if (typeof percent !== "number" || !Number.isFinite(percent) || percent < 0) {
109
+ return "unknown";
110
+ }
111
+ return percent >= 100 ? "rejected" : "allowed";
108
112
  }
113
+ /** Normalize a reported window while retaining uncertainty in its status. */
109
114
  function toQuotaWindow(kind, group, window, nowSeconds) {
110
115
  if (!window) {
111
116
  return null;
@@ -115,7 +120,7 @@ function toQuotaWindow(kind, group, window, nowSeconds) {
115
120
  kind,
116
121
  group,
117
122
  used,
118
- status: deriveWindowStatus(used),
123
+ status: deriveWindowStatus(window.used_percent),
119
124
  resetsAt: windowResetEpochSeconds(window, nowSeconds),
120
125
  isActive: true,
121
126
  };
@@ -137,11 +142,16 @@ export function codexRateLimitsToQuota(rateLimits, now = Date.now()) {
137
142
  windows.push(secondaryWindow);
138
143
  }
139
144
  return {
145
+ unifiedStatus: rateLimits?.limit_reached === true || rateLimits?.allowed === false
146
+ ? "rejected"
147
+ : rateLimits?.allowed === true
148
+ ? "allowed"
149
+ : "unknown",
140
150
  sessionUsed,
141
- sessionStatus: deriveWindowStatus(sessionUsed),
151
+ sessionStatus: deriveWindowStatus(primary?.used_percent),
142
152
  sessionResetAt: windowResetEpochSeconds(primary, nowSeconds),
143
153
  weeklyUsed,
144
- weeklyStatus: deriveWindowStatus(weeklyUsed),
154
+ weeklyStatus: deriveWindowStatus(secondary?.used_percent),
145
155
  weeklyResetAt: windowResetEpochSeconds(secondary, nowSeconds),
146
156
  // Codex has no overage/fallback concept; keep neutral defaults.
147
157
  fallbackPercentage: 0,
@@ -169,7 +179,7 @@ export function parseCodexRateLimitHeaders(headers, now = Date.now()) {
169
179
  const rateLimits = "rate_limits" in parsed && parsed.rate_limits
170
180
  ? parsed.rate_limits
171
181
  : parsed;
172
- return codexRateLimitsToQuota(rateLimits, now);
182
+ return { ...codexRateLimitsToQuota(rateLimits, now), source: "headers" };
173
183
  }
174
184
  catch {
175
185
  return null;
@@ -204,6 +214,8 @@ export async function fetchCodexAccountUsage(account, options = {}) {
204
214
  const usage = (await response.json());
205
215
  const rateLimits = usage.rate_limit
206
216
  ? {
217
+ allowed: usage.rate_limit.allowed,
218
+ limit_reached: usage.rate_limit.limit_reached,
207
219
  primary: usage.rate_limit.primary_window,
208
220
  secondary: usage.rate_limit.secondary_window,
209
221
  }
@@ -8,6 +8,7 @@
8
8
  */
9
9
  import { ClaudeStreamSerializer, generateToolUseId } from "./claudeFormat.js";
10
10
  import { extractCodexUsage } from "./codexUsage.js";
11
+ import { proxyTokenUsage } from "./proxyTokenUsage.js";
11
12
  export class CodexFallbackResponseError extends Error {
12
13
  status;
13
14
  responseBody;
@@ -367,13 +368,16 @@ export function parseCodexFallbackSSE(sse) {
367
368
  }
368
369
  const parsedUsage = extractCodexUsage(payload);
369
370
  if (parsedUsage) {
370
- usage = {
371
- input: parsedUsage.inputTokens,
372
- output: parsedUsage.outputTokens,
373
- total: parsedUsage.inputTokens + parsedUsage.outputTokens,
371
+ usage = proxyTokenUsage({
372
+ inputTokens: parsedUsage.inputTokens,
373
+ outputTokens: parsedUsage.outputTokens,
374
+ reasoningTokens: parsedUsage.reasoningTokensObserved
375
+ ? parsedUsage.reasoningTokens
376
+ : undefined,
374
377
  cacheReadTokens: parsedUsage.cacheReadTokens,
375
378
  cacheCreationTokens: parsedUsage.cacheCreationTokens,
376
- };
379
+ inputIncludesCachedTokens: true,
380
+ });
377
381
  }
378
382
  textFromResponse = outputTextFromResponse(payload);
379
383
  }
@@ -516,13 +520,16 @@ export async function createCodexFallbackStream(response, model) {
516
520
  completed = true;
517
521
  const parsedUsage = extractCodexUsage(payload);
518
522
  if (parsedUsage) {
519
- usage = {
520
- input: parsedUsage.inputTokens,
521
- output: parsedUsage.outputTokens,
522
- total: parsedUsage.inputTokens + parsedUsage.outputTokens,
523
+ usage = proxyTokenUsage({
524
+ inputTokens: parsedUsage.inputTokens,
525
+ outputTokens: parsedUsage.outputTokens,
526
+ reasoningTokens: parsedUsage.reasoningTokensObserved
527
+ ? parsedUsage.reasoningTokens
528
+ : undefined,
523
529
  cacheReadTokens: parsedUsage.cacheReadTokens,
524
530
  cacheCreationTokens: parsedUsage.cacheCreationTokens,
525
- };
531
+ inputIncludesCachedTokens: true,
532
+ });
526
533
  }
527
534
  const responseBody = payload.response;
528
535
  if (isRecord(responseBody) && Array.isArray(responseBody.output)) {
@@ -107,6 +107,9 @@ export function extractCodexUsage(payload) {
107
107
  cacheReadTokens: nonNegativeInt(inputDetails?.cached_tokens),
108
108
  cacheCreationTokens: nonNegativeInt(inputDetails?.cache_write_tokens),
109
109
  reasoningTokens: nonNegativeInt(outputDetails?.reasoning_tokens),
110
+ reasoningTokensObserved: typeof outputDetails?.reasoning_tokens === "number" &&
111
+ Number.isFinite(outputDetails.reasoning_tokens) &&
112
+ outputDetails.reasoning_tokens >= 0,
110
113
  };
111
114
  }
112
115
  /**