@juspay/neurolink 8.5.0 → 8.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/adapters/providerImageAdapter.js +56 -9
- package/dist/config/conversationMemory.d.ts +6 -0
- package/dist/config/conversationMemory.js +14 -0
- package/dist/constants/enums.d.ts +23 -3
- package/dist/constants/enums.js +30 -4
- package/dist/constants/tokens.d.ts +27 -12
- package/dist/constants/tokens.js +46 -12
- package/dist/core/modules/GenerationHandler.js +20 -5
- package/dist/core/modules/MessageBuilder.js +4 -0
- package/dist/lib/adapters/providerImageAdapter.js +56 -9
- package/dist/lib/config/conversationMemory.d.ts +6 -0
- package/dist/lib/config/conversationMemory.js +14 -0
- package/dist/lib/constants/enums.d.ts +23 -3
- package/dist/lib/constants/enums.js +30 -4
- package/dist/lib/constants/tokens.d.ts +27 -12
- package/dist/lib/constants/tokens.js +46 -12
- package/dist/lib/core/modules/GenerationHandler.js +20 -5
- package/dist/lib/core/modules/MessageBuilder.js +4 -0
- package/dist/lib/models/modelRegistry.js +93 -0
- package/dist/lib/providers/googleAiStudio.d.ts +27 -0
- package/dist/lib/providers/googleAiStudio.js +27 -0
- package/dist/lib/providers/googleVertex.d.ts +35 -0
- package/dist/lib/providers/googleVertex.js +38 -0
- package/dist/lib/types/generateTypes.d.ts +49 -0
- package/dist/lib/utils/messageBuilder.js +18 -1
- package/dist/models/modelRegistry.js +93 -0
- package/dist/providers/googleAiStudio.d.ts +27 -0
- package/dist/providers/googleAiStudio.js +27 -0
- package/dist/providers/googleVertex.d.ts +35 -0
- package/dist/providers/googleVertex.js +38 -0
- package/dist/types/generateTypes.d.ts +49 -0
- package/dist/utils/messageBuilder.js +18 -1
- package/package.json +1 -1
|
@@ -6,6 +6,33 @@ import { BaseProvider } from "../core/baseProvider.js";
|
|
|
6
6
|
/**
|
|
7
7
|
* Google AI Studio provider implementation using BaseProvider
|
|
8
8
|
* Migrated from original GoogleAIStudio class to new factory pattern
|
|
9
|
+
*
|
|
10
|
+
* @important Structured Output Limitation
|
|
11
|
+
* Google Gemini models cannot combine function calling (tools) with structured
|
|
12
|
+
* output (JSON schema). When using schemas with output.format: "json", you MUST
|
|
13
|
+
* set disableTools: true.
|
|
14
|
+
*
|
|
15
|
+
* Error without disableTools:
|
|
16
|
+
* "Function calling with a response mime type: 'application/json' is unsupported"
|
|
17
|
+
*
|
|
18
|
+
* This is a Google API limitation documented at:
|
|
19
|
+
* https://ai.google.dev/gemini-api/docs/function-calling
|
|
20
|
+
*
|
|
21
|
+
* @example
|
|
22
|
+
* ```typescript
|
|
23
|
+
* // ✅ Correct usage with schemas
|
|
24
|
+
* const provider = new GoogleAIStudioProvider("gemini-2.5-flash");
|
|
25
|
+
* const result = await provider.generate({
|
|
26
|
+
* input: { text: "Analyze data" },
|
|
27
|
+
* schema: MySchema,
|
|
28
|
+
* output: { format: "json" },
|
|
29
|
+
* disableTools: true // Required
|
|
30
|
+
* });
|
|
31
|
+
* ```
|
|
32
|
+
*
|
|
33
|
+
* @note Gemini 3 Pro Preview (November 2025) will support combining tools + schemas
|
|
34
|
+
* @note "Too many states for serving" errors can occur with complex schemas + tools.
|
|
35
|
+
* Solution: Simplify schema or use disableTools: true
|
|
9
36
|
*/
|
|
10
37
|
export declare class GoogleAIStudioProvider extends BaseProvider {
|
|
11
38
|
constructor(modelName?: string, sdk?: unknown);
|
|
@@ -27,6 +27,33 @@ if (!process.env.GOOGLE_GENERATIVE_AI_API_KEY &&
|
|
|
27
27
|
/**
|
|
28
28
|
* Google AI Studio provider implementation using BaseProvider
|
|
29
29
|
* Migrated from original GoogleAIStudio class to new factory pattern
|
|
30
|
+
*
|
|
31
|
+
* @important Structured Output Limitation
|
|
32
|
+
* Google Gemini models cannot combine function calling (tools) with structured
|
|
33
|
+
* output (JSON schema). When using schemas with output.format: "json", you MUST
|
|
34
|
+
* set disableTools: true.
|
|
35
|
+
*
|
|
36
|
+
* Error without disableTools:
|
|
37
|
+
* "Function calling with a response mime type: 'application/json' is unsupported"
|
|
38
|
+
*
|
|
39
|
+
* This is a Google API limitation documented at:
|
|
40
|
+
* https://ai.google.dev/gemini-api/docs/function-calling
|
|
41
|
+
*
|
|
42
|
+
* @example
|
|
43
|
+
* ```typescript
|
|
44
|
+
* // ✅ Correct usage with schemas
|
|
45
|
+
* const provider = new GoogleAIStudioProvider("gemini-2.5-flash");
|
|
46
|
+
* const result = await provider.generate({
|
|
47
|
+
* input: { text: "Analyze data" },
|
|
48
|
+
* schema: MySchema,
|
|
49
|
+
* output: { format: "json" },
|
|
50
|
+
* disableTools: true // Required
|
|
51
|
+
* });
|
|
52
|
+
* ```
|
|
53
|
+
*
|
|
54
|
+
* @note Gemini 3 Pro Preview (November 2025) will support combining tools + schemas
|
|
55
|
+
* @note "Too many states for serving" errors can occur with complex schemas + tools.
|
|
56
|
+
* Solution: Simplify schema or use disableTools: true
|
|
30
57
|
*/
|
|
31
58
|
export class GoogleAIStudioProvider extends BaseProvider {
|
|
32
59
|
constructor(modelName, sdk) {
|
|
@@ -13,6 +13,41 @@ import { BaseProvider } from "../core/baseProvider.js";
|
|
|
13
13
|
* - Fresh model creation for each request
|
|
14
14
|
* - Enhanced error handling with setup guidance
|
|
15
15
|
* - Tool registration and context management
|
|
16
|
+
*
|
|
17
|
+
* @important Structured Output Limitation (Gemini Models Only)
|
|
18
|
+
* Google Gemini models on Vertex AI cannot combine function calling (tools) with
|
|
19
|
+
* structured output (JSON schema). When using schemas, you MUST set disableTools: true.
|
|
20
|
+
*
|
|
21
|
+
* Error without disableTools:
|
|
22
|
+
* "Function calling with a response mime type: 'application/json' is unsupported"
|
|
23
|
+
*
|
|
24
|
+
* This limitation ONLY affects Gemini models. Anthropic Claude models via Vertex
|
|
25
|
+
* AI do NOT have this limitation and support both tools + schemas simultaneously.
|
|
26
|
+
*
|
|
27
|
+
* @example Gemini models with schemas
|
|
28
|
+
* ```typescript
|
|
29
|
+
* const provider = new GoogleVertexProvider("gemini-2.5-flash");
|
|
30
|
+
* const result = await provider.generate({
|
|
31
|
+
* input: { text: "Analyze data" },
|
|
32
|
+
* schema: MySchema,
|
|
33
|
+
* output: { format: "json" },
|
|
34
|
+
* disableTools: true // Required for Gemini models
|
|
35
|
+
* });
|
|
36
|
+
* ```
|
|
37
|
+
*
|
|
38
|
+
* @example Claude models (no limitation)
|
|
39
|
+
* ```typescript
|
|
40
|
+
* const provider = new GoogleVertexProvider("claude-3-5-sonnet-20241022");
|
|
41
|
+
* const result = await provider.generate({
|
|
42
|
+
* input: { text: "Analyze data" },
|
|
43
|
+
* schema: MySchema,
|
|
44
|
+
* output: { format: "json" }
|
|
45
|
+
* // No disableTools needed - Claude supports both
|
|
46
|
+
* });
|
|
47
|
+
* ```
|
|
48
|
+
*
|
|
49
|
+
* @note Gemini 3 Pro Preview (November 2025) will support combining tools + schemas
|
|
50
|
+
* @see https://cloud.google.com/vertex-ai/docs/generative-ai/learn/models
|
|
16
51
|
*/
|
|
17
52
|
export declare class GoogleVertexProvider extends BaseProvider {
|
|
18
53
|
private projectId;
|
|
@@ -234,6 +234,41 @@ const isAnthropicModel = (modelName) => {
|
|
|
234
234
|
* - Fresh model creation for each request
|
|
235
235
|
* - Enhanced error handling with setup guidance
|
|
236
236
|
* - Tool registration and context management
|
|
237
|
+
*
|
|
238
|
+
* @important Structured Output Limitation (Gemini Models Only)
|
|
239
|
+
* Google Gemini models on Vertex AI cannot combine function calling (tools) with
|
|
240
|
+
* structured output (JSON schema). When using schemas, you MUST set disableTools: true.
|
|
241
|
+
*
|
|
242
|
+
* Error without disableTools:
|
|
243
|
+
* "Function calling with a response mime type: 'application/json' is unsupported"
|
|
244
|
+
*
|
|
245
|
+
* This limitation ONLY affects Gemini models. Anthropic Claude models via Vertex
|
|
246
|
+
* AI do NOT have this limitation and support both tools + schemas simultaneously.
|
|
247
|
+
*
|
|
248
|
+
* @example Gemini models with schemas
|
|
249
|
+
* ```typescript
|
|
250
|
+
* const provider = new GoogleVertexProvider("gemini-2.5-flash");
|
|
251
|
+
* const result = await provider.generate({
|
|
252
|
+
* input: { text: "Analyze data" },
|
|
253
|
+
* schema: MySchema,
|
|
254
|
+
* output: { format: "json" },
|
|
255
|
+
* disableTools: true // Required for Gemini models
|
|
256
|
+
* });
|
|
257
|
+
* ```
|
|
258
|
+
*
|
|
259
|
+
* @example Claude models (no limitation)
|
|
260
|
+
* ```typescript
|
|
261
|
+
* const provider = new GoogleVertexProvider("claude-3-5-sonnet-20241022");
|
|
262
|
+
* const result = await provider.generate({
|
|
263
|
+
* input: { text: "Analyze data" },
|
|
264
|
+
* schema: MySchema,
|
|
265
|
+
* output: { format: "json" }
|
|
266
|
+
* // No disableTools needed - Claude supports both
|
|
267
|
+
* });
|
|
268
|
+
* ```
|
|
269
|
+
*
|
|
270
|
+
* @note Gemini 3 Pro Preview (November 2025) will support combining tools + schemas
|
|
271
|
+
* @see https://cloud.google.com/vertex-ai/docs/generative-ai/learn/models
|
|
237
272
|
*/
|
|
238
273
|
export class GoogleVertexProvider extends BaseProvider {
|
|
239
274
|
projectId;
|
|
@@ -1363,11 +1398,14 @@ export class GoogleVertexProvider extends BaseProvider {
|
|
|
1363
1398
|
getModelSuggestions(requestedModel) {
|
|
1364
1399
|
const availableModels = {
|
|
1365
1400
|
google: [
|
|
1401
|
+
"gemini-3-pro-preview-11-2025",
|
|
1402
|
+
"gemini-3-pro-latest",
|
|
1366
1403
|
"gemini-3-pro-preview",
|
|
1367
1404
|
"gemini-2.5-pro",
|
|
1368
1405
|
"gemini-2.5-flash",
|
|
1369
1406
|
"gemini-2.5-flash-lite",
|
|
1370
1407
|
"gemini-2.0-flash-001",
|
|
1408
|
+
"gemini-2.0-flash-lite",
|
|
1371
1409
|
"gemini-1.5-pro",
|
|
1372
1410
|
"gemini-1.5-flash",
|
|
1373
1411
|
],
|
|
@@ -34,9 +34,58 @@ export type GenerateOptions = {
|
|
|
34
34
|
temperature?: number;
|
|
35
35
|
maxTokens?: number;
|
|
36
36
|
systemPrompt?: string;
|
|
37
|
+
/**
|
|
38
|
+
* Zod schema for structured output validation
|
|
39
|
+
*
|
|
40
|
+
* @important Google Gemini Limitation
|
|
41
|
+
* Google Vertex AI and Google AI Studio cannot combine function calling with
|
|
42
|
+
* structured output. You MUST use `disableTools: true` when using schemas with
|
|
43
|
+
* Google providers.
|
|
44
|
+
*
|
|
45
|
+
* Error without disableTools: "Function calling with a response mime type:
|
|
46
|
+
* 'application/json' is unsupported"
|
|
47
|
+
*
|
|
48
|
+
* This is a documented Google API limitation, not a NeuroLink bug.
|
|
49
|
+
* All frameworks (LangChain, Vercel AI SDK, Agno, Instructor) use this approach.
|
|
50
|
+
*
|
|
51
|
+
* @example
|
|
52
|
+
* ```typescript
|
|
53
|
+
* // ✅ Correct for Google providers
|
|
54
|
+
* const result = await neurolink.generate({
|
|
55
|
+
* schema: MySchema,
|
|
56
|
+
* provider: "vertex",
|
|
57
|
+
* disableTools: true // Required for Google
|
|
58
|
+
* });
|
|
59
|
+
*
|
|
60
|
+
* // ✅ No restriction for other providers
|
|
61
|
+
* const result = await neurolink.generate({
|
|
62
|
+
* schema: MySchema,
|
|
63
|
+
* provider: "openai" // Works without disableTools
|
|
64
|
+
* });
|
|
65
|
+
* ```
|
|
66
|
+
*
|
|
67
|
+
* @see https://ai.google.dev/gemini-api/docs/function-calling
|
|
68
|
+
*/
|
|
37
69
|
schema?: ValidationSchema;
|
|
38
70
|
tools?: Record<string, Tool>;
|
|
39
71
|
timeout?: number | string;
|
|
72
|
+
/**
|
|
73
|
+
* Disable tool execution (including built-in tools)
|
|
74
|
+
*
|
|
75
|
+
* @required For Google Gemini providers when using schemas
|
|
76
|
+
* Google Vertex AI and Google AI Studio require this flag when using
|
|
77
|
+
* structured output (schemas) due to Google API limitations.
|
|
78
|
+
*
|
|
79
|
+
* @example
|
|
80
|
+
* ```typescript
|
|
81
|
+
* // Required for Google providers with schemas
|
|
82
|
+
* await neurolink.generate({
|
|
83
|
+
* schema: MySchema,
|
|
84
|
+
* provider: "vertex",
|
|
85
|
+
* disableTools: true
|
|
86
|
+
* });
|
|
87
|
+
* ```
|
|
88
|
+
*/
|
|
40
89
|
disableTools?: boolean;
|
|
41
90
|
enableEvaluation?: boolean;
|
|
42
91
|
enableAnalytics?: boolean;
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Centralized logic for building message arrays from TextGenerationOptions
|
|
4
4
|
* Enhanced with multimodal support for images
|
|
5
5
|
*/
|
|
6
|
-
import { CONVERSATION_INSTRUCTIONS } from "../config/conversationMemory.js";
|
|
6
|
+
import { CONVERSATION_INSTRUCTIONS, STRUCTURED_OUTPUT_INSTRUCTIONS, } from "../config/conversationMemory.js";
|
|
7
7
|
import { ProviderImageAdapter, MultimodalLogger, } from "../adapters/providerImageAdapter.js";
|
|
8
8
|
import { logger } from "./logger.js";
|
|
9
9
|
import { FileDetector } from "./fileDetector.js";
|
|
@@ -199,6 +199,15 @@ function formatCSVMetadata(metadata) {
|
|
|
199
199
|
}
|
|
200
200
|
return parts.length > 0 ? `**Metadata**: ${parts.join(" | ")}` : "";
|
|
201
201
|
}
|
|
202
|
+
/**
|
|
203
|
+
* Check if structured output mode should be enabled
|
|
204
|
+
* Structured output is used when a schema is provided with json/structured format
|
|
205
|
+
*/
|
|
206
|
+
function shouldUseStructuredOutput(options) {
|
|
207
|
+
return (!!options.schema &&
|
|
208
|
+
(options.output?.format === "json" ||
|
|
209
|
+
options.output?.format === "structured"));
|
|
210
|
+
}
|
|
202
211
|
/**
|
|
203
212
|
* Build a properly formatted message array for AI providers
|
|
204
213
|
* Combines system prompt, conversation history, and current user prompt
|
|
@@ -215,6 +224,10 @@ export async function buildMessagesArray(options) {
|
|
|
215
224
|
if (hasConversationHistory) {
|
|
216
225
|
systemPrompt = `${systemPrompt.trim()}${CONVERSATION_INSTRUCTIONS}`;
|
|
217
226
|
}
|
|
227
|
+
// Add structured output instructions when schema is provided with json/structured format
|
|
228
|
+
if (shouldUseStructuredOutput(options)) {
|
|
229
|
+
systemPrompt = `${systemPrompt.trim()}${STRUCTURED_OUTPUT_INSTRUCTIONS}`;
|
|
230
|
+
}
|
|
218
231
|
// Add system message if we have one
|
|
219
232
|
if (systemPrompt.trim()) {
|
|
220
233
|
messages.push({
|
|
@@ -473,6 +486,10 @@ export async function buildMultimodalMessagesArray(options, provider, model) {
|
|
|
473
486
|
if (hasConversationHistory) {
|
|
474
487
|
systemPrompt = `${systemPrompt.trim()}${CONVERSATION_INSTRUCTIONS}`;
|
|
475
488
|
}
|
|
489
|
+
// Add structured output instructions when schema is provided with json/structured format
|
|
490
|
+
if (shouldUseStructuredOutput(options)) {
|
|
491
|
+
systemPrompt = `${systemPrompt.trim()}${STRUCTURED_OUTPUT_INSTRUCTIONS}`;
|
|
492
|
+
}
|
|
476
493
|
// Add file handling guidance when multimodal files are present
|
|
477
494
|
const hasCSVFiles = (options.input.csvFiles && options.input.csvFiles.length > 0) ||
|
|
478
495
|
(options.input.files &&
|
|
@@ -188,6 +188,99 @@ export const MODEL_REGISTRY = {
|
|
|
188
188
|
category: "general",
|
|
189
189
|
},
|
|
190
190
|
// Anthropic Models
|
|
191
|
+
[AnthropicModels.CLAUDE_OPUS_4_5]: {
|
|
192
|
+
id: AnthropicModels.CLAUDE_OPUS_4_5,
|
|
193
|
+
name: "Claude Opus 4.5",
|
|
194
|
+
provider: AIProviderName.ANTHROPIC,
|
|
195
|
+
description: "Anthropic's most capable model with exceptional reasoning, coding, and multimodal capabilities",
|
|
196
|
+
capabilities: {
|
|
197
|
+
vision: true,
|
|
198
|
+
functionCalling: true,
|
|
199
|
+
codeGeneration: true,
|
|
200
|
+
reasoning: true,
|
|
201
|
+
multimodal: true,
|
|
202
|
+
streaming: true,
|
|
203
|
+
jsonMode: false,
|
|
204
|
+
},
|
|
205
|
+
pricing: {
|
|
206
|
+
inputCostPer1K: 0.015,
|
|
207
|
+
outputCostPer1K: 0.075,
|
|
208
|
+
currency: "USD",
|
|
209
|
+
},
|
|
210
|
+
performance: {
|
|
211
|
+
speed: "medium",
|
|
212
|
+
quality: "high",
|
|
213
|
+
accuracy: "high",
|
|
214
|
+
},
|
|
215
|
+
limits: {
|
|
216
|
+
maxContextTokens: 200000,
|
|
217
|
+
maxOutputTokens: 64000,
|
|
218
|
+
maxRequestsPerMinute: 50,
|
|
219
|
+
},
|
|
220
|
+
useCases: {
|
|
221
|
+
coding: 10,
|
|
222
|
+
creative: 10,
|
|
223
|
+
analysis: 10,
|
|
224
|
+
conversation: 9,
|
|
225
|
+
reasoning: 10,
|
|
226
|
+
translation: 9,
|
|
227
|
+
summarization: 9,
|
|
228
|
+
},
|
|
229
|
+
aliases: [
|
|
230
|
+
"claude-4.5-opus",
|
|
231
|
+
"claude-opus-latest",
|
|
232
|
+
"opus-4.5",
|
|
233
|
+
"anthropic-flagship",
|
|
234
|
+
],
|
|
235
|
+
deprecated: false,
|
|
236
|
+
isLocal: false,
|
|
237
|
+
releaseDate: "2025-11-24",
|
|
238
|
+
category: "reasoning",
|
|
239
|
+
},
|
|
240
|
+
[AnthropicModels.CLAUDE_SONNET_4_5]: {
|
|
241
|
+
id: AnthropicModels.CLAUDE_SONNET_4_5,
|
|
242
|
+
name: "Claude Sonnet 4.5",
|
|
243
|
+
provider: AIProviderName.ANTHROPIC,
|
|
244
|
+
description: "Balanced Claude model with excellent performance across all tasks including vision and reasoning",
|
|
245
|
+
capabilities: {
|
|
246
|
+
vision: true,
|
|
247
|
+
functionCalling: true,
|
|
248
|
+
codeGeneration: true,
|
|
249
|
+
reasoning: true,
|
|
250
|
+
multimodal: true,
|
|
251
|
+
streaming: true,
|
|
252
|
+
jsonMode: false,
|
|
253
|
+
},
|
|
254
|
+
pricing: {
|
|
255
|
+
inputCostPer1K: 0.003,
|
|
256
|
+
outputCostPer1K: 0.015,
|
|
257
|
+
currency: "USD",
|
|
258
|
+
},
|
|
259
|
+
performance: {
|
|
260
|
+
speed: "medium",
|
|
261
|
+
quality: "high",
|
|
262
|
+
accuracy: "high",
|
|
263
|
+
},
|
|
264
|
+
limits: {
|
|
265
|
+
maxContextTokens: 200000,
|
|
266
|
+
maxOutputTokens: 64000,
|
|
267
|
+
maxRequestsPerMinute: 100,
|
|
268
|
+
},
|
|
269
|
+
useCases: {
|
|
270
|
+
coding: 10,
|
|
271
|
+
creative: 9,
|
|
272
|
+
analysis: 9,
|
|
273
|
+
conversation: 9,
|
|
274
|
+
reasoning: 10,
|
|
275
|
+
translation: 8,
|
|
276
|
+
summarization: 8,
|
|
277
|
+
},
|
|
278
|
+
aliases: ["claude-4.5-sonnet", "claude-sonnet-latest", "sonnet-4.5"],
|
|
279
|
+
deprecated: false,
|
|
280
|
+
isLocal: false,
|
|
281
|
+
releaseDate: "2025-09-29",
|
|
282
|
+
category: "coding",
|
|
283
|
+
},
|
|
191
284
|
[AnthropicModels.CLAUDE_4_5_HAIKU]: {
|
|
192
285
|
id: AnthropicModels.CLAUDE_4_5_HAIKU,
|
|
193
286
|
name: "Claude 4.5 Haiku",
|
|
@@ -6,6 +6,33 @@ import { BaseProvider } from "../core/baseProvider.js";
|
|
|
6
6
|
/**
|
|
7
7
|
* Google AI Studio provider implementation using BaseProvider
|
|
8
8
|
* Migrated from original GoogleAIStudio class to new factory pattern
|
|
9
|
+
*
|
|
10
|
+
* @important Structured Output Limitation
|
|
11
|
+
* Google Gemini models cannot combine function calling (tools) with structured
|
|
12
|
+
* output (JSON schema). When using schemas with output.format: "json", you MUST
|
|
13
|
+
* set disableTools: true.
|
|
14
|
+
*
|
|
15
|
+
* Error without disableTools:
|
|
16
|
+
* "Function calling with a response mime type: 'application/json' is unsupported"
|
|
17
|
+
*
|
|
18
|
+
* This is a Google API limitation documented at:
|
|
19
|
+
* https://ai.google.dev/gemini-api/docs/function-calling
|
|
20
|
+
*
|
|
21
|
+
* @example
|
|
22
|
+
* ```typescript
|
|
23
|
+
* // ✅ Correct usage with schemas
|
|
24
|
+
* const provider = new GoogleAIStudioProvider("gemini-2.5-flash");
|
|
25
|
+
* const result = await provider.generate({
|
|
26
|
+
* input: { text: "Analyze data" },
|
|
27
|
+
* schema: MySchema,
|
|
28
|
+
* output: { format: "json" },
|
|
29
|
+
* disableTools: true // Required
|
|
30
|
+
* });
|
|
31
|
+
* ```
|
|
32
|
+
*
|
|
33
|
+
* @note Gemini 3 Pro Preview (November 2025) will support combining tools + schemas
|
|
34
|
+
* @note "Too many states for serving" errors can occur with complex schemas + tools.
|
|
35
|
+
* Solution: Simplify schema or use disableTools: true
|
|
9
36
|
*/
|
|
10
37
|
export declare class GoogleAIStudioProvider extends BaseProvider {
|
|
11
38
|
constructor(modelName?: string, sdk?: unknown);
|
|
@@ -27,6 +27,33 @@ if (!process.env.GOOGLE_GENERATIVE_AI_API_KEY &&
|
|
|
27
27
|
/**
|
|
28
28
|
* Google AI Studio provider implementation using BaseProvider
|
|
29
29
|
* Migrated from original GoogleAIStudio class to new factory pattern
|
|
30
|
+
*
|
|
31
|
+
* @important Structured Output Limitation
|
|
32
|
+
* Google Gemini models cannot combine function calling (tools) with structured
|
|
33
|
+
* output (JSON schema). When using schemas with output.format: "json", you MUST
|
|
34
|
+
* set disableTools: true.
|
|
35
|
+
*
|
|
36
|
+
* Error without disableTools:
|
|
37
|
+
* "Function calling with a response mime type: 'application/json' is unsupported"
|
|
38
|
+
*
|
|
39
|
+
* This is a Google API limitation documented at:
|
|
40
|
+
* https://ai.google.dev/gemini-api/docs/function-calling
|
|
41
|
+
*
|
|
42
|
+
* @example
|
|
43
|
+
* ```typescript
|
|
44
|
+
* // ✅ Correct usage with schemas
|
|
45
|
+
* const provider = new GoogleAIStudioProvider("gemini-2.5-flash");
|
|
46
|
+
* const result = await provider.generate({
|
|
47
|
+
* input: { text: "Analyze data" },
|
|
48
|
+
* schema: MySchema,
|
|
49
|
+
* output: { format: "json" },
|
|
50
|
+
* disableTools: true // Required
|
|
51
|
+
* });
|
|
52
|
+
* ```
|
|
53
|
+
*
|
|
54
|
+
* @note Gemini 3 Pro Preview (November 2025) will support combining tools + schemas
|
|
55
|
+
* @note "Too many states for serving" errors can occur with complex schemas + tools.
|
|
56
|
+
* Solution: Simplify schema or use disableTools: true
|
|
30
57
|
*/
|
|
31
58
|
export class GoogleAIStudioProvider extends BaseProvider {
|
|
32
59
|
constructor(modelName, sdk) {
|
|
@@ -13,6 +13,41 @@ import { BaseProvider } from "../core/baseProvider.js";
|
|
|
13
13
|
* - Fresh model creation for each request
|
|
14
14
|
* - Enhanced error handling with setup guidance
|
|
15
15
|
* - Tool registration and context management
|
|
16
|
+
*
|
|
17
|
+
* @important Structured Output Limitation (Gemini Models Only)
|
|
18
|
+
* Google Gemini models on Vertex AI cannot combine function calling (tools) with
|
|
19
|
+
* structured output (JSON schema). When using schemas, you MUST set disableTools: true.
|
|
20
|
+
*
|
|
21
|
+
* Error without disableTools:
|
|
22
|
+
* "Function calling with a response mime type: 'application/json' is unsupported"
|
|
23
|
+
*
|
|
24
|
+
* This limitation ONLY affects Gemini models. Anthropic Claude models via Vertex
|
|
25
|
+
* AI do NOT have this limitation and support both tools + schemas simultaneously.
|
|
26
|
+
*
|
|
27
|
+
* @example Gemini models with schemas
|
|
28
|
+
* ```typescript
|
|
29
|
+
* const provider = new GoogleVertexProvider("gemini-2.5-flash");
|
|
30
|
+
* const result = await provider.generate({
|
|
31
|
+
* input: { text: "Analyze data" },
|
|
32
|
+
* schema: MySchema,
|
|
33
|
+
* output: { format: "json" },
|
|
34
|
+
* disableTools: true // Required for Gemini models
|
|
35
|
+
* });
|
|
36
|
+
* ```
|
|
37
|
+
*
|
|
38
|
+
* @example Claude models (no limitation)
|
|
39
|
+
* ```typescript
|
|
40
|
+
* const provider = new GoogleVertexProvider("claude-3-5-sonnet-20241022");
|
|
41
|
+
* const result = await provider.generate({
|
|
42
|
+
* input: { text: "Analyze data" },
|
|
43
|
+
* schema: MySchema,
|
|
44
|
+
* output: { format: "json" }
|
|
45
|
+
* // No disableTools needed - Claude supports both
|
|
46
|
+
* });
|
|
47
|
+
* ```
|
|
48
|
+
*
|
|
49
|
+
* @note Gemini 3 Pro Preview (November 2025) will support combining tools + schemas
|
|
50
|
+
* @see https://cloud.google.com/vertex-ai/docs/generative-ai/learn/models
|
|
16
51
|
*/
|
|
17
52
|
export declare class GoogleVertexProvider extends BaseProvider {
|
|
18
53
|
private projectId;
|
|
@@ -234,6 +234,41 @@ const isAnthropicModel = (modelName) => {
|
|
|
234
234
|
* - Fresh model creation for each request
|
|
235
235
|
* - Enhanced error handling with setup guidance
|
|
236
236
|
* - Tool registration and context management
|
|
237
|
+
*
|
|
238
|
+
* @important Structured Output Limitation (Gemini Models Only)
|
|
239
|
+
* Google Gemini models on Vertex AI cannot combine function calling (tools) with
|
|
240
|
+
* structured output (JSON schema). When using schemas, you MUST set disableTools: true.
|
|
241
|
+
*
|
|
242
|
+
* Error without disableTools:
|
|
243
|
+
* "Function calling with a response mime type: 'application/json' is unsupported"
|
|
244
|
+
*
|
|
245
|
+
* This limitation ONLY affects Gemini models. Anthropic Claude models via Vertex
|
|
246
|
+
* AI do NOT have this limitation and support both tools + schemas simultaneously.
|
|
247
|
+
*
|
|
248
|
+
* @example Gemini models with schemas
|
|
249
|
+
* ```typescript
|
|
250
|
+
* const provider = new GoogleVertexProvider("gemini-2.5-flash");
|
|
251
|
+
* const result = await provider.generate({
|
|
252
|
+
* input: { text: "Analyze data" },
|
|
253
|
+
* schema: MySchema,
|
|
254
|
+
* output: { format: "json" },
|
|
255
|
+
* disableTools: true // Required for Gemini models
|
|
256
|
+
* });
|
|
257
|
+
* ```
|
|
258
|
+
*
|
|
259
|
+
* @example Claude models (no limitation)
|
|
260
|
+
* ```typescript
|
|
261
|
+
* const provider = new GoogleVertexProvider("claude-3-5-sonnet-20241022");
|
|
262
|
+
* const result = await provider.generate({
|
|
263
|
+
* input: { text: "Analyze data" },
|
|
264
|
+
* schema: MySchema,
|
|
265
|
+
* output: { format: "json" }
|
|
266
|
+
* // No disableTools needed - Claude supports both
|
|
267
|
+
* });
|
|
268
|
+
* ```
|
|
269
|
+
*
|
|
270
|
+
* @note Gemini 3 Pro Preview (November 2025) will support combining tools + schemas
|
|
271
|
+
* @see https://cloud.google.com/vertex-ai/docs/generative-ai/learn/models
|
|
237
272
|
*/
|
|
238
273
|
export class GoogleVertexProvider extends BaseProvider {
|
|
239
274
|
projectId;
|
|
@@ -1363,11 +1398,14 @@ export class GoogleVertexProvider extends BaseProvider {
|
|
|
1363
1398
|
getModelSuggestions(requestedModel) {
|
|
1364
1399
|
const availableModels = {
|
|
1365
1400
|
google: [
|
|
1401
|
+
"gemini-3-pro-preview-11-2025",
|
|
1402
|
+
"gemini-3-pro-latest",
|
|
1366
1403
|
"gemini-3-pro-preview",
|
|
1367
1404
|
"gemini-2.5-pro",
|
|
1368
1405
|
"gemini-2.5-flash",
|
|
1369
1406
|
"gemini-2.5-flash-lite",
|
|
1370
1407
|
"gemini-2.0-flash-001",
|
|
1408
|
+
"gemini-2.0-flash-lite",
|
|
1371
1409
|
"gemini-1.5-pro",
|
|
1372
1410
|
"gemini-1.5-flash",
|
|
1373
1411
|
],
|
|
@@ -34,9 +34,58 @@ export type GenerateOptions = {
|
|
|
34
34
|
temperature?: number;
|
|
35
35
|
maxTokens?: number;
|
|
36
36
|
systemPrompt?: string;
|
|
37
|
+
/**
|
|
38
|
+
* Zod schema for structured output validation
|
|
39
|
+
*
|
|
40
|
+
* @important Google Gemini Limitation
|
|
41
|
+
* Google Vertex AI and Google AI Studio cannot combine function calling with
|
|
42
|
+
* structured output. You MUST use `disableTools: true` when using schemas with
|
|
43
|
+
* Google providers.
|
|
44
|
+
*
|
|
45
|
+
* Error without disableTools: "Function calling with a response mime type:
|
|
46
|
+
* 'application/json' is unsupported"
|
|
47
|
+
*
|
|
48
|
+
* This is a documented Google API limitation, not a NeuroLink bug.
|
|
49
|
+
* All frameworks (LangChain, Vercel AI SDK, Agno, Instructor) use this approach.
|
|
50
|
+
*
|
|
51
|
+
* @example
|
|
52
|
+
* ```typescript
|
|
53
|
+
* // ✅ Correct for Google providers
|
|
54
|
+
* const result = await neurolink.generate({
|
|
55
|
+
* schema: MySchema,
|
|
56
|
+
* provider: "vertex",
|
|
57
|
+
* disableTools: true // Required for Google
|
|
58
|
+
* });
|
|
59
|
+
*
|
|
60
|
+
* // ✅ No restriction for other providers
|
|
61
|
+
* const result = await neurolink.generate({
|
|
62
|
+
* schema: MySchema,
|
|
63
|
+
* provider: "openai" // Works without disableTools
|
|
64
|
+
* });
|
|
65
|
+
* ```
|
|
66
|
+
*
|
|
67
|
+
* @see https://ai.google.dev/gemini-api/docs/function-calling
|
|
68
|
+
*/
|
|
37
69
|
schema?: ValidationSchema;
|
|
38
70
|
tools?: Record<string, Tool>;
|
|
39
71
|
timeout?: number | string;
|
|
72
|
+
/**
|
|
73
|
+
* Disable tool execution (including built-in tools)
|
|
74
|
+
*
|
|
75
|
+
* @required For Google Gemini providers when using schemas
|
|
76
|
+
* Google Vertex AI and Google AI Studio require this flag when using
|
|
77
|
+
* structured output (schemas) due to Google API limitations.
|
|
78
|
+
*
|
|
79
|
+
* @example
|
|
80
|
+
* ```typescript
|
|
81
|
+
* // Required for Google providers with schemas
|
|
82
|
+
* await neurolink.generate({
|
|
83
|
+
* schema: MySchema,
|
|
84
|
+
* provider: "vertex",
|
|
85
|
+
* disableTools: true
|
|
86
|
+
* });
|
|
87
|
+
* ```
|
|
88
|
+
*/
|
|
40
89
|
disableTools?: boolean;
|
|
41
90
|
enableEvaluation?: boolean;
|
|
42
91
|
enableAnalytics?: boolean;
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Centralized logic for building message arrays from TextGenerationOptions
|
|
4
4
|
* Enhanced with multimodal support for images
|
|
5
5
|
*/
|
|
6
|
-
import { CONVERSATION_INSTRUCTIONS } from "../config/conversationMemory.js";
|
|
6
|
+
import { CONVERSATION_INSTRUCTIONS, STRUCTURED_OUTPUT_INSTRUCTIONS, } from "../config/conversationMemory.js";
|
|
7
7
|
import { ProviderImageAdapter, MultimodalLogger, } from "../adapters/providerImageAdapter.js";
|
|
8
8
|
import { logger } from "./logger.js";
|
|
9
9
|
import { FileDetector } from "./fileDetector.js";
|
|
@@ -199,6 +199,15 @@ function formatCSVMetadata(metadata) {
|
|
|
199
199
|
}
|
|
200
200
|
return parts.length > 0 ? `**Metadata**: ${parts.join(" | ")}` : "";
|
|
201
201
|
}
|
|
202
|
+
/**
|
|
203
|
+
* Check if structured output mode should be enabled
|
|
204
|
+
* Structured output is used when a schema is provided with json/structured format
|
|
205
|
+
*/
|
|
206
|
+
function shouldUseStructuredOutput(options) {
|
|
207
|
+
return (!!options.schema &&
|
|
208
|
+
(options.output?.format === "json" ||
|
|
209
|
+
options.output?.format === "structured"));
|
|
210
|
+
}
|
|
202
211
|
/**
|
|
203
212
|
* Build a properly formatted message array for AI providers
|
|
204
213
|
* Combines system prompt, conversation history, and current user prompt
|
|
@@ -215,6 +224,10 @@ export async function buildMessagesArray(options) {
|
|
|
215
224
|
if (hasConversationHistory) {
|
|
216
225
|
systemPrompt = `${systemPrompt.trim()}${CONVERSATION_INSTRUCTIONS}`;
|
|
217
226
|
}
|
|
227
|
+
// Add structured output instructions when schema is provided with json/structured format
|
|
228
|
+
if (shouldUseStructuredOutput(options)) {
|
|
229
|
+
systemPrompt = `${systemPrompt.trim()}${STRUCTURED_OUTPUT_INSTRUCTIONS}`;
|
|
230
|
+
}
|
|
218
231
|
// Add system message if we have one
|
|
219
232
|
if (systemPrompt.trim()) {
|
|
220
233
|
messages.push({
|
|
@@ -473,6 +486,10 @@ export async function buildMultimodalMessagesArray(options, provider, model) {
|
|
|
473
486
|
if (hasConversationHistory) {
|
|
474
487
|
systemPrompt = `${systemPrompt.trim()}${CONVERSATION_INSTRUCTIONS}`;
|
|
475
488
|
}
|
|
489
|
+
// Add structured output instructions when schema is provided with json/structured format
|
|
490
|
+
if (shouldUseStructuredOutput(options)) {
|
|
491
|
+
systemPrompt = `${systemPrompt.trim()}${STRUCTURED_OUTPUT_INSTRUCTIONS}`;
|
|
492
|
+
}
|
|
476
493
|
// Add file handling guidance when multimodal files are present
|
|
477
494
|
const hasCSVFiles = (options.input.csvFiles && options.input.csvFiles.length > 0) ||
|
|
478
495
|
(options.input.files &&
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@juspay/neurolink",
|
|
3
|
-
"version": "8.5.
|
|
3
|
+
"version": "8.5.1",
|
|
4
4
|
"description": "Universal AI Development Platform with working MCP integration, multi-provider support, and professional CLI. Built-in tools operational, 58+ external MCP servers discoverable. Connect to filesystem, GitHub, database operations, and more. Build, test, and deploy AI applications with 9 major providers: OpenAI, Anthropic, Google AI, AWS Bedrock, Azure, Hugging Face, Ollama, and Mistral AI.",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Juspay Technologies",
|