@juspay/neurolink 8.19.0 → 8.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/adapters/providerImageAdapter.d.ts +12 -0
- package/dist/adapters/providerImageAdapter.js +30 -3
- package/dist/config/conversationMemory.d.ts +2 -1
- package/dist/config/conversationMemory.js +15 -7
- package/dist/core/baseProvider.js +23 -13
- package/dist/core/modules/GenerationHandler.d.ts +5 -0
- package/dist/core/modules/GenerationHandler.js +56 -9
- package/dist/lib/adapters/providerImageAdapter.d.ts +12 -0
- package/dist/lib/adapters/providerImageAdapter.js +30 -3
- package/dist/lib/config/conversationMemory.d.ts +2 -1
- package/dist/lib/config/conversationMemory.js +15 -7
- package/dist/lib/core/baseProvider.js +23 -13
- package/dist/lib/core/modules/GenerationHandler.d.ts +5 -0
- package/dist/lib/core/modules/GenerationHandler.js +56 -9
- package/dist/lib/mcp/servers/agent/directToolsServer.js +5 -0
- package/dist/lib/mcp/toolRegistry.js +5 -0
- package/dist/lib/utils/fileDetector.d.ts +25 -0
- package/dist/lib/utils/fileDetector.js +433 -10
- package/dist/lib/utils/messageBuilder.js +6 -2
- package/dist/mcp/servers/agent/directToolsServer.js +5 -0
- package/dist/mcp/toolRegistry.js +5 -0
- package/dist/utils/fileDetector.d.ts +25 -0
- package/dist/utils/fileDetector.js +433 -10
- package/dist/utils/messageBuilder.js +6 -2
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
## [8.19.1](https://github.com/juspay/neurolink/compare/v8.19.0...v8.19.1) (2025-12-20)
|
|
2
|
+
|
|
3
|
+
### Bug Fixes
|
|
4
|
+
|
|
5
|
+
- **(files):** comprehensive extension-less file detection with fallback parsing (FD-018) ([7e9dbc7](https://github.com/juspay/neurolink/commit/7e9dbc78df48f6df051c7845824977f360f8feee))
|
|
6
|
+
|
|
1
7
|
## [8.19.0](https://github.com/juspay/neurolink/compare/v8.18.0...v8.19.0) (2025-12-18)
|
|
2
8
|
|
|
3
9
|
### Features
|
|
@@ -60,4 +60,16 @@ export declare class ProviderImageAdapter {
|
|
|
60
60
|
* Get all vision-capable providers
|
|
61
61
|
*/
|
|
62
62
|
static getVisionProviders(): string[];
|
|
63
|
+
/**
|
|
64
|
+
* Count total "images" in a message (actual images + PDF pages)
|
|
65
|
+
* PDF pages count toward image limits for providers
|
|
66
|
+
*/
|
|
67
|
+
static countImagesInMessage(images: Array<Buffer | string>, pdfPages?: number | null): number;
|
|
68
|
+
/**
|
|
69
|
+
* Extract page count from PDF metadata array
|
|
70
|
+
* Returns total pages across all PDFs
|
|
71
|
+
*/
|
|
72
|
+
static countImagesInPages(pdfMetadataArray: Array<{
|
|
73
|
+
pageCount?: number | null;
|
|
74
|
+
}> | undefined): number;
|
|
63
75
|
}
|
|
@@ -416,13 +416,19 @@ export class ProviderImageAdapter {
|
|
|
416
416
|
adaptedPayload = this.formatForOpenAI(text, images);
|
|
417
417
|
break;
|
|
418
418
|
case "litellm":
|
|
419
|
-
|
|
419
|
+
// LiteLLM uses same format as OpenAI but validate with litellm provider name
|
|
420
|
+
this.validateImageCount(images.length, "litellm");
|
|
421
|
+
adaptedPayload = this.formatForOpenAI(text, images, true);
|
|
420
422
|
break;
|
|
421
423
|
case "mistral":
|
|
422
|
-
|
|
424
|
+
// Mistral uses same format as OpenAI but validate with mistral provider name
|
|
425
|
+
this.validateImageCount(images.length, "mistral");
|
|
426
|
+
adaptedPayload = this.formatForOpenAI(text, images, true);
|
|
423
427
|
break;
|
|
424
428
|
case "bedrock":
|
|
425
|
-
|
|
429
|
+
// Bedrock uses same format as Anthropic but validate with bedrock provider name
|
|
430
|
+
this.validateImageCount(images.length, "bedrock");
|
|
431
|
+
adaptedPayload = this.formatForAnthropic(text, images, true);
|
|
426
432
|
break;
|
|
427
433
|
default:
|
|
428
434
|
throw new Error(`Vision not supported for provider: ${provider}`);
|
|
@@ -666,4 +672,25 @@ export class ProviderImageAdapter {
|
|
|
666
672
|
static getVisionProviders() {
|
|
667
673
|
return Object.keys(VISION_CAPABILITIES);
|
|
668
674
|
}
|
|
675
|
+
/**
|
|
676
|
+
* Count total "images" in a message (actual images + PDF pages)
|
|
677
|
+
* PDF pages count toward image limits for providers
|
|
678
|
+
*/
|
|
679
|
+
static countImagesInMessage(images, pdfPages) {
|
|
680
|
+
const imageCount = images?.length || 0;
|
|
681
|
+
const pageCount = pdfPages ?? 0;
|
|
682
|
+
return imageCount + pageCount;
|
|
683
|
+
}
|
|
684
|
+
/**
|
|
685
|
+
* Extract page count from PDF metadata array
|
|
686
|
+
* Returns total pages across all PDFs
|
|
687
|
+
*/
|
|
688
|
+
static countImagesInPages(pdfMetadataArray) {
|
|
689
|
+
if (!pdfMetadataArray || pdfMetadataArray.length === 0) {
|
|
690
|
+
return 0;
|
|
691
|
+
}
|
|
692
|
+
return pdfMetadataArray.reduce((total, pdf) => {
|
|
693
|
+
return total + (pdf.pageCount ?? 0);
|
|
694
|
+
}, 0);
|
|
695
|
+
}
|
|
669
696
|
}
|
|
@@ -24,8 +24,9 @@ export declare const CONVERSATION_INSTRUCTIONS = "\n\nIMPORTANT: You are continu
|
|
|
24
24
|
* Structured output instructions for JSON/structured output mode
|
|
25
25
|
* Used to ensure AI providers output only valid JSON without conversational filler
|
|
26
26
|
* This addresses the issue where models add text like "Excellent!" before JSON output
|
|
27
|
+
* and the case where tools are used but final output must still be pure JSON
|
|
27
28
|
*/
|
|
28
|
-
export declare const STRUCTURED_OUTPUT_INSTRUCTIONS = "\
|
|
29
|
+
export declare const STRUCTURED_OUTPUT_INSTRUCTIONS = "\nOutput ONLY valid JSON. No markdown, text, or decorations\u2014ever.\n\nFORBIDDEN: markdown code blocks, text before/after JSON, explanations, preambles, summaries, conversational text about tools.\n\nREQUIRED: response starts with { and ends with }, valid JSON only, no additional characters.\n\nIF YOU CALLED TOOLS: Incorporate data directly into the JSON structure. Do NOT explain what you did.\n\nWRONG: ```json\n{\"field\": \"value\"}\n```\nWRONG: Based on the data, here's the result: {\"field\": \"value\"}\nCORRECT: {\"field\": \"value\"}\n\nYour entire response = raw JSON object. Nothing else.";
|
|
29
30
|
/**
|
|
30
31
|
* Get default configuration values for conversation memory
|
|
31
32
|
* Reads environment variables when called (not at module load time)
|
|
@@ -30,16 +30,24 @@ Always reference and build upon this conversation history when relevant. If the
|
|
|
30
30
|
* Structured output instructions for JSON/structured output mode
|
|
31
31
|
* Used to ensure AI providers output only valid JSON without conversational filler
|
|
32
32
|
* This addresses the issue where models add text like "Excellent!" before JSON output
|
|
33
|
+
* and the case where tools are used but final output must still be pure JSON
|
|
33
34
|
*/
|
|
34
35
|
export const STRUCTURED_OUTPUT_INSTRUCTIONS = `
|
|
36
|
+
Output ONLY valid JSON. No markdown, text, or decorations—ever.
|
|
35
37
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
38
|
+
FORBIDDEN: markdown code blocks, text before/after JSON, explanations, preambles, summaries, conversational text about tools.
|
|
39
|
+
|
|
40
|
+
REQUIRED: response starts with { and ends with }, valid JSON only, no additional characters.
|
|
41
|
+
|
|
42
|
+
IF YOU CALLED TOOLS: Incorporate data directly into the JSON structure. Do NOT explain what you did.
|
|
43
|
+
|
|
44
|
+
WRONG: \`\`\`json
|
|
45
|
+
{"field": "value"}
|
|
46
|
+
\`\`\`
|
|
47
|
+
WRONG: Based on the data, here's the result: {"field": "value"}
|
|
48
|
+
CORRECT: {"field": "value"}
|
|
49
|
+
|
|
50
|
+
Your entire response = raw JSON object. Nothing else.`;
|
|
43
51
|
/**
|
|
44
52
|
* Get default configuration values for conversation memory
|
|
45
53
|
* Reads environment variables when called (not at module load time)
|
|
@@ -327,18 +327,21 @@ export class BaseProvider {
|
|
|
327
327
|
// This is optimal for simple read-aloud scenarios
|
|
328
328
|
if (options.tts?.enabled && !options.tts?.useAiResponse) {
|
|
329
329
|
const textToSynthesize = options.prompt ?? options.input?.text ?? "";
|
|
330
|
-
|
|
330
|
+
// Build base result structure - common to both paths
|
|
331
331
|
const baseResult = {
|
|
332
332
|
content: textToSynthesize,
|
|
333
|
-
audio: ttsResult,
|
|
334
333
|
provider: options.provider ?? this.providerName,
|
|
335
334
|
model: this.modelName,
|
|
336
|
-
usage: {
|
|
337
|
-
input: 0,
|
|
338
|
-
output: 0,
|
|
339
|
-
total: 0,
|
|
340
|
-
},
|
|
335
|
+
usage: { input: 0, output: 0, total: 0 },
|
|
341
336
|
};
|
|
337
|
+
try {
|
|
338
|
+
const ttsResult = await TTSProcessor.synthesize(textToSynthesize, options.provider ?? this.providerName, options.tts);
|
|
339
|
+
baseResult.audio = ttsResult;
|
|
340
|
+
}
|
|
341
|
+
catch (ttsError) {
|
|
342
|
+
logger.error(`TTS synthesis failed in Mode 1 (direct input synthesis):`, ttsError);
|
|
343
|
+
// baseResult remains without audio - graceful degradation
|
|
344
|
+
}
|
|
342
345
|
// Call enhanceResult for consistency - enables analytics/evaluation for TTS-only requests
|
|
343
346
|
return await this.enhanceResult(baseResult, options, startTime);
|
|
344
347
|
}
|
|
@@ -359,12 +362,19 @@ export class BaseProvider {
|
|
|
359
362
|
const provider = options.provider ?? this.providerName;
|
|
360
363
|
// Validate AI response and provider before synthesis
|
|
361
364
|
if (aiResponse && provider) {
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
365
|
+
try {
|
|
366
|
+
const ttsResult = await TTSProcessor.synthesize(aiResponse, provider, options.tts);
|
|
367
|
+
// Add audio to enhanced result (TTSProcessor already includes latency in metadata)
|
|
368
|
+
enhancedResult = {
|
|
369
|
+
...enhancedResult,
|
|
370
|
+
audio: ttsResult,
|
|
371
|
+
};
|
|
372
|
+
}
|
|
373
|
+
catch (ttsError) {
|
|
374
|
+
// Log TTS error but continue with text-only result
|
|
375
|
+
logger.error(`TTS synthesis failed in Mode 2 (AI response synthesis):`, ttsError);
|
|
376
|
+
// enhancedResult remains unchanged (no audio field added)
|
|
377
|
+
}
|
|
368
378
|
}
|
|
369
379
|
else {
|
|
370
380
|
logger.warn(`TTS synthesis skipped despite being enabled`, {
|
|
@@ -29,6 +29,11 @@ export declare class GenerationHandler {
|
|
|
29
29
|
functionId?: string;
|
|
30
30
|
metadata?: Record<string, string | number | boolean>;
|
|
31
31
|
} | undefined, handleToolStorageFn: (toolCalls: unknown[], toolResults: unknown[], options: TextGenerationOptions, timestamp: Date) => Promise<void>);
|
|
32
|
+
/**
|
|
33
|
+
* Helper method to call generateText with optional structured output
|
|
34
|
+
* @private
|
|
35
|
+
*/
|
|
36
|
+
private callGenerateText;
|
|
32
37
|
/**
|
|
33
38
|
* Execute the generation with AI SDK
|
|
34
39
|
*/
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
*
|
|
13
13
|
* @module core/modules/GenerationHandler
|
|
14
14
|
*/
|
|
15
|
-
import { generateText, Output } from "ai";
|
|
15
|
+
import { generateText, Output, NoObjectGeneratedError } from "ai";
|
|
16
16
|
import { logger } from "../../utils/logger.js";
|
|
17
17
|
import { DEFAULT_MAX_STEPS } from "../constants.js";
|
|
18
18
|
/**
|
|
@@ -32,11 +32,12 @@ export class GenerationHandler {
|
|
|
32
32
|
this.handleToolStorageFn = handleToolStorageFn;
|
|
33
33
|
}
|
|
34
34
|
/**
|
|
35
|
-
*
|
|
35
|
+
* Helper method to call generateText with optional structured output
|
|
36
|
+
* @private
|
|
36
37
|
*/
|
|
37
|
-
async
|
|
38
|
-
const
|
|
39
|
-
|
|
38
|
+
async callGenerateText(model, messages, tools, options, shouldUseTools, includeStructuredOutput) {
|
|
39
|
+
const useStructuredOutput = includeStructuredOutput &&
|
|
40
|
+
!!options.schema &&
|
|
40
41
|
(options.output?.format === "json" ||
|
|
41
42
|
options.output?.format === "structured");
|
|
42
43
|
return await generateText({
|
|
@@ -64,6 +65,34 @@ export class GenerationHandler {
|
|
|
64
65
|
},
|
|
65
66
|
});
|
|
66
67
|
}
|
|
68
|
+
/**
|
|
69
|
+
* Execute the generation with AI SDK
|
|
70
|
+
*/
|
|
71
|
+
async executeGeneration(model, messages, tools, options) {
|
|
72
|
+
const shouldUseTools = !options.disableTools && this.supportsToolsFn();
|
|
73
|
+
const useStructuredOutput = !!options.schema &&
|
|
74
|
+
(options.output?.format === "json" ||
|
|
75
|
+
options.output?.format === "structured");
|
|
76
|
+
try {
|
|
77
|
+
return await this.callGenerateText(model, messages, tools, options, shouldUseTools, true);
|
|
78
|
+
}
|
|
79
|
+
catch (error) {
|
|
80
|
+
// If NoObjectGeneratedError is thrown when using schema + tools together,
|
|
81
|
+
// fall back to generating without experimental_output and extract JSON manually
|
|
82
|
+
if (error instanceof NoObjectGeneratedError && useStructuredOutput) {
|
|
83
|
+
logger.debug("[GenerationHandler] NoObjectGeneratedError caught - falling back to manual JSON extraction", {
|
|
84
|
+
provider: this.providerName,
|
|
85
|
+
model: this.modelName,
|
|
86
|
+
error: error.message,
|
|
87
|
+
});
|
|
88
|
+
// Retry without experimental_output - the formatEnhancedResult method
|
|
89
|
+
// will extract JSON from the text response
|
|
90
|
+
return await this.callGenerateText(model, messages, tools, options, shouldUseTools, false);
|
|
91
|
+
}
|
|
92
|
+
// Re-throw other errors
|
|
93
|
+
throw error;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
67
96
|
/**
|
|
68
97
|
* Log generation completion information
|
|
69
98
|
*/
|
|
@@ -164,11 +193,29 @@ export class GenerationHandler {
|
|
|
164
193
|
options.output?.format === "structured");
|
|
165
194
|
let content;
|
|
166
195
|
if (useStructuredOutput) {
|
|
167
|
-
|
|
168
|
-
|
|
196
|
+
try {
|
|
197
|
+
const experimentalOutput = generateResult.experimental_output;
|
|
198
|
+
if (experimentalOutput !== undefined) {
|
|
199
|
+
content = JSON.stringify(experimentalOutput);
|
|
200
|
+
}
|
|
201
|
+
else {
|
|
202
|
+
// Fall back to text parsing
|
|
203
|
+
const rawText = generateResult.text || "";
|
|
204
|
+
const strippedText = rawText
|
|
205
|
+
.replace(/^```(?:json)?\s*\n?/i, "")
|
|
206
|
+
.replace(/\n?```\s*$/i, "")
|
|
207
|
+
.trim();
|
|
208
|
+
content = strippedText;
|
|
209
|
+
}
|
|
169
210
|
}
|
|
170
|
-
|
|
171
|
-
|
|
211
|
+
catch (outputError) {
|
|
212
|
+
// experimental_output is a getter that can throw NoObjectGeneratedError
|
|
213
|
+
// Fall back to text parsing when structured output fails
|
|
214
|
+
logger.debug("[GenerationHandler] experimental_output threw, falling back to text parsing", {
|
|
215
|
+
error: outputError instanceof Error
|
|
216
|
+
? outputError.message
|
|
217
|
+
: String(outputError),
|
|
218
|
+
});
|
|
172
219
|
const rawText = generateResult.text || "";
|
|
173
220
|
const strippedText = rawText
|
|
174
221
|
.replace(/^```(?:json)?\s*\n?/i, "")
|
|
@@ -60,4 +60,16 @@ export declare class ProviderImageAdapter {
|
|
|
60
60
|
* Get all vision-capable providers
|
|
61
61
|
*/
|
|
62
62
|
static getVisionProviders(): string[];
|
|
63
|
+
/**
|
|
64
|
+
* Count total "images" in a message (actual images + PDF pages)
|
|
65
|
+
* PDF pages count toward image limits for providers
|
|
66
|
+
*/
|
|
67
|
+
static countImagesInMessage(images: Array<Buffer | string>, pdfPages?: number | null): number;
|
|
68
|
+
/**
|
|
69
|
+
* Extract page count from PDF metadata array
|
|
70
|
+
* Returns total pages across all PDFs
|
|
71
|
+
*/
|
|
72
|
+
static countImagesInPages(pdfMetadataArray: Array<{
|
|
73
|
+
pageCount?: number | null;
|
|
74
|
+
}> | undefined): number;
|
|
63
75
|
}
|
|
@@ -416,13 +416,19 @@ export class ProviderImageAdapter {
|
|
|
416
416
|
adaptedPayload = this.formatForOpenAI(text, images);
|
|
417
417
|
break;
|
|
418
418
|
case "litellm":
|
|
419
|
-
|
|
419
|
+
// LiteLLM uses same format as OpenAI but validate with litellm provider name
|
|
420
|
+
this.validateImageCount(images.length, "litellm");
|
|
421
|
+
adaptedPayload = this.formatForOpenAI(text, images, true);
|
|
420
422
|
break;
|
|
421
423
|
case "mistral":
|
|
422
|
-
|
|
424
|
+
// Mistral uses same format as OpenAI but validate with mistral provider name
|
|
425
|
+
this.validateImageCount(images.length, "mistral");
|
|
426
|
+
adaptedPayload = this.formatForOpenAI(text, images, true);
|
|
423
427
|
break;
|
|
424
428
|
case "bedrock":
|
|
425
|
-
|
|
429
|
+
// Bedrock uses same format as Anthropic but validate with bedrock provider name
|
|
430
|
+
this.validateImageCount(images.length, "bedrock");
|
|
431
|
+
adaptedPayload = this.formatForAnthropic(text, images, true);
|
|
426
432
|
break;
|
|
427
433
|
default:
|
|
428
434
|
throw new Error(`Vision not supported for provider: ${provider}`);
|
|
@@ -666,5 +672,26 @@ export class ProviderImageAdapter {
|
|
|
666
672
|
static getVisionProviders() {
|
|
667
673
|
return Object.keys(VISION_CAPABILITIES);
|
|
668
674
|
}
|
|
675
|
+
/**
|
|
676
|
+
* Count total "images" in a message (actual images + PDF pages)
|
|
677
|
+
* PDF pages count toward image limits for providers
|
|
678
|
+
*/
|
|
679
|
+
static countImagesInMessage(images, pdfPages) {
|
|
680
|
+
const imageCount = images?.length || 0;
|
|
681
|
+
const pageCount = pdfPages ?? 0;
|
|
682
|
+
return imageCount + pageCount;
|
|
683
|
+
}
|
|
684
|
+
/**
|
|
685
|
+
* Extract page count from PDF metadata array
|
|
686
|
+
* Returns total pages across all PDFs
|
|
687
|
+
*/
|
|
688
|
+
static countImagesInPages(pdfMetadataArray) {
|
|
689
|
+
if (!pdfMetadataArray || pdfMetadataArray.length === 0) {
|
|
690
|
+
return 0;
|
|
691
|
+
}
|
|
692
|
+
return pdfMetadataArray.reduce((total, pdf) => {
|
|
693
|
+
return total + (pdf.pageCount ?? 0);
|
|
694
|
+
}, 0);
|
|
695
|
+
}
|
|
669
696
|
}
|
|
670
697
|
//# sourceMappingURL=providerImageAdapter.js.map
|
|
@@ -24,8 +24,9 @@ export declare const CONVERSATION_INSTRUCTIONS = "\n\nIMPORTANT: You are continu
|
|
|
24
24
|
* Structured output instructions for JSON/structured output mode
|
|
25
25
|
* Used to ensure AI providers output only valid JSON without conversational filler
|
|
26
26
|
* This addresses the issue where models add text like "Excellent!" before JSON output
|
|
27
|
+
* and the case where tools are used but final output must still be pure JSON
|
|
27
28
|
*/
|
|
28
|
-
export declare const STRUCTURED_OUTPUT_INSTRUCTIONS = "\
|
|
29
|
+
export declare const STRUCTURED_OUTPUT_INSTRUCTIONS = "\nOutput ONLY valid JSON. No markdown, text, or decorations\u2014ever.\n\nFORBIDDEN: markdown code blocks, text before/after JSON, explanations, preambles, summaries, conversational text about tools.\n\nREQUIRED: response starts with { and ends with }, valid JSON only, no additional characters.\n\nIF YOU CALLED TOOLS: Incorporate data directly into the JSON structure. Do NOT explain what you did.\n\nWRONG: ```json\n{\"field\": \"value\"}\n```\nWRONG: Based on the data, here's the result: {\"field\": \"value\"}\nCORRECT: {\"field\": \"value\"}\n\nYour entire response = raw JSON object. Nothing else.";
|
|
29
30
|
/**
|
|
30
31
|
* Get default configuration values for conversation memory
|
|
31
32
|
* Reads environment variables when called (not at module load time)
|
|
@@ -30,16 +30,24 @@ Always reference and build upon this conversation history when relevant. If the
|
|
|
30
30
|
* Structured output instructions for JSON/structured output mode
|
|
31
31
|
* Used to ensure AI providers output only valid JSON without conversational filler
|
|
32
32
|
* This addresses the issue where models add text like "Excellent!" before JSON output
|
|
33
|
+
* and the case where tools are used but final output must still be pure JSON
|
|
33
34
|
*/
|
|
34
35
|
export const STRUCTURED_OUTPUT_INSTRUCTIONS = `
|
|
36
|
+
Output ONLY valid JSON. No markdown, text, or decorations—ever.
|
|
35
37
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
38
|
+
FORBIDDEN: markdown code blocks, text before/after JSON, explanations, preambles, summaries, conversational text about tools.
|
|
39
|
+
|
|
40
|
+
REQUIRED: response starts with { and ends with }, valid JSON only, no additional characters.
|
|
41
|
+
|
|
42
|
+
IF YOU CALLED TOOLS: Incorporate data directly into the JSON structure. Do NOT explain what you did.
|
|
43
|
+
|
|
44
|
+
WRONG: \`\`\`json
|
|
45
|
+
{"field": "value"}
|
|
46
|
+
\`\`\`
|
|
47
|
+
WRONG: Based on the data, here's the result: {"field": "value"}
|
|
48
|
+
CORRECT: {"field": "value"}
|
|
49
|
+
|
|
50
|
+
Your entire response = raw JSON object. Nothing else.`;
|
|
43
51
|
/**
|
|
44
52
|
* Get default configuration values for conversation memory
|
|
45
53
|
* Reads environment variables when called (not at module load time)
|
|
@@ -327,18 +327,21 @@ export class BaseProvider {
|
|
|
327
327
|
// This is optimal for simple read-aloud scenarios
|
|
328
328
|
if (options.tts?.enabled && !options.tts?.useAiResponse) {
|
|
329
329
|
const textToSynthesize = options.prompt ?? options.input?.text ?? "";
|
|
330
|
-
|
|
330
|
+
// Build base result structure - common to both paths
|
|
331
331
|
const baseResult = {
|
|
332
332
|
content: textToSynthesize,
|
|
333
|
-
audio: ttsResult,
|
|
334
333
|
provider: options.provider ?? this.providerName,
|
|
335
334
|
model: this.modelName,
|
|
336
|
-
usage: {
|
|
337
|
-
input: 0,
|
|
338
|
-
output: 0,
|
|
339
|
-
total: 0,
|
|
340
|
-
},
|
|
335
|
+
usage: { input: 0, output: 0, total: 0 },
|
|
341
336
|
};
|
|
337
|
+
try {
|
|
338
|
+
const ttsResult = await TTSProcessor.synthesize(textToSynthesize, options.provider ?? this.providerName, options.tts);
|
|
339
|
+
baseResult.audio = ttsResult;
|
|
340
|
+
}
|
|
341
|
+
catch (ttsError) {
|
|
342
|
+
logger.error(`TTS synthesis failed in Mode 1 (direct input synthesis):`, ttsError);
|
|
343
|
+
// baseResult remains without audio - graceful degradation
|
|
344
|
+
}
|
|
342
345
|
// Call enhanceResult for consistency - enables analytics/evaluation for TTS-only requests
|
|
343
346
|
return await this.enhanceResult(baseResult, options, startTime);
|
|
344
347
|
}
|
|
@@ -359,12 +362,19 @@ export class BaseProvider {
|
|
|
359
362
|
const provider = options.provider ?? this.providerName;
|
|
360
363
|
// Validate AI response and provider before synthesis
|
|
361
364
|
if (aiResponse && provider) {
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
365
|
+
try {
|
|
366
|
+
const ttsResult = await TTSProcessor.synthesize(aiResponse, provider, options.tts);
|
|
367
|
+
// Add audio to enhanced result (TTSProcessor already includes latency in metadata)
|
|
368
|
+
enhancedResult = {
|
|
369
|
+
...enhancedResult,
|
|
370
|
+
audio: ttsResult,
|
|
371
|
+
};
|
|
372
|
+
}
|
|
373
|
+
catch (ttsError) {
|
|
374
|
+
// Log TTS error but continue with text-only result
|
|
375
|
+
logger.error(`TTS synthesis failed in Mode 2 (AI response synthesis):`, ttsError);
|
|
376
|
+
// enhancedResult remains unchanged (no audio field added)
|
|
377
|
+
}
|
|
368
378
|
}
|
|
369
379
|
else {
|
|
370
380
|
logger.warn(`TTS synthesis skipped despite being enabled`, {
|
|
@@ -29,6 +29,11 @@ export declare class GenerationHandler {
|
|
|
29
29
|
functionId?: string;
|
|
30
30
|
metadata?: Record<string, string | number | boolean>;
|
|
31
31
|
} | undefined, handleToolStorageFn: (toolCalls: unknown[], toolResults: unknown[], options: TextGenerationOptions, timestamp: Date) => Promise<void>);
|
|
32
|
+
/**
|
|
33
|
+
* Helper method to call generateText with optional structured output
|
|
34
|
+
* @private
|
|
35
|
+
*/
|
|
36
|
+
private callGenerateText;
|
|
32
37
|
/**
|
|
33
38
|
* Execute the generation with AI SDK
|
|
34
39
|
*/
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
*
|
|
13
13
|
* @module core/modules/GenerationHandler
|
|
14
14
|
*/
|
|
15
|
-
import { generateText, Output } from "ai";
|
|
15
|
+
import { generateText, Output, NoObjectGeneratedError } from "ai";
|
|
16
16
|
import { logger } from "../../utils/logger.js";
|
|
17
17
|
import { DEFAULT_MAX_STEPS } from "../constants.js";
|
|
18
18
|
/**
|
|
@@ -32,11 +32,12 @@ export class GenerationHandler {
|
|
|
32
32
|
this.handleToolStorageFn = handleToolStorageFn;
|
|
33
33
|
}
|
|
34
34
|
/**
|
|
35
|
-
*
|
|
35
|
+
* Helper method to call generateText with optional structured output
|
|
36
|
+
* @private
|
|
36
37
|
*/
|
|
37
|
-
async
|
|
38
|
-
const
|
|
39
|
-
|
|
38
|
+
async callGenerateText(model, messages, tools, options, shouldUseTools, includeStructuredOutput) {
|
|
39
|
+
const useStructuredOutput = includeStructuredOutput &&
|
|
40
|
+
!!options.schema &&
|
|
40
41
|
(options.output?.format === "json" ||
|
|
41
42
|
options.output?.format === "structured");
|
|
42
43
|
return await generateText({
|
|
@@ -64,6 +65,34 @@ export class GenerationHandler {
|
|
|
64
65
|
},
|
|
65
66
|
});
|
|
66
67
|
}
|
|
68
|
+
/**
|
|
69
|
+
* Execute the generation with AI SDK
|
|
70
|
+
*/
|
|
71
|
+
async executeGeneration(model, messages, tools, options) {
|
|
72
|
+
const shouldUseTools = !options.disableTools && this.supportsToolsFn();
|
|
73
|
+
const useStructuredOutput = !!options.schema &&
|
|
74
|
+
(options.output?.format === "json" ||
|
|
75
|
+
options.output?.format === "structured");
|
|
76
|
+
try {
|
|
77
|
+
return await this.callGenerateText(model, messages, tools, options, shouldUseTools, true);
|
|
78
|
+
}
|
|
79
|
+
catch (error) {
|
|
80
|
+
// If NoObjectGeneratedError is thrown when using schema + tools together,
|
|
81
|
+
// fall back to generating without experimental_output and extract JSON manually
|
|
82
|
+
if (error instanceof NoObjectGeneratedError && useStructuredOutput) {
|
|
83
|
+
logger.debug("[GenerationHandler] NoObjectGeneratedError caught - falling back to manual JSON extraction", {
|
|
84
|
+
provider: this.providerName,
|
|
85
|
+
model: this.modelName,
|
|
86
|
+
error: error.message,
|
|
87
|
+
});
|
|
88
|
+
// Retry without experimental_output - the formatEnhancedResult method
|
|
89
|
+
// will extract JSON from the text response
|
|
90
|
+
return await this.callGenerateText(model, messages, tools, options, shouldUseTools, false);
|
|
91
|
+
}
|
|
92
|
+
// Re-throw other errors
|
|
93
|
+
throw error;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
67
96
|
/**
|
|
68
97
|
* Log generation completion information
|
|
69
98
|
*/
|
|
@@ -164,11 +193,29 @@ export class GenerationHandler {
|
|
|
164
193
|
options.output?.format === "structured");
|
|
165
194
|
let content;
|
|
166
195
|
if (useStructuredOutput) {
|
|
167
|
-
|
|
168
|
-
|
|
196
|
+
try {
|
|
197
|
+
const experimentalOutput = generateResult.experimental_output;
|
|
198
|
+
if (experimentalOutput !== undefined) {
|
|
199
|
+
content = JSON.stringify(experimentalOutput);
|
|
200
|
+
}
|
|
201
|
+
else {
|
|
202
|
+
// Fall back to text parsing
|
|
203
|
+
const rawText = generateResult.text || "";
|
|
204
|
+
const strippedText = rawText
|
|
205
|
+
.replace(/^```(?:json)?\s*\n?/i, "")
|
|
206
|
+
.replace(/\n?```\s*$/i, "")
|
|
207
|
+
.trim();
|
|
208
|
+
content = strippedText;
|
|
209
|
+
}
|
|
169
210
|
}
|
|
170
|
-
|
|
171
|
-
|
|
211
|
+
catch (outputError) {
|
|
212
|
+
// experimental_output is a getter that can throw NoObjectGeneratedError
|
|
213
|
+
// Fall back to text parsing when structured output fails
|
|
214
|
+
logger.debug("[GenerationHandler] experimental_output threw, falling back to text parsing", {
|
|
215
|
+
error: outputError instanceof Error
|
|
216
|
+
? outputError.message
|
|
217
|
+
: String(outputError),
|
|
218
|
+
});
|
|
172
219
|
const rawText = generateResult.text || "";
|
|
173
220
|
const strippedText = rawText
|
|
174
221
|
.replace(/^```(?:json)?\s*\n?/i, "")
|
|
@@ -23,6 +23,11 @@ export const directToolsServer = createMCPServer({
|
|
|
23
23
|
*/
|
|
24
24
|
if (!shouldDisableBuiltinTools()) {
|
|
25
25
|
Object.entries(directAgentTools).forEach(([toolName, toolDef]) => {
|
|
26
|
+
// Skip undefined tools
|
|
27
|
+
if (!toolDef) {
|
|
28
|
+
logger.warn(`Skipping undefined tool during direct tools server registration: ${toolName}`);
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
26
31
|
// The toolDef is a Vercel AI SDK Tool object
|
|
27
32
|
// Extract properties from the Tool object
|
|
28
33
|
const toolSpec = toolDef._spec || toolDef;
|
|
@@ -47,6 +47,11 @@ export class MCPToolRegistry extends MCPRegistry {
|
|
|
47
47
|
registerDirectTools() {
|
|
48
48
|
registryLogger.debug("Auto-registering direct tools...");
|
|
49
49
|
for (const [toolName, toolDef] of Object.entries(directAgentTools)) {
|
|
50
|
+
// Skip undefined tools
|
|
51
|
+
if (!toolDef) {
|
|
52
|
+
registryLogger.warn(`Skipping undefined tool during registration: ${toolName}`);
|
|
53
|
+
continue;
|
|
54
|
+
}
|
|
50
55
|
const toolId = `direct.${toolName}`;
|
|
51
56
|
const toolInfo = {
|
|
52
57
|
name: toolName,
|
|
@@ -31,6 +31,31 @@ export declare class FileDetector {
|
|
|
31
31
|
* @returns Processed file result with type and content
|
|
32
32
|
*/
|
|
33
33
|
static detectAndProcess(input: FileInput, options?: FileDetectorOptions): Promise<FileProcessingResult>;
|
|
34
|
+
/**
|
|
35
|
+
* Try fallback parsing for a specific file type
|
|
36
|
+
* Used when file detection returns "unknown" but we want to try parsing anyway
|
|
37
|
+
*/
|
|
38
|
+
private static tryFallbackParsing;
|
|
39
|
+
/**
|
|
40
|
+
* Check if content is valid text (UTF-8, mostly printable)
|
|
41
|
+
*/
|
|
42
|
+
private static isValidText;
|
|
43
|
+
/**
|
|
44
|
+
* Guess the MIME type for text content based on content patterns
|
|
45
|
+
*/
|
|
46
|
+
private static guessTextMimeType;
|
|
47
|
+
/**
|
|
48
|
+
* Strict YAML detection for guessTextMimeType
|
|
49
|
+
* Similar to ContentHeuristicStrategy but requires at least 2 indicators
|
|
50
|
+
* to avoid false positives from simple key: value patterns
|
|
51
|
+
*/
|
|
52
|
+
private static looksLikeYAMLStrict;
|
|
53
|
+
/**
|
|
54
|
+
* Strict XML detection for guessTextMimeType
|
|
55
|
+
* Ensures content has proper XML declaration or valid tag structure with closing tags
|
|
56
|
+
* Prevents false positives from arbitrary content starting with <
|
|
57
|
+
*/
|
|
58
|
+
private static looksLikeXMLStrict;
|
|
34
59
|
/**
|
|
35
60
|
* Detect file type using multi-strategy approach
|
|
36
61
|
* Stops at first strategy with confidence >= threshold (default: 80%)
|