@juspay/neurolink 9.25.2 → 9.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/adapters/providerImageAdapter.d.ts +3 -27
- package/dist/adapters/providerImageAdapter.js +9 -199
- package/dist/agent/directTools.d.ts +35 -3
- package/dist/agent/directTools.js +122 -0
- package/dist/cli/commands/config.d.ts +6 -6
- package/dist/context/contextCompactor.d.ts +1 -2
- package/dist/context/contextCompactor.js +7 -1
- package/dist/context/prompts/summarizationPrompt.d.ts +3 -3
- package/dist/context/prompts/summarizationPrompt.js +16 -9
- package/dist/context/stages/structuredSummarizer.d.ts +2 -2
- package/dist/context/stages/structuredSummarizer.js +80 -30
- package/dist/lib/adapters/providerImageAdapter.d.ts +3 -27
- package/dist/lib/adapters/providerImageAdapter.js +9 -199
- package/dist/lib/agent/directTools.d.ts +33 -1
- package/dist/lib/agent/directTools.js +122 -0
- package/dist/lib/context/contextCompactor.d.ts +1 -2
- package/dist/lib/context/contextCompactor.js +7 -1
- package/dist/lib/context/prompts/summarizationPrompt.d.ts +3 -3
- package/dist/lib/context/prompts/summarizationPrompt.js +16 -9
- package/dist/lib/context/stages/structuredSummarizer.d.ts +2 -2
- package/dist/lib/context/stages/structuredSummarizer.js +80 -30
- package/dist/lib/mcp/servers/agent/directToolsServer.js +2 -0
- package/dist/lib/mcp/toolRegistry.d.ts +8 -0
- package/dist/lib/mcp/toolRegistry.js +20 -0
- package/dist/lib/neurolink.d.ts +10 -0
- package/dist/lib/neurolink.js +281 -17
- package/dist/lib/providers/googleAiStudio.js +13 -7
- package/dist/lib/types/configTypes.d.ts +3 -0
- package/dist/lib/types/contextTypes.d.ts +5 -2
- package/dist/lib/types/contextTypes.js +8 -8
- package/dist/lib/types/generateTypes.d.ts +25 -0
- package/dist/lib/types/modelTypes.d.ts +2 -2
- package/dist/lib/utils/messageBuilder.js +2 -0
- package/dist/lib/utils/modelAliasResolver.d.ts +17 -0
- package/dist/lib/utils/modelAliasResolver.js +55 -0
- package/dist/lib/utils/pdfProcessor.d.ts +1 -1
- package/dist/lib/utils/pdfProcessor.js +7 -7
- package/dist/lib/utils/toolUtils.d.ts +8 -0
- package/dist/lib/utils/toolUtils.js +15 -0
- package/dist/lib/workflow/config.d.ts +24 -24
- package/dist/mcp/servers/agent/directToolsServer.js +2 -0
- package/dist/mcp/toolRegistry.d.ts +8 -0
- package/dist/mcp/toolRegistry.js +20 -0
- package/dist/neurolink.d.ts +10 -0
- package/dist/neurolink.js +281 -17
- package/dist/providers/googleAiStudio.js +13 -7
- package/dist/server/utils/validation.d.ts +2 -2
- package/dist/types/configTypes.d.ts +3 -0
- package/dist/types/contextTypes.d.ts +5 -2
- package/dist/types/contextTypes.js +8 -8
- package/dist/types/generateTypes.d.ts +25 -0
- package/dist/utils/messageBuilder.js +2 -0
- package/dist/utils/modelAliasResolver.d.ts +17 -0
- package/dist/utils/modelAliasResolver.js +54 -0
- package/dist/utils/pdfProcessor.d.ts +1 -1
- package/dist/utils/pdfProcessor.js +7 -7
- package/dist/utils/toolUtils.d.ts +8 -0
- package/dist/utils/toolUtils.js +15 -0
- package/dist/workflow/config.d.ts +82 -82
- package/package.json +1 -1
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Summarization Prompt Builder
|
|
3
3
|
*
|
|
4
|
-
* Builds prompts for summarizing conversation context into a
|
|
4
|
+
* Builds prompts for summarizing conversation context into a 10-section structure.
|
|
5
5
|
* Supports both initial summarization and incremental merging with existing summaries.
|
|
6
6
|
*/
|
|
7
|
-
const
|
|
7
|
+
const SUMMARY_SECTIONS = [
|
|
8
8
|
"Primary Request and Intent",
|
|
9
9
|
"Key Technical Concepts",
|
|
10
10
|
"Files and Code Sections",
|
|
@@ -14,6 +14,7 @@ const NINE_SECTIONS = [
|
|
|
14
14
|
"Current Work",
|
|
15
15
|
"Next Step",
|
|
16
16
|
"Required Files",
|
|
17
|
+
"Constraints and Established Rules",
|
|
17
18
|
];
|
|
18
19
|
function buildFileContextSection(filesRead, filesModified) {
|
|
19
20
|
const hasFiles = (filesRead && filesRead.length > 0) ||
|
|
@@ -38,7 +39,7 @@ function buildFileContextSection(filesRead, filesModified) {
|
|
|
38
39
|
}
|
|
39
40
|
function buildInitialPrompt(options) {
|
|
40
41
|
const fileContext = buildFileContextSection(options.filesRead, options.filesModified);
|
|
41
|
-
return `You are a context summarization assistant. Your task is to analyze the conversation and create a structured summary following a
|
|
42
|
+
return `You are a context summarization assistant. Your task is to analyze the conversation and create a structured summary following a 10-section format.
|
|
42
43
|
|
|
43
44
|
Create a summary with the following sections:
|
|
44
45
|
|
|
@@ -67,7 +68,12 @@ What is being actively worked on right now?
|
|
|
67
68
|
What is the immediate next action to take?
|
|
68
69
|
|
|
69
70
|
### 9. Required Files
|
|
70
|
-
What files will need to be accessed or modified to continue
|
|
71
|
+
What files will need to be accessed or modified to continue?
|
|
72
|
+
|
|
73
|
+
### 10. Constraints and Established Rules
|
|
74
|
+
What user-imposed constraints, behavioral directives, coding standards, agreed-upon decisions, or established rules must persist across the conversation? Include any "always do X", "never do Y", naming conventions, architectural patterns, or preferences the user has stated.${fileContext}
|
|
75
|
+
|
|
76
|
+
IMPORTANT: Section 10 (Constraints and Established Rules) must ALWAYS be preserved in full. User constraints and established agreements are never "no longer relevant" — they remain in effect for the entire session unless the user explicitly revokes them.
|
|
71
77
|
|
|
72
78
|
Analyze the conversation thoroughly and fill in each section with relevant information. If a section is not applicable, write "N/A" for that section.`;
|
|
73
79
|
}
|
|
@@ -80,18 +86,19 @@ Existing Summary:
|
|
|
80
86
|
${options.previousSummary}
|
|
81
87
|
---
|
|
82
88
|
|
|
83
|
-
Update sections with new information while preserving important context from the existing summary. Maintain the same
|
|
89
|
+
Update sections with new information while preserving important context from the existing summary. Maintain the same 10-section structure.
|
|
84
90
|
|
|
85
91
|
Instructions:
|
|
86
92
|
1. Review the existing summary above
|
|
87
93
|
2. Analyze the new conversation content
|
|
88
94
|
3. MERGE the new information into the appropriate sections
|
|
89
95
|
4. Update sections with relevant new information
|
|
90
|
-
5. Remove information that is no longer relevant
|
|
96
|
+
5. Remove information that is no longer relevant EXCEPT for Section 10 (Constraints and Established Rules) — user constraints and established agreements must ALWAYS be preserved unless the user explicitly revoked them
|
|
91
97
|
6. Keep the summary concise but comprehensive
|
|
92
|
-
7. Maintain the
|
|
98
|
+
7. Maintain the 10-section format
|
|
99
|
+
8. Always carry forward ALL entries from Section 10 of the existing summary, adding any new constraints found in the new content${fileContext}
|
|
93
100
|
|
|
94
|
-
Output the updated summary following the same
|
|
101
|
+
Output the updated summary following the same 10-section structure.`;
|
|
95
102
|
}
|
|
96
103
|
/**
|
|
97
104
|
* Builds a summarization prompt based on the provided options.
|
|
@@ -107,4 +114,4 @@ export function buildSummarizationPrompt(options) {
|
|
|
107
114
|
}
|
|
108
115
|
return buildInitialPrompt(options);
|
|
109
116
|
}
|
|
110
|
-
export {
|
|
117
|
+
export { SUMMARY_SECTIONS };
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Stage 3: Structured LLM Summarization
|
|
3
3
|
*
|
|
4
|
-
* Uses the structured
|
|
4
|
+
* Uses the structured 10-section prompt to summarize older messages
|
|
5
5
|
* while preserving recent ones.
|
|
6
6
|
*/
|
|
7
|
-
import type { ChatMessage } from "../../types/conversation.js";
|
|
8
7
|
import type { SummarizeConfig, SummarizeResult } from "../../types/contextTypes.js";
|
|
8
|
+
import type { ChatMessage } from "../../types/conversation.js";
|
|
9
9
|
export type { SummarizeConfig, SummarizeResult, } from "../../types/contextTypes.js";
|
|
10
10
|
export declare function summarizeMessages(messages: ChatMessage[], config?: SummarizeConfig): Promise<SummarizeResult>;
|
|
@@ -1,19 +1,59 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Stage 3: Structured LLM Summarization
|
|
3
3
|
*
|
|
4
|
-
* Uses the structured
|
|
4
|
+
* Uses the structured 10-section prompt to summarize older messages
|
|
5
5
|
* while preserving recent ones.
|
|
6
6
|
*/
|
|
7
|
-
import { generateSummary } from "../../utils/conversationMemory.js";
|
|
8
7
|
import { randomUUID } from "crypto";
|
|
8
|
+
import { generateSummary } from "../../utils/conversationMemory.js";
|
|
9
|
+
import { estimateTokens } from "../../utils/tokenEstimation.js";
|
|
10
|
+
import { logger } from "../../utils/logger.js";
|
|
11
|
+
/**
|
|
12
|
+
* Find the split index using token counting — walk backward from the end,
|
|
13
|
+
* accumulating token counts until we've reserved `targetRecentTokens` worth
|
|
14
|
+
* of recent content. Everything before the split index gets summarized.
|
|
15
|
+
*/
|
|
16
|
+
function findSplitIndexByTokens(messages, targetRecentTokens, provider) {
|
|
17
|
+
let recentTokens = 0;
|
|
18
|
+
let splitIndex = messages.length;
|
|
19
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
20
|
+
const content = typeof messages[i].content === "string"
|
|
21
|
+
? messages[i].content
|
|
22
|
+
: JSON.stringify(messages[i].content);
|
|
23
|
+
const msgTokens = estimateTokens(content, provider);
|
|
24
|
+
if (recentTokens + msgTokens > targetRecentTokens) {
|
|
25
|
+
splitIndex = i + 1;
|
|
26
|
+
break;
|
|
27
|
+
}
|
|
28
|
+
recentTokens += msgTokens;
|
|
29
|
+
}
|
|
30
|
+
// Ensure at least one message is summarized
|
|
31
|
+
return Math.max(1, splitIndex);
|
|
32
|
+
}
|
|
9
33
|
export async function summarizeMessages(messages, config) {
|
|
10
34
|
const keepRecentRatio = config?.keepRecentRatio ?? 0.3;
|
|
11
35
|
if (messages.length <= 4) {
|
|
12
36
|
return { summarized: false, messages };
|
|
13
37
|
}
|
|
14
|
-
//
|
|
15
|
-
|
|
16
|
-
|
|
38
|
+
// Determine split point: prefer token-based when a target budget is available,
|
|
39
|
+
// fall back to message-count-based split for backward compatibility.
|
|
40
|
+
let splitIndex;
|
|
41
|
+
if (config?.targetTokens && config.targetTokens > 0) {
|
|
42
|
+
// Keep `keepRecentRatio` fraction of the target budget as recent context
|
|
43
|
+
const targetRecentTokens = Math.floor(config.targetTokens * keepRecentRatio);
|
|
44
|
+
// NOTE: config.provider is the summarization provider, not the generation
|
|
45
|
+
// provider. Ideally we'd use the generation/budget provider for accurate
|
|
46
|
+
// token estimation, but SummarizeConfig doesn't carry a separate
|
|
47
|
+
// budgetProvider field. This is a known design limitation.
|
|
48
|
+
splitIndex = findSplitIndexByTokens(messages, targetRecentTokens, config.provider);
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
// Legacy: message-count-based split
|
|
52
|
+
const keepCount = Math.max(4, Math.ceil(messages.length * keepRecentRatio));
|
|
53
|
+
splitIndex = messages.length - keepCount;
|
|
54
|
+
}
|
|
55
|
+
// Clamp so at least the last message is always preserved (never summarize everything)
|
|
56
|
+
splitIndex = Math.min(splitIndex, messages.length - 1);
|
|
17
57
|
if (splitIndex <= 0) {
|
|
18
58
|
return { summarized: false, messages };
|
|
19
59
|
}
|
|
@@ -21,29 +61,39 @@ export async function summarizeMessages(messages, config) {
|
|
|
21
61
|
const recentMessages = messages.slice(splitIndex);
|
|
22
62
|
// Find previous summary if exists
|
|
23
63
|
const previousSummary = messagesToSummarize.find((m) => m.metadata?.isSummary)?.content;
|
|
24
|
-
//
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
return {
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
64
|
+
// Build effective memory config: use provided memoryConfig, or construct from provider/model
|
|
65
|
+
const effectiveMemoryConfig = config?.memoryConfig ? { ...config.memoryConfig } : {};
|
|
66
|
+
// Fill in summarization provider/model from compactor config if not already set
|
|
67
|
+
if (!effectiveMemoryConfig.summarizationProvider && config?.provider) {
|
|
68
|
+
effectiveMemoryConfig.summarizationProvider = config.provider;
|
|
69
|
+
}
|
|
70
|
+
if (!effectiveMemoryConfig.summarizationModel && config?.model) {
|
|
71
|
+
effectiveMemoryConfig.summarizationModel = config.model;
|
|
72
|
+
}
|
|
73
|
+
// Only skip if there's genuinely no provider available
|
|
74
|
+
if (!effectiveMemoryConfig.summarizationProvider &&
|
|
75
|
+
!effectiveMemoryConfig.summarizationModel) {
|
|
76
|
+
logger.debug("[ContextCompactor] Stage 3 skipped: no summarization provider or model available");
|
|
77
|
+
return { summarized: false, messages };
|
|
78
|
+
}
|
|
79
|
+
const summaryText = await generateSummary(messagesToSummarize, effectiveMemoryConfig, "[ContextCompactor]", previousSummary);
|
|
80
|
+
if (!summaryText) {
|
|
81
|
+
return { summarized: false, messages };
|
|
82
|
+
}
|
|
83
|
+
const summaryMessage = {
|
|
84
|
+
id: `summary-${randomUUID()}`,
|
|
85
|
+
role: "user",
|
|
86
|
+
content: `[Previous conversation summary]:\n\n${summaryText}`,
|
|
87
|
+
timestamp: new Date().toISOString(),
|
|
88
|
+
metadata: {
|
|
89
|
+
isSummary: true,
|
|
90
|
+
summarizesFrom: messagesToSummarize[0]?.id,
|
|
91
|
+
summarizesTo: messagesToSummarize[messagesToSummarize.length - 1]?.id,
|
|
92
|
+
},
|
|
93
|
+
};
|
|
94
|
+
return {
|
|
95
|
+
summarized: true,
|
|
96
|
+
messages: [summaryMessage, ...recentMessages],
|
|
97
|
+
summaryText,
|
|
98
|
+
};
|
|
49
99
|
}
|
|
@@ -14,34 +14,10 @@ export declare class MultimodalLogger {
|
|
|
14
14
|
*/
|
|
15
15
|
export declare class ProviderImageAdapter {
|
|
16
16
|
/**
|
|
17
|
-
*
|
|
17
|
+
* Validate image count against provider limits.
|
|
18
|
+
* Warns at 80% threshold, throws error if limit exceeded.
|
|
18
19
|
*/
|
|
19
|
-
static
|
|
20
|
-
/**
|
|
21
|
-
* Format content for OpenAI (GPT-4o format)
|
|
22
|
-
*/
|
|
23
|
-
private static formatForOpenAI;
|
|
24
|
-
/**
|
|
25
|
-
* Format content for Google AI (Gemini format)
|
|
26
|
-
*/
|
|
27
|
-
private static formatForGoogleAI;
|
|
28
|
-
/**
|
|
29
|
-
* Format content for Anthropic (Claude format)
|
|
30
|
-
*/
|
|
31
|
-
private static formatForAnthropic;
|
|
32
|
-
/**
|
|
33
|
-
* Format content for Vertex AI (model-specific routing)
|
|
34
|
-
*/
|
|
35
|
-
private static formatForVertex;
|
|
36
|
-
/**
|
|
37
|
-
* Validate image count against provider limits
|
|
38
|
-
* Warns at 80% threshold, throws error if limit exceeded
|
|
39
|
-
*/
|
|
40
|
-
private static validateImageCount;
|
|
41
|
-
/**
|
|
42
|
-
* Validate that provider and model support vision
|
|
43
|
-
*/
|
|
44
|
-
private static validateVisionSupport;
|
|
20
|
+
static validateImageCount(imageCount: number, provider: string, model?: string): void;
|
|
45
21
|
/**
|
|
46
22
|
* Convert simple images array to advanced content format
|
|
47
23
|
* @param text - Text content to include
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
* Provider Image Adapter - Smart routing for multimodal content
|
|
3
3
|
* Handles provider-specific image formatting and vision capability validation
|
|
4
4
|
*/
|
|
5
|
-
import { logger } from "../utils/logger.js";
|
|
6
5
|
import { ImageProcessor } from "../utils/imageProcessor.js";
|
|
6
|
+
import { logger } from "../utils/logger.js";
|
|
7
7
|
/**
|
|
8
8
|
* Simplified logger for essential error reporting only
|
|
9
9
|
*/
|
|
@@ -436,184 +436,15 @@ const VISION_CAPABILITIES = {
|
|
|
436
436
|
* Provider Image Adapter - Smart routing and formatting
|
|
437
437
|
*/
|
|
438
438
|
export class ProviderImageAdapter {
|
|
439
|
+
// NOTE: The legacy `adaptForProvider` method and its private helpers
|
|
440
|
+
// (formatForOpenAI, formatForGoogleAI, formatForAnthropic, formatForVertex,
|
|
441
|
+
// validateVisionSupport) were removed as dead code. The production image
|
|
442
|
+
// pipeline uses `convertSimpleImagesToProviderFormat` in messageBuilder.ts
|
|
443
|
+
// with Vercel AI SDK's native ImagePart format. Image count limits are
|
|
444
|
+
// enforced via the public `validateImageCount` method below.
|
|
439
445
|
/**
|
|
440
|
-
*
|
|
441
|
-
|
|
442
|
-
static async adaptForProvider(text, images, provider, model) {
|
|
443
|
-
try {
|
|
444
|
-
// Validate provider supports vision
|
|
445
|
-
this.validateVisionSupport(provider, model);
|
|
446
|
-
let adaptedPayload;
|
|
447
|
-
// Process images based on provider requirements
|
|
448
|
-
switch (provider.toLowerCase()) {
|
|
449
|
-
case "openai":
|
|
450
|
-
adaptedPayload = this.formatForOpenAI(text, images);
|
|
451
|
-
break;
|
|
452
|
-
case "azure":
|
|
453
|
-
case "azure-openai":
|
|
454
|
-
// Azure uses same format as OpenAI but validate with azure provider name
|
|
455
|
-
this.validateImageCount(images.length, "azure");
|
|
456
|
-
adaptedPayload = this.formatForOpenAI(text, images, true);
|
|
457
|
-
break;
|
|
458
|
-
case "google-ai":
|
|
459
|
-
case "google":
|
|
460
|
-
adaptedPayload = this.formatForGoogleAI(text, images);
|
|
461
|
-
break;
|
|
462
|
-
case "anthropic":
|
|
463
|
-
adaptedPayload = this.formatForAnthropic(text, images);
|
|
464
|
-
break;
|
|
465
|
-
case "vertex":
|
|
466
|
-
adaptedPayload = this.formatForVertex(text, images, model);
|
|
467
|
-
break;
|
|
468
|
-
case "ollama":
|
|
469
|
-
// Ollama uses same format as OpenAI but validate with ollama provider name
|
|
470
|
-
this.validateImageCount(images.length, "ollama");
|
|
471
|
-
adaptedPayload = this.formatForOpenAI(text, images, true);
|
|
472
|
-
break;
|
|
473
|
-
case "huggingface":
|
|
474
|
-
adaptedPayload = this.formatForOpenAI(text, images);
|
|
475
|
-
break;
|
|
476
|
-
case "sagemaker":
|
|
477
|
-
adaptedPayload = this.formatForOpenAI(text, images);
|
|
478
|
-
break;
|
|
479
|
-
case "litellm":
|
|
480
|
-
// LiteLLM uses same format as OpenAI but validate with litellm provider name
|
|
481
|
-
this.validateImageCount(images.length, "litellm");
|
|
482
|
-
adaptedPayload = this.formatForOpenAI(text, images, true);
|
|
483
|
-
break;
|
|
484
|
-
case "mistral":
|
|
485
|
-
// Mistral uses same format as OpenAI but validate with mistral provider name
|
|
486
|
-
this.validateImageCount(images.length, "mistral");
|
|
487
|
-
adaptedPayload = this.formatForOpenAI(text, images, true);
|
|
488
|
-
break;
|
|
489
|
-
case "bedrock":
|
|
490
|
-
// Bedrock uses same format as Anthropic but validate with bedrock provider name
|
|
491
|
-
this.validateImageCount(images.length, "bedrock");
|
|
492
|
-
adaptedPayload = this.formatForAnthropic(text, images, true);
|
|
493
|
-
break;
|
|
494
|
-
case "openrouter":
|
|
495
|
-
// OpenRouter routes to underlying providers, use OpenAI format
|
|
496
|
-
this.validateImageCount(images.length, "openrouter");
|
|
497
|
-
adaptedPayload = this.formatForOpenAI(text, images);
|
|
498
|
-
break;
|
|
499
|
-
default:
|
|
500
|
-
throw new Error(`Vision not supported for provider: ${provider}`);
|
|
501
|
-
}
|
|
502
|
-
return adaptedPayload;
|
|
503
|
-
}
|
|
504
|
-
catch (error) {
|
|
505
|
-
MultimodalLogger.logError("ADAPTATION", error, {
|
|
506
|
-
provider,
|
|
507
|
-
model,
|
|
508
|
-
imageCount: images.length,
|
|
509
|
-
});
|
|
510
|
-
throw error;
|
|
511
|
-
}
|
|
512
|
-
}
|
|
513
|
-
/**
|
|
514
|
-
* Format content for OpenAI (GPT-4o format)
|
|
515
|
-
*/
|
|
516
|
-
static formatForOpenAI(text, images, skipValidation = false) {
|
|
517
|
-
// Validate image count before processing (unless called from another formatter)
|
|
518
|
-
if (!skipValidation) {
|
|
519
|
-
this.validateImageCount(images.length, "openai");
|
|
520
|
-
}
|
|
521
|
-
const content = [{ type: "text", text }];
|
|
522
|
-
images.forEach((image, index) => {
|
|
523
|
-
try {
|
|
524
|
-
const imageUrl = ImageProcessor.processImageForOpenAI(image);
|
|
525
|
-
content.push({
|
|
526
|
-
type: "image_url",
|
|
527
|
-
image_url: { url: imageUrl },
|
|
528
|
-
});
|
|
529
|
-
}
|
|
530
|
-
catch (error) {
|
|
531
|
-
MultimodalLogger.logError("PROCESS_IMAGE", error, {
|
|
532
|
-
index,
|
|
533
|
-
provider: "openai",
|
|
534
|
-
});
|
|
535
|
-
throw error;
|
|
536
|
-
}
|
|
537
|
-
});
|
|
538
|
-
return { messages: [{ role: "user", content }] };
|
|
539
|
-
}
|
|
540
|
-
/**
|
|
541
|
-
* Format content for Google AI (Gemini format)
|
|
542
|
-
*/
|
|
543
|
-
static formatForGoogleAI(text, images, skipValidation = false) {
|
|
544
|
-
// Validate image count before processing (unless called from another formatter)
|
|
545
|
-
if (!skipValidation) {
|
|
546
|
-
this.validateImageCount(images.length, "google-ai");
|
|
547
|
-
}
|
|
548
|
-
const parts = [{ text }];
|
|
549
|
-
images.forEach((image, index) => {
|
|
550
|
-
try {
|
|
551
|
-
const { mimeType, data } = ImageProcessor.processImageForGoogle(image);
|
|
552
|
-
parts.push({
|
|
553
|
-
inlineData: { mimeType, data },
|
|
554
|
-
});
|
|
555
|
-
}
|
|
556
|
-
catch (error) {
|
|
557
|
-
MultimodalLogger.logError("PROCESS_IMAGE", error, {
|
|
558
|
-
index,
|
|
559
|
-
provider: "google-ai",
|
|
560
|
-
});
|
|
561
|
-
throw error;
|
|
562
|
-
}
|
|
563
|
-
});
|
|
564
|
-
return { contents: [{ parts }] };
|
|
565
|
-
}
|
|
566
|
-
/**
|
|
567
|
-
* Format content for Anthropic (Claude format)
|
|
568
|
-
*/
|
|
569
|
-
static formatForAnthropic(text, images, skipValidation = false) {
|
|
570
|
-
// Validate image count before processing (unless called from another formatter)
|
|
571
|
-
if (!skipValidation) {
|
|
572
|
-
this.validateImageCount(images.length, "anthropic");
|
|
573
|
-
}
|
|
574
|
-
const content = [{ type: "text", text }];
|
|
575
|
-
images.forEach((image, index) => {
|
|
576
|
-
try {
|
|
577
|
-
const { mediaType, data } = ImageProcessor.processImageForAnthropic(image);
|
|
578
|
-
content.push({
|
|
579
|
-
type: "image",
|
|
580
|
-
source: {
|
|
581
|
-
type: "base64",
|
|
582
|
-
media_type: mediaType,
|
|
583
|
-
data,
|
|
584
|
-
},
|
|
585
|
-
});
|
|
586
|
-
}
|
|
587
|
-
catch (error) {
|
|
588
|
-
MultimodalLogger.logError("PROCESS_IMAGE", error, {
|
|
589
|
-
index,
|
|
590
|
-
provider: "anthropic",
|
|
591
|
-
});
|
|
592
|
-
throw error;
|
|
593
|
-
}
|
|
594
|
-
});
|
|
595
|
-
return { messages: [{ role: "user", content }] };
|
|
596
|
-
}
|
|
597
|
-
/**
|
|
598
|
-
* Format content for Vertex AI (model-specific routing)
|
|
599
|
-
*/
|
|
600
|
-
static formatForVertex(text, images, model) {
|
|
601
|
-
// Validate image count with model-specific limits before processing
|
|
602
|
-
this.validateImageCount(images.length, "vertex", model);
|
|
603
|
-
// Route based on model type, skip validation in delegated methods
|
|
604
|
-
if (model.includes("gemini")) {
|
|
605
|
-
return this.formatForGoogleAI(text, images, true);
|
|
606
|
-
}
|
|
607
|
-
else if (model.includes("claude")) {
|
|
608
|
-
return this.formatForAnthropic(text, images, true);
|
|
609
|
-
}
|
|
610
|
-
else {
|
|
611
|
-
return this.formatForGoogleAI(text, images, true);
|
|
612
|
-
}
|
|
613
|
-
}
|
|
614
|
-
/**
|
|
615
|
-
* Validate image count against provider limits
|
|
616
|
-
* Warns at 80% threshold, throws error if limit exceeded
|
|
446
|
+
* Validate image count against provider limits.
|
|
447
|
+
* Warns at 80% threshold, throws error if limit exceeded.
|
|
617
448
|
*/
|
|
618
449
|
static validateImageCount(imageCount, provider, model) {
|
|
619
450
|
const normalizedProvider = provider.toLowerCase();
|
|
@@ -659,27 +490,6 @@ export class ProviderImageAdapter {
|
|
|
659
490
|
`Maximum allowed: ${limit}. Please reduce the number of images.`);
|
|
660
491
|
}
|
|
661
492
|
}
|
|
662
|
-
/**
|
|
663
|
-
* Validate that provider and model support vision
|
|
664
|
-
*/
|
|
665
|
-
static validateVisionSupport(provider, model) {
|
|
666
|
-
const normalizedProvider = normalizeVisionProvider(provider);
|
|
667
|
-
const supportedModels = VISION_CAPABILITIES[normalizedProvider];
|
|
668
|
-
if (!supportedModels) {
|
|
669
|
-
throw new Error(`Provider ${provider} does not support vision processing. ` +
|
|
670
|
-
`Supported providers: ${Object.keys(VISION_CAPABILITIES).join(", ")}`);
|
|
671
|
-
}
|
|
672
|
-
const isSupported = supportedModels.some((supportedModel) => model.toLowerCase().includes(supportedModel.toLowerCase()));
|
|
673
|
-
// Proxy providers route to arbitrary underlying models — skip the allowlist
|
|
674
|
-
// check for unknown models and let the underlying provider error if needed.
|
|
675
|
-
if (!isSupported && PROXY_PROVIDERS.has(normalizedProvider)) {
|
|
676
|
-
return;
|
|
677
|
-
}
|
|
678
|
-
if (!isSupported) {
|
|
679
|
-
throw new Error(`Provider ${provider} with model ${model} does not support vision processing. ` +
|
|
680
|
-
`Supported models for ${provider}: ${supportedModels.join(", ")}`);
|
|
681
|
-
}
|
|
682
|
-
}
|
|
683
493
|
/**
|
|
684
494
|
* Convert simple images array to advanced content format
|
|
685
495
|
* @param text - Text content to include
|
|
@@ -464,6 +464,36 @@ export declare const directAgentTools: {
|
|
|
464
464
|
}>;
|
|
465
465
|
};
|
|
466
466
|
};
|
|
467
|
+
/**
|
|
468
|
+
* Bash command execution tool - exported separately for opt-in use.
|
|
469
|
+
*
|
|
470
|
+
* SECURITY: This tool is NOT included in directAgentTools by default.
|
|
471
|
+
* It must be explicitly enabled via:
|
|
472
|
+
* - Environment variable: NEUROLINK_ENABLE_BASH_TOOL=true
|
|
473
|
+
* - Config: toolConfig.enableBashTool = true
|
|
474
|
+
*
|
|
475
|
+
* Import this directly when you need bash execution capabilities:
|
|
476
|
+
* import { bashTool } from '../agent/directTools.js';
|
|
477
|
+
*/
|
|
478
|
+
export declare const bashTool: import("ai").Tool<z.ZodObject<{
|
|
479
|
+
command: z.ZodString;
|
|
480
|
+
timeout: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
|
|
481
|
+
cwd: z.ZodOptional<z.ZodString>;
|
|
482
|
+
}, "strip", z.ZodTypeAny, {
|
|
483
|
+
timeout: number;
|
|
484
|
+
command: string;
|
|
485
|
+
cwd?: string | undefined;
|
|
486
|
+
}, {
|
|
487
|
+
command: string;
|
|
488
|
+
timeout?: number | undefined;
|
|
489
|
+
cwd?: string | undefined;
|
|
490
|
+
}>, unknown> & {
|
|
491
|
+
execute: (args: {
|
|
492
|
+
timeout: number;
|
|
493
|
+
command: string;
|
|
494
|
+
cwd?: string | undefined;
|
|
495
|
+
}, options: import("ai").ToolExecutionOptions) => PromiseLike<unknown>;
|
|
496
|
+
};
|
|
467
497
|
/**
|
|
468
498
|
* Type aliases for specific tool categories
|
|
469
499
|
*/
|
|
@@ -481,7 +511,9 @@ export type UtilityToolsMap = {
|
|
|
481
511
|
calculateMath: typeof directAgentTools.calculateMath;
|
|
482
512
|
listDirectory: typeof directAgentTools.listDirectory;
|
|
483
513
|
};
|
|
484
|
-
export type AllToolsMap = typeof directAgentTools
|
|
514
|
+
export type AllToolsMap = typeof directAgentTools & {
|
|
515
|
+
executeBashCommand?: typeof bashTool;
|
|
516
|
+
};
|
|
485
517
|
/**
|
|
486
518
|
* Get a subset of tools for specific use cases with improved type safety
|
|
487
519
|
*/
|