@juspay/neurolink 7.35.0 → 7.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/adapters/providerImageAdapter.d.ts +56 -0
- package/dist/adapters/providerImageAdapter.js +257 -0
- package/dist/cli/commands/config.d.ts +20 -20
- package/dist/cli/factories/commandFactory.d.ts +1 -0
- package/dist/cli/factories/commandFactory.js +26 -3
- package/dist/config/taskClassificationConfig.d.ts +51 -0
- package/dist/config/taskClassificationConfig.js +148 -0
- package/dist/core/baseProvider.js +99 -45
- package/dist/core/types.d.ts +3 -0
- package/dist/lib/adapters/providerImageAdapter.d.ts +56 -0
- package/dist/lib/adapters/providerImageAdapter.js +257 -0
- package/dist/lib/config/taskClassificationConfig.d.ts +51 -0
- package/dist/lib/config/taskClassificationConfig.js +148 -0
- package/dist/lib/core/baseProvider.js +99 -45
- package/dist/lib/core/types.d.ts +3 -0
- package/dist/lib/neurolink.d.ts +20 -0
- package/dist/lib/neurolink.js +276 -8
- package/dist/lib/types/content.d.ts +78 -0
- package/dist/lib/types/content.js +5 -0
- package/dist/lib/types/conversation.d.ts +19 -0
- package/dist/lib/types/generateTypes.d.ts +4 -1
- package/dist/lib/types/index.d.ts +2 -0
- package/dist/lib/types/index.js +2 -0
- package/dist/lib/types/streamTypes.d.ts +6 -3
- package/dist/lib/types/taskClassificationTypes.d.ts +52 -0
- package/dist/lib/types/taskClassificationTypes.js +5 -0
- package/dist/lib/utils/imageProcessor.d.ts +84 -0
- package/dist/lib/utils/imageProcessor.js +362 -0
- package/dist/lib/utils/messageBuilder.d.ts +8 -1
- package/dist/lib/utils/messageBuilder.js +279 -0
- package/dist/lib/utils/modelRouter.d.ts +107 -0
- package/dist/lib/utils/modelRouter.js +292 -0
- package/dist/lib/utils/promptRedaction.d.ts +29 -0
- package/dist/lib/utils/promptRedaction.js +62 -0
- package/dist/lib/utils/taskClassificationUtils.d.ts +55 -0
- package/dist/lib/utils/taskClassificationUtils.js +149 -0
- package/dist/lib/utils/taskClassifier.d.ts +23 -0
- package/dist/lib/utils/taskClassifier.js +94 -0
- package/dist/neurolink.d.ts +20 -0
- package/dist/neurolink.js +276 -8
- package/dist/types/content.d.ts +78 -0
- package/dist/types/content.js +5 -0
- package/dist/types/conversation.d.ts +19 -0
- package/dist/types/generateTypes.d.ts +4 -1
- package/dist/types/index.d.ts +2 -0
- package/dist/types/index.js +2 -0
- package/dist/types/streamTypes.d.ts +6 -3
- package/dist/types/taskClassificationTypes.d.ts +52 -0
- package/dist/types/taskClassificationTypes.js +5 -0
- package/dist/utils/imageProcessor.d.ts +84 -0
- package/dist/utils/imageProcessor.js +362 -0
- package/dist/utils/messageBuilder.d.ts +8 -1
- package/dist/utils/messageBuilder.js +279 -0
- package/dist/utils/modelRouter.d.ts +107 -0
- package/dist/utils/modelRouter.js +292 -0
- package/dist/utils/promptRedaction.d.ts +29 -0
- package/dist/utils/promptRedaction.js +62 -0
- package/dist/utils/taskClassificationUtils.d.ts +55 -0
- package/dist/utils/taskClassificationUtils.js +149 -0
- package/dist/utils/taskClassifier.d.ts +23 -0
- package/dist/utils/taskClassifier.js +94 -0
- package/package.json +1 -1
package/dist/lib/neurolink.d.ts
CHANGED
@@ -62,6 +62,7 @@ export declare class NeuroLink {
|
|
62
62
|
private conversationMemory?;
|
63
63
|
private conversationMemoryNeedsInit;
|
64
64
|
private conversationMemoryConfig?;
|
65
|
+
private enableOrchestration;
|
65
66
|
/**
|
66
67
|
* Creates a new NeuroLink instance for AI text generation with MCP tool integration.
|
67
68
|
*
|
@@ -70,6 +71,7 @@ export declare class NeuroLink {
|
|
70
71
|
* @param config.conversationMemory.enabled - Whether to enable conversation memory (default: false)
|
71
72
|
* @param config.conversationMemory.maxSessions - Maximum number of concurrent sessions (default: 100)
|
72
73
|
* @param config.conversationMemory.maxTurnsPerSession - Maximum conversation turns per session (default: 50)
|
74
|
+
* @param config.enableOrchestration - Whether to enable smart model orchestration (default: false)
|
73
75
|
*
|
74
76
|
* @example
|
75
77
|
* ```typescript
|
@@ -84,6 +86,11 @@ export declare class NeuroLink {
|
|
84
86
|
* maxTurnsPerSession: 20
|
85
87
|
* }
|
86
88
|
* });
|
89
|
+
*
|
90
|
+
* // With orchestration enabled
|
91
|
+
* const neurolink = new NeuroLink({
|
92
|
+
* enableOrchestration: true
|
93
|
+
* });
|
87
94
|
* ```
|
88
95
|
*
|
89
96
|
* @throws {Error} When provider registry setup fails
|
@@ -92,6 +99,7 @@ export declare class NeuroLink {
|
|
92
99
|
*/
|
93
100
|
constructor(config?: {
|
94
101
|
conversationMemory?: Partial<ConversationMemoryConfig>;
|
102
|
+
enableOrchestration?: boolean;
|
95
103
|
});
|
96
104
|
/**
|
97
105
|
* Log constructor start with comprehensive environment analysis
|
@@ -158,6 +166,18 @@ export declare class NeuroLink {
|
|
158
166
|
* Log MCP initialization completion
|
159
167
|
*/
|
160
168
|
private logMCPInitComplete;
|
169
|
+
/**
|
170
|
+
* Apply orchestration to determine optimal provider and model
|
171
|
+
* @param options - Original GenerateOptions
|
172
|
+
* @returns Modified options with orchestrated provider marked in context, or empty object if validation fails
|
173
|
+
*/
|
174
|
+
private applyOrchestration;
|
175
|
+
/**
|
176
|
+
* Apply orchestration to determine optimal provider and model for streaming
|
177
|
+
* @param options - Original StreamOptions
|
178
|
+
* @returns Modified options with orchestrated provider marked in context, or empty object if validation fails
|
179
|
+
*/
|
180
|
+
private applyStreamOrchestration;
|
161
181
|
/**
|
162
182
|
* MAIN ENTRY POINT: Enhanced generate method with new function signature
|
163
183
|
* Replaces both generateText and legacy methods
|
package/dist/lib/neurolink.js
CHANGED
@@ -35,6 +35,9 @@ import { getConversationMessages, storeConversationTurn, } from "./utils/convers
|
|
35
35
|
import { ExternalServerManager } from "./mcp/externalServerManager.js";
|
36
36
|
// Import direct tools server for automatic registration
|
37
37
|
import { directToolsServer } from "./mcp/servers/agent/directToolsServer.js";
|
38
|
+
// Import orchestration components
|
39
|
+
import { ModelRouter } from "./utils/modelRouter.js";
|
40
|
+
import { BinaryTaskClassifier } from "./utils/taskClassifier.js";
|
38
41
|
import { isNonNullObject } from "./utils/typeUtils.js";
|
39
42
|
// Core types imported from "./types/index.js"
|
40
43
|
export class NeuroLink {
|
@@ -75,6 +78,8 @@ export class NeuroLink {
|
|
75
78
|
conversationMemory;
|
76
79
|
conversationMemoryNeedsInit = false;
|
77
80
|
conversationMemoryConfig;
|
81
|
+
// Add orchestration property
|
82
|
+
enableOrchestration;
|
78
83
|
/**
|
79
84
|
* Creates a new NeuroLink instance for AI text generation with MCP tool integration.
|
80
85
|
*
|
@@ -83,6 +88,7 @@ export class NeuroLink {
|
|
83
88
|
* @param config.conversationMemory.enabled - Whether to enable conversation memory (default: false)
|
84
89
|
* @param config.conversationMemory.maxSessions - Maximum number of concurrent sessions (default: 100)
|
85
90
|
* @param config.conversationMemory.maxTurnsPerSession - Maximum conversation turns per session (default: 50)
|
91
|
+
* @param config.enableOrchestration - Whether to enable smart model orchestration (default: false)
|
86
92
|
*
|
87
93
|
* @example
|
88
94
|
* ```typescript
|
@@ -97,6 +103,11 @@ export class NeuroLink {
|
|
97
103
|
* maxTurnsPerSession: 20
|
98
104
|
* }
|
99
105
|
* });
|
106
|
+
*
|
107
|
+
* // With orchestration enabled
|
108
|
+
* const neurolink = new NeuroLink({
|
109
|
+
* enableOrchestration: true
|
110
|
+
* });
|
100
111
|
* ```
|
101
112
|
*
|
102
113
|
* @throws {Error} When provider registry setup fails
|
@@ -112,6 +123,8 @@ export class NeuroLink {
|
|
112
123
|
const constructorStartTime = Date.now();
|
113
124
|
const constructorHrTimeStart = process.hrtime.bigint();
|
114
125
|
const constructorId = `neurolink-constructor-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
126
|
+
// Initialize orchestration setting
|
127
|
+
this.enableOrchestration = config?.enableOrchestration ?? false;
|
115
128
|
this.logConstructorStart(constructorId, constructorStartTime, constructorHrTimeStart, config);
|
116
129
|
this.initializeProviderRegistry(constructorId, constructorStartTime, constructorHrTimeStart);
|
117
130
|
this.initializeConversationMemory(config, constructorId, constructorStartTime, constructorHrTimeStart);
|
@@ -758,6 +771,206 @@ export class NeuroLink {
|
|
758
771
|
mcpLogger.debug("💡 Memory cleanup suggestion: MCP initialization used significant memory. Consider calling MemoryManager.forceGC() after heavy operations.");
|
759
772
|
}
|
760
773
|
}
|
774
|
+
/**
|
775
|
+
* Apply orchestration to determine optimal provider and model
|
776
|
+
* @param options - Original GenerateOptions
|
777
|
+
* @returns Modified options with orchestrated provider marked in context, or empty object if validation fails
|
778
|
+
*/
|
779
|
+
async applyOrchestration(options) {
|
780
|
+
const startTime = Date.now();
|
781
|
+
try {
|
782
|
+
// Ensure input.text exists before proceeding
|
783
|
+
if (!options.input?.text || typeof options.input.text !== "string") {
|
784
|
+
logger.debug("Orchestration skipped - no valid input text", {
|
785
|
+
hasInput: !!options.input,
|
786
|
+
hasText: !!options.input?.text,
|
787
|
+
textType: typeof options.input?.text,
|
788
|
+
});
|
789
|
+
return {}; // Return empty object to preserve existing fallback behavior
|
790
|
+
}
|
791
|
+
// Compute classification once to avoid duplicate calls
|
792
|
+
const classification = BinaryTaskClassifier.classify(options.input.text);
|
793
|
+
// Use the model router to get the optimal route
|
794
|
+
const route = ModelRouter.route(options.input.text);
|
795
|
+
// Validate that the routed provider is available and configured
|
796
|
+
const isProviderAvailable = await this.hasProviderEnvVars(route.provider);
|
797
|
+
if (!isProviderAvailable && route.provider !== "ollama") {
|
798
|
+
logger.debug("Orchestration provider validation failed", {
|
799
|
+
taskType: classification.type,
|
800
|
+
routedProvider: route.provider,
|
801
|
+
routedModel: route.model,
|
802
|
+
reason: "Provider not configured or missing environment variables",
|
803
|
+
orchestrationTime: `${Date.now() - startTime}ms`,
|
804
|
+
});
|
805
|
+
return {}; // Return empty object to preserve existing fallback behavior
|
806
|
+
}
|
807
|
+
// For Ollama, check if service is running and model is available
|
808
|
+
if (route.provider === "ollama") {
|
809
|
+
try {
|
810
|
+
const response = await fetch("http://localhost:11434/api/tags", {
|
811
|
+
method: "GET",
|
812
|
+
signal: AbortSignal.timeout(2000),
|
813
|
+
});
|
814
|
+
if (!response.ok) {
|
815
|
+
logger.debug("Orchestration provider validation failed", {
|
816
|
+
taskType: classification.type,
|
817
|
+
routedProvider: route.provider,
|
818
|
+
routedModel: route.model,
|
819
|
+
reason: "Ollama service not responding",
|
820
|
+
orchestrationTime: `${Date.now() - startTime}ms`,
|
821
|
+
});
|
822
|
+
return {}; // Return empty object to preserve existing fallback behavior
|
823
|
+
}
|
824
|
+
const { models } = await response.json();
|
825
|
+
const modelIsAvailable = models.some((m) => m.name === (route.model || "llama3.2:latest"));
|
826
|
+
if (!modelIsAvailable) {
|
827
|
+
logger.debug("Orchestration provider validation failed", {
|
828
|
+
taskType: classification.type,
|
829
|
+
routedProvider: route.provider,
|
830
|
+
routedModel: route.model,
|
831
|
+
reason: `Ollama model '${route.model || "llama3.2:latest"}' not found`,
|
832
|
+
orchestrationTime: `${Date.now() - startTime}ms`,
|
833
|
+
});
|
834
|
+
return {}; // Return empty object to preserve existing fallback behavior
|
835
|
+
}
|
836
|
+
}
|
837
|
+
catch (error) {
|
838
|
+
logger.debug("Orchestration provider validation failed", {
|
839
|
+
taskType: classification.type,
|
840
|
+
routedProvider: route.provider,
|
841
|
+
routedModel: route.model,
|
842
|
+
reason: error instanceof Error ? error.message : "Ollama service check failed",
|
843
|
+
orchestrationTime: `${Date.now() - startTime}ms`,
|
844
|
+
});
|
845
|
+
return {}; // Return empty object to preserve existing fallback behavior
|
846
|
+
}
|
847
|
+
}
|
848
|
+
logger.debug("Orchestration route determined", {
|
849
|
+
taskType: classification.type,
|
850
|
+
selectedProvider: route.provider,
|
851
|
+
selectedModel: route.model,
|
852
|
+
confidence: route.confidence,
|
853
|
+
reasoning: route.reasoning,
|
854
|
+
orchestrationTime: `${Date.now() - startTime}ms`,
|
855
|
+
});
|
856
|
+
// Mark preferred provider in context instead of directly setting provider
|
857
|
+
// This preserves global fallback behavior while indicating orchestration preference
|
858
|
+
return {
|
859
|
+
model: route.model,
|
860
|
+
context: {
|
861
|
+
...(options.context || {}),
|
862
|
+
__orchestratedPreferredProvider: route.provider,
|
863
|
+
},
|
864
|
+
};
|
865
|
+
}
|
866
|
+
catch (error) {
|
867
|
+
logger.error("Orchestration failed", {
|
868
|
+
error: error instanceof Error ? error.message : String(error),
|
869
|
+
orchestrationTime: `${Date.now() - startTime}ms`,
|
870
|
+
});
|
871
|
+
throw error;
|
872
|
+
}
|
873
|
+
}
|
874
|
+
/**
|
875
|
+
* Apply orchestration to determine optimal provider and model for streaming
|
876
|
+
* @param options - Original StreamOptions
|
877
|
+
* @returns Modified options with orchestrated provider marked in context, or empty object if validation fails
|
878
|
+
*/
|
879
|
+
async applyStreamOrchestration(options) {
|
880
|
+
const startTime = Date.now();
|
881
|
+
try {
|
882
|
+
// Ensure input.text exists before proceeding
|
883
|
+
if (!options.input?.text || typeof options.input.text !== "string") {
|
884
|
+
logger.debug("Stream orchestration skipped - no valid input text", {
|
885
|
+
hasInput: !!options.input,
|
886
|
+
hasText: !!options.input?.text,
|
887
|
+
textType: typeof options.input?.text,
|
888
|
+
});
|
889
|
+
return {}; // Return empty object to preserve existing fallback behavior
|
890
|
+
}
|
891
|
+
// Compute classification once to avoid duplicate calls
|
892
|
+
const classification = BinaryTaskClassifier.classify(options.input.text);
|
893
|
+
// Use the model router to get the optimal route
|
894
|
+
const route = ModelRouter.route(options.input.text);
|
895
|
+
// Validate that the routed provider is available and configured
|
896
|
+
const isProviderAvailable = await this.hasProviderEnvVars(route.provider);
|
897
|
+
if (!isProviderAvailable && route.provider !== "ollama") {
|
898
|
+
logger.debug("Stream orchestration provider validation failed", {
|
899
|
+
taskType: classification.type,
|
900
|
+
routedProvider: route.provider,
|
901
|
+
routedModel: route.model,
|
902
|
+
reason: "Provider not configured or missing environment variables",
|
903
|
+
orchestrationTime: `${Date.now() - startTime}ms`,
|
904
|
+
});
|
905
|
+
return {}; // Return empty object to preserve existing fallback behavior
|
906
|
+
}
|
907
|
+
// For Ollama, check if service is running and model is available
|
908
|
+
if (route.provider === "ollama") {
|
909
|
+
try {
|
910
|
+
const response = await fetch("http://localhost:11434/api/tags", {
|
911
|
+
method: "GET",
|
912
|
+
signal: AbortSignal.timeout(2000),
|
913
|
+
});
|
914
|
+
if (!response.ok) {
|
915
|
+
logger.debug("Stream orchestration provider validation failed", {
|
916
|
+
taskType: classification.type,
|
917
|
+
routedProvider: route.provider,
|
918
|
+
routedModel: route.model,
|
919
|
+
reason: "Ollama service not responding",
|
920
|
+
orchestrationTime: `${Date.now() - startTime}ms`,
|
921
|
+
});
|
922
|
+
return {}; // Return empty object to preserve existing fallback behavior
|
923
|
+
}
|
924
|
+
const { models } = await response.json();
|
925
|
+
const modelIsAvailable = models.some((m) => m.name === (route.model || "llama3.2:latest"));
|
926
|
+
if (!modelIsAvailable) {
|
927
|
+
logger.debug("Stream orchestration provider validation failed", {
|
928
|
+
taskType: classification.type,
|
929
|
+
routedProvider: route.provider,
|
930
|
+
routedModel: route.model,
|
931
|
+
reason: `Ollama model '${route.model || "llama3.2:latest"}' not found`,
|
932
|
+
orchestrationTime: `${Date.now() - startTime}ms`,
|
933
|
+
});
|
934
|
+
return {}; // Return empty object to preserve existing fallback behavior
|
935
|
+
}
|
936
|
+
}
|
937
|
+
catch (error) {
|
938
|
+
logger.debug("Stream orchestration provider validation failed", {
|
939
|
+
taskType: classification.type,
|
940
|
+
routedProvider: route.provider,
|
941
|
+
routedModel: route.model,
|
942
|
+
reason: error instanceof Error ? error.message : "Ollama service check failed",
|
943
|
+
orchestrationTime: `${Date.now() - startTime}ms`,
|
944
|
+
});
|
945
|
+
return {}; // Return empty object to preserve existing fallback behavior
|
946
|
+
}
|
947
|
+
}
|
948
|
+
logger.debug("Stream orchestration route determined", {
|
949
|
+
taskType: classification.type,
|
950
|
+
selectedProvider: route.provider,
|
951
|
+
selectedModel: route.model,
|
952
|
+
confidence: route.confidence,
|
953
|
+
reasoning: route.reasoning,
|
954
|
+
orchestrationTime: `${Date.now() - startTime}ms`,
|
955
|
+
});
|
956
|
+
// Mark preferred provider in context instead of directly setting provider
|
957
|
+
// This preserves global fallback behavior while indicating orchestration preference
|
958
|
+
return {
|
959
|
+
model: route.model,
|
960
|
+
context: {
|
961
|
+
...(options.context || {}),
|
962
|
+
__orchestratedPreferredProvider: route.provider,
|
963
|
+
},
|
964
|
+
};
|
965
|
+
}
|
966
|
+
catch (error) {
|
967
|
+
logger.error("Stream orchestration failed", {
|
968
|
+
error: error instanceof Error ? error.message : String(error),
|
969
|
+
orchestrationTime: `${Date.now() - startTime}ms`,
|
970
|
+
});
|
971
|
+
throw error;
|
972
|
+
}
|
973
|
+
}
|
761
974
|
/**
|
762
975
|
* MAIN ENTRY POINT: Enhanced generate method with new function signature
|
763
976
|
* Replaces both generateText and legacy methods
|
@@ -833,6 +1046,27 @@ export class NeuroLink {
|
|
833
1046
|
throw new Error("Input text is required and must be a non-empty string");
|
834
1047
|
}
|
835
1048
|
const startTime = Date.now();
|
1049
|
+
// Apply orchestration if enabled and no specific provider/model requested
|
1050
|
+
if (this.enableOrchestration && !options.provider && !options.model) {
|
1051
|
+
try {
|
1052
|
+
const orchestratedOptions = await this.applyOrchestration(options);
|
1053
|
+
logger.debug("Orchestration applied", {
|
1054
|
+
originalProvider: options.provider || "auto",
|
1055
|
+
orchestratedProvider: orchestratedOptions.provider,
|
1056
|
+
orchestratedModel: orchestratedOptions.model,
|
1057
|
+
prompt: options.input.text.substring(0, 100),
|
1058
|
+
});
|
1059
|
+
// Use orchestrated options
|
1060
|
+
Object.assign(options, orchestratedOptions);
|
1061
|
+
}
|
1062
|
+
catch (error) {
|
1063
|
+
logger.warn("Orchestration failed, continuing with original options", {
|
1064
|
+
error: error instanceof Error ? error.message : String(error),
|
1065
|
+
originalProvider: options.provider || "auto",
|
1066
|
+
});
|
1067
|
+
// Continue with original options if orchestration fails
|
1068
|
+
}
|
1069
|
+
}
|
836
1070
|
// Emit generation start event (NeuroLink format - keep existing)
|
837
1071
|
this.emitter.emit("generation:start", {
|
838
1072
|
provider: options.provider || "auto",
|
@@ -854,7 +1088,7 @@ export class NeuroLink {
|
|
854
1088
|
// Continue with warning rather than throwing - graceful degradation
|
855
1089
|
}
|
856
1090
|
}
|
857
|
-
// Convert to TextGenerationOptions
|
1091
|
+
// 🔧 CRITICAL FIX: Convert to TextGenerationOptions while preserving the input object for multimodal support
|
858
1092
|
const baseOptions = {
|
859
1093
|
prompt: options.input.text,
|
860
1094
|
provider: options.provider,
|
@@ -868,6 +1102,7 @@ export class NeuroLink {
|
|
868
1102
|
context: options.context,
|
869
1103
|
evaluationDomain: options.evaluationDomain,
|
870
1104
|
toolUsageContext: options.toolUsageContext,
|
1105
|
+
input: options.input, // This includes text, images, and content arrays
|
871
1106
|
};
|
872
1107
|
// Apply factory enhancement using centralized utilities
|
873
1108
|
const textOptions = enhanceTextGenerationOptions(baseOptions, factoryResult);
|
@@ -1428,14 +1663,21 @@ export class NeuroLink {
|
|
1428
1663
|
"ollama",
|
1429
1664
|
];
|
1430
1665
|
const requestedProvider = options.provider === "auto" ? undefined : options.provider;
|
1431
|
-
//
|
1432
|
-
const
|
1433
|
-
?
|
1434
|
-
:
|
1666
|
+
// Check for orchestrated preferred provider in context
|
1667
|
+
const preferredOrchestrated = options.context && typeof options.context === 'object' && '__orchestratedPreferredProvider' in options.context
|
1668
|
+
? options.context.__orchestratedPreferredProvider
|
1669
|
+
: undefined;
|
1670
|
+
// Build provider list with orchestrated preference first, then fallback to full list
|
1671
|
+
const tryProviders = preferredOrchestrated
|
1672
|
+
? [preferredOrchestrated, ...providerPriority.filter((p) => p !== preferredOrchestrated)]
|
1673
|
+
: requestedProvider
|
1674
|
+
? [requestedProvider]
|
1675
|
+
: providerPriority;
|
1435
1676
|
logger.debug(`[${functionTag}] Starting direct generation`, {
|
1436
1677
|
requestedProvider: requestedProvider || "auto",
|
1678
|
+
preferredOrchestrated: preferredOrchestrated || "none",
|
1437
1679
|
tryProviders,
|
1438
|
-
allowFallback: !requestedProvider,
|
1680
|
+
allowFallback: !requestedProvider || !!preferredOrchestrated,
|
1439
1681
|
});
|
1440
1682
|
let lastError = null;
|
1441
1683
|
// Try each provider in order
|
@@ -1650,6 +1892,28 @@ export class NeuroLink {
|
|
1650
1892
|
await this.initializeConversationMemoryForGeneration(streamId, startTime, hrTimeStart);
|
1651
1893
|
// Initialize MCP
|
1652
1894
|
await this.initializeMCP();
|
1895
|
+
const _originalPrompt = options.input.text;
|
1896
|
+
// Apply orchestration if enabled and no specific provider/model requested
|
1897
|
+
if (this.enableOrchestration && !options.provider && !options.model) {
|
1898
|
+
try {
|
1899
|
+
const orchestratedOptions = await this.applyStreamOrchestration(options);
|
1900
|
+
logger.debug("Stream orchestration applied", {
|
1901
|
+
originalProvider: options.provider || "auto",
|
1902
|
+
orchestratedProvider: orchestratedOptions.provider,
|
1903
|
+
orchestratedModel: orchestratedOptions.model,
|
1904
|
+
prompt: options.input.text?.substring(0, 100),
|
1905
|
+
});
|
1906
|
+
// Use orchestrated options
|
1907
|
+
Object.assign(options, orchestratedOptions);
|
1908
|
+
}
|
1909
|
+
catch (error) {
|
1910
|
+
logger.warn("Stream orchestration failed, continuing with original options", {
|
1911
|
+
error: error instanceof Error ? error.message : String(error),
|
1912
|
+
originalProvider: options.provider || "auto",
|
1913
|
+
});
|
1914
|
+
// Continue with original options if orchestration fails
|
1915
|
+
}
|
1916
|
+
}
|
1653
1917
|
factoryResult = processStreamingFactoryOptions(options);
|
1654
1918
|
enhancedOptions = createCleanStreamOptions(options);
|
1655
1919
|
if (options.input?.text) {
|
@@ -1664,7 +1928,9 @@ export class NeuroLink {
|
|
1664
1928
|
const processedStream = (async function* (self) {
|
1665
1929
|
try {
|
1666
1930
|
for await (const chunk of mcpStream) {
|
1667
|
-
if (chunk &&
|
1931
|
+
if (chunk &&
|
1932
|
+
"content" in chunk &&
|
1933
|
+
typeof chunk.content === "string") {
|
1668
1934
|
accumulatedContent += chunk.content;
|
1669
1935
|
// Emit chunk event for compatibility
|
1670
1936
|
self.emitter.emit("response:chunk", chunk.content);
|
@@ -1941,7 +2207,9 @@ export class NeuroLink {
|
|
1941
2207
|
const fallbackProcessedStream = (async function* (self) {
|
1942
2208
|
try {
|
1943
2209
|
for await (const chunk of fallbackStreamResult.stream) {
|
1944
|
-
if (chunk &&
|
2210
|
+
if (chunk &&
|
2211
|
+
"content" in chunk &&
|
2212
|
+
typeof chunk.content === "string") {
|
1945
2213
|
fallbackAccumulatedContent += chunk.content;
|
1946
2214
|
// Emit chunk event
|
1947
2215
|
self.emitter.emit("response:chunk", chunk.content);
|
@@ -0,0 +1,78 @@
|
|
1
|
+
/**
|
2
|
+
* Content type definitions for multimodal support
|
3
|
+
* Supports text and image content with provider-specific formatting
|
4
|
+
*/
|
5
|
+
/**
|
6
|
+
* Text content type for multimodal messages
|
7
|
+
*/
|
8
|
+
export interface TextContent {
|
9
|
+
type: "text";
|
10
|
+
text: string;
|
11
|
+
}
|
12
|
+
/**
|
13
|
+
* Image content type for multimodal messages
|
14
|
+
*/
|
15
|
+
export interface ImageContent {
|
16
|
+
type: "image";
|
17
|
+
data: Buffer | string;
|
18
|
+
mediaType?: "image/jpeg" | "image/png" | "image/gif" | "image/webp" | "image/bmp" | "image/tiff";
|
19
|
+
metadata?: {
|
20
|
+
description?: string;
|
21
|
+
quality?: "low" | "high" | "auto";
|
22
|
+
dimensions?: {
|
23
|
+
width: number;
|
24
|
+
height: number;
|
25
|
+
};
|
26
|
+
filename?: string;
|
27
|
+
};
|
28
|
+
}
|
29
|
+
/**
|
30
|
+
* Union type for all content types
|
31
|
+
*/
|
32
|
+
export type Content = TextContent | ImageContent;
|
33
|
+
/**
|
34
|
+
* Vision capability information for providers
|
35
|
+
*/
|
36
|
+
export interface VisionCapability {
|
37
|
+
provider: string;
|
38
|
+
supportedModels: string[];
|
39
|
+
maxImageSize?: number;
|
40
|
+
supportedFormats: string[];
|
41
|
+
maxImagesPerRequest?: number;
|
42
|
+
}
|
43
|
+
/**
|
44
|
+
* Provider-specific image format requirements
|
45
|
+
*/
|
46
|
+
export interface ProviderImageFormat {
|
47
|
+
provider: string;
|
48
|
+
format: "data_uri" | "base64" | "inline_data" | "source";
|
49
|
+
requiresPrefix?: boolean;
|
50
|
+
mimeTypeField?: string;
|
51
|
+
dataField?: string;
|
52
|
+
}
|
53
|
+
/**
|
54
|
+
* Image processing result
|
55
|
+
*/
|
56
|
+
export interface ProcessedImage {
|
57
|
+
data: string;
|
58
|
+
mediaType: string;
|
59
|
+
size: number;
|
60
|
+
format: "data_uri" | "base64" | "inline_data" | "source";
|
61
|
+
}
|
62
|
+
/**
|
63
|
+
* Multimodal message structure for provider adapters
|
64
|
+
*/
|
65
|
+
export interface MultimodalMessage {
|
66
|
+
role: "user" | "assistant" | "system";
|
67
|
+
content: Content[];
|
68
|
+
}
|
69
|
+
/**
|
70
|
+
* Provider-specific multimodal payload
|
71
|
+
*/
|
72
|
+
export interface ProviderMultimodalPayload {
|
73
|
+
provider: string;
|
74
|
+
model: string;
|
75
|
+
messages?: MultimodalMessage[];
|
76
|
+
contents?: unknown[];
|
77
|
+
[key: string]: unknown;
|
78
|
+
}
|
@@ -66,6 +66,25 @@ export interface ChatMessage {
|
|
66
66
|
/** Content of the message */
|
67
67
|
content: string;
|
68
68
|
}
|
69
|
+
/**
|
70
|
+
* Content format for multimodal messages (used internally)
|
71
|
+
*/
|
72
|
+
export interface MessageContent {
|
73
|
+
type: string;
|
74
|
+
text?: string;
|
75
|
+
image?: string;
|
76
|
+
mimeType?: string;
|
77
|
+
[key: string]: unknown;
|
78
|
+
}
|
79
|
+
/**
|
80
|
+
* Extended chat message for multimodal support (internal use)
|
81
|
+
*/
|
82
|
+
export interface MultimodalChatMessage {
|
83
|
+
/** Role of the message sender */
|
84
|
+
role: "user" | "assistant" | "system";
|
85
|
+
/** Content of the message - can be text or multimodal content array */
|
86
|
+
content: string | MessageContent[];
|
87
|
+
}
|
69
88
|
/**
|
70
89
|
* Events emitted by conversation memory system
|
71
90
|
*/
|
@@ -6,13 +6,16 @@ import type { EvaluationData } from "./evaluation.js";
|
|
6
6
|
import type { ChatMessage, ConversationMemoryConfig } from "./conversation.js";
|
7
7
|
import type { MiddlewareFactoryOptions } from "./middlewareTypes.js";
|
8
8
|
import type { JsonValue } from "./common.js";
|
9
|
+
import type { TextContent, ImageContent } from "./content.js";
|
9
10
|
/**
|
10
11
|
* Generate function options type - Primary method for content generation
|
11
|
-
*
|
12
|
+
* Supports multimodal content while maintaining backward compatibility
|
12
13
|
*/
|
13
14
|
export type GenerateOptions = {
|
14
15
|
input: {
|
15
16
|
text: string;
|
17
|
+
images?: Array<Buffer | string>;
|
18
|
+
content?: Array<TextContent | ImageContent>;
|
16
19
|
};
|
17
20
|
output?: {
|
18
21
|
format?: "text" | "structured" | "json";
|
@@ -5,10 +5,12 @@ export * from "./common.js";
|
|
5
5
|
export * from "./tools.js";
|
6
6
|
export * from "./providers.js";
|
7
7
|
export * from "./cli.js";
|
8
|
+
export * from "./taskClassificationTypes.js";
|
8
9
|
export type { Unknown, UnknownRecord, UnknownArray, JsonValue, JsonObject, JsonArray, ErrorInfo, Result, FunctionParameters, } from "./common.js";
|
9
10
|
export type { ToolArgs, ToolContext, ToolResult, ToolDefinition, SimpleTool, AvailableTool, ToolExecution, } from "./tools.js";
|
10
11
|
export type { AISDKModel, ProviderError, ProviderConfig } from "./providers.js";
|
11
12
|
export type { BaseCommandArgs, GenerateCommandArgs, MCPCommandArgs, ModelsCommandArgs, CommandResult, GenerateResult, StreamChunk, } from "./cli.js";
|
13
|
+
export type { TaskType, TaskClassification, ClassificationScores, ClassificationStats, ClassificationValidation, } from "./taskClassificationTypes.js";
|
12
14
|
export type { MCPTransportType, MCPServerConnectionStatus, MCPServerCategory, MCPServerStatus, MCPDiscoveredServer, MCPConnectedServer, MCPToolInfo, MCPExecutableTool, MCPServerMetadata, MCPToolMetadata, MCPServerRegistryEntry, } from "./mcpTypes.js";
|
13
15
|
export type { ExternalMCPServerInstance, ExternalMCPServerStatus, ExternalMCPToolInfo, ExternalMCPServerHealth, ExternalMCPConfigValidation, ExternalMCPOperationResult, ExternalMCPToolContext, ExternalMCPToolResult, ExternalMCPServerEvents, ExternalMCPManagerConfig, } from "./externalMcp.js";
|
14
16
|
export type { ModelCapability, ModelUseCase, ModelFilter, ModelResolutionContext, ModelStats, ModelPricing, } from "./providers.js";
|
package/dist/lib/types/index.js
CHANGED
@@ -9,6 +9,8 @@ export * from "./tools.js";
|
|
9
9
|
export * from "./providers.js";
|
10
10
|
// CLI types
|
11
11
|
export * from "./cli.js";
|
12
|
+
// Task classification types
|
13
|
+
export * from "./taskClassificationTypes.js";
|
12
14
|
// Generate types - NEW
|
13
15
|
export * from "./generateTypes.js";
|
14
16
|
// Analytics types - NEW
|
@@ -5,6 +5,7 @@ import type { AnalyticsData, TokenUsage } from "./analytics.js";
|
|
5
5
|
import type { EvaluationData } from "./evaluation.js";
|
6
6
|
import type { UnknownRecord, JsonValue } from "./common.js";
|
7
7
|
import type { ChatMessage } from "./conversation.js";
|
8
|
+
import type { TextContent, ImageContent } from "./content.js";
|
8
9
|
import type { MiddlewareFactoryOptions } from "./middlewareTypes.js";
|
9
10
|
/**
|
10
11
|
* Progress tracking and metadata for streaming operations
|
@@ -118,10 +119,12 @@ export interface AudioChunk {
|
|
118
119
|
channels: number;
|
119
120
|
encoding: PCMEncoding;
|
120
121
|
}
|
121
|
-
export
|
122
|
+
export interface StreamOptions {
|
122
123
|
input: {
|
123
|
-
text
|
124
|
+
text: string;
|
124
125
|
audio?: AudioInputSpec;
|
126
|
+
images?: Array<Buffer | string>;
|
127
|
+
content?: Array<TextContent | ImageContent>;
|
125
128
|
};
|
126
129
|
output?: {
|
127
130
|
format?: "text" | "structured" | "json";
|
@@ -166,7 +169,7 @@ export type StreamOptions = {
|
|
166
169
|
};
|
167
170
|
conversationMessages?: ChatMessage[];
|
168
171
|
middleware?: MiddlewareFactoryOptions;
|
169
|
-
}
|
172
|
+
}
|
170
173
|
/**
|
171
174
|
* Stream function result type - Primary output format for streaming
|
172
175
|
* Future-ready for multi-modal outputs while maintaining text focus
|
@@ -0,0 +1,52 @@
|
|
1
|
+
/**
|
2
|
+
* Task Classification Types
|
3
|
+
* Type definitions for the task classification system
|
4
|
+
*/
|
5
|
+
/**
|
6
|
+
* Supported task types for classification
|
7
|
+
*/
|
8
|
+
export type TaskType = "fast" | "reasoning";
|
9
|
+
/**
|
10
|
+
* Result of task classification analysis
|
11
|
+
*/
|
12
|
+
export interface TaskClassification {
|
13
|
+
/** The classified task type */
|
14
|
+
type: TaskType;
|
15
|
+
/** Confidence score (0-1) in the classification */
|
16
|
+
confidence: number;
|
17
|
+
/** Human-readable explanation of the classification decision */
|
18
|
+
reasoning: string;
|
19
|
+
}
|
20
|
+
/**
|
21
|
+
* Internal scoring data used during classification analysis
|
22
|
+
*/
|
23
|
+
export interface ClassificationScores {
|
24
|
+
/** Score indicating likelihood of fast task */
|
25
|
+
fastScore: number;
|
26
|
+
/** Score indicating likelihood of reasoning task */
|
27
|
+
reasoningScore: number;
|
28
|
+
/** Array of reasons contributing to the scores */
|
29
|
+
reasons: string[];
|
30
|
+
}
|
31
|
+
/**
|
32
|
+
* Statistics for batch classification analysis
|
33
|
+
*/
|
34
|
+
export interface ClassificationStats {
|
35
|
+
/** Total number of prompts analyzed */
|
36
|
+
total: number;
|
37
|
+
/** Number of prompts classified as fast */
|
38
|
+
fast: number;
|
39
|
+
/** Number of prompts classified as reasoning */
|
40
|
+
reasoning: number;
|
41
|
+
/** Average confidence across all classifications */
|
42
|
+
averageConfidence: number;
|
43
|
+
}
|
44
|
+
/**
|
45
|
+
* Validation result for testing classification accuracy
|
46
|
+
*/
|
47
|
+
export interface ClassificationValidation {
|
48
|
+
/** Whether the classification matched the expected result */
|
49
|
+
correct: boolean;
|
50
|
+
/** The actual classification result */
|
51
|
+
classification: TaskClassification;
|
52
|
+
}
|