@juspay/neurolink 7.35.0 → 7.37.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/dist/adapters/providerImageAdapter.d.ts +56 -0
  3. package/dist/adapters/providerImageAdapter.js +257 -0
  4. package/dist/cli/commands/config.d.ts +20 -20
  5. package/dist/cli/factories/commandFactory.d.ts +1 -0
  6. package/dist/cli/factories/commandFactory.js +26 -3
  7. package/dist/config/taskClassificationConfig.d.ts +51 -0
  8. package/dist/config/taskClassificationConfig.js +148 -0
  9. package/dist/core/baseProvider.js +99 -45
  10. package/dist/core/types.d.ts +3 -0
  11. package/dist/lib/adapters/providerImageAdapter.d.ts +56 -0
  12. package/dist/lib/adapters/providerImageAdapter.js +257 -0
  13. package/dist/lib/config/taskClassificationConfig.d.ts +51 -0
  14. package/dist/lib/config/taskClassificationConfig.js +148 -0
  15. package/dist/lib/core/baseProvider.js +99 -45
  16. package/dist/lib/core/types.d.ts +3 -0
  17. package/dist/lib/neurolink.d.ts +20 -0
  18. package/dist/lib/neurolink.js +276 -8
  19. package/dist/lib/types/content.d.ts +78 -0
  20. package/dist/lib/types/content.js +5 -0
  21. package/dist/lib/types/conversation.d.ts +19 -0
  22. package/dist/lib/types/generateTypes.d.ts +4 -1
  23. package/dist/lib/types/index.d.ts +2 -0
  24. package/dist/lib/types/index.js +2 -0
  25. package/dist/lib/types/streamTypes.d.ts +6 -3
  26. package/dist/lib/types/taskClassificationTypes.d.ts +52 -0
  27. package/dist/lib/types/taskClassificationTypes.js +5 -0
  28. package/dist/lib/utils/imageProcessor.d.ts +84 -0
  29. package/dist/lib/utils/imageProcessor.js +362 -0
  30. package/dist/lib/utils/messageBuilder.d.ts +8 -1
  31. package/dist/lib/utils/messageBuilder.js +279 -0
  32. package/dist/lib/utils/modelRouter.d.ts +107 -0
  33. package/dist/lib/utils/modelRouter.js +292 -0
  34. package/dist/lib/utils/promptRedaction.d.ts +29 -0
  35. package/dist/lib/utils/promptRedaction.js +62 -0
  36. package/dist/lib/utils/taskClassificationUtils.d.ts +55 -0
  37. package/dist/lib/utils/taskClassificationUtils.js +149 -0
  38. package/dist/lib/utils/taskClassifier.d.ts +23 -0
  39. package/dist/lib/utils/taskClassifier.js +94 -0
  40. package/dist/neurolink.d.ts +20 -0
  41. package/dist/neurolink.js +276 -8
  42. package/dist/types/content.d.ts +78 -0
  43. package/dist/types/content.js +5 -0
  44. package/dist/types/conversation.d.ts +19 -0
  45. package/dist/types/generateTypes.d.ts +4 -1
  46. package/dist/types/index.d.ts +2 -0
  47. package/dist/types/index.js +2 -0
  48. package/dist/types/streamTypes.d.ts +6 -3
  49. package/dist/types/taskClassificationTypes.d.ts +52 -0
  50. package/dist/types/taskClassificationTypes.js +5 -0
  51. package/dist/utils/imageProcessor.d.ts +84 -0
  52. package/dist/utils/imageProcessor.js +362 -0
  53. package/dist/utils/messageBuilder.d.ts +8 -1
  54. package/dist/utils/messageBuilder.js +279 -0
  55. package/dist/utils/modelRouter.d.ts +107 -0
  56. package/dist/utils/modelRouter.js +292 -0
  57. package/dist/utils/promptRedaction.d.ts +29 -0
  58. package/dist/utils/promptRedaction.js +62 -0
  59. package/dist/utils/taskClassificationUtils.d.ts +55 -0
  60. package/dist/utils/taskClassificationUtils.js +149 -0
  61. package/dist/utils/taskClassifier.d.ts +23 -0
  62. package/dist/utils/taskClassifier.js +94 -0
  63. package/package.json +1 -1
@@ -62,6 +62,7 @@ export declare class NeuroLink {
62
62
  private conversationMemory?;
63
63
  private conversationMemoryNeedsInit;
64
64
  private conversationMemoryConfig?;
65
+ private enableOrchestration;
65
66
  /**
66
67
  * Creates a new NeuroLink instance for AI text generation with MCP tool integration.
67
68
  *
@@ -70,6 +71,7 @@ export declare class NeuroLink {
70
71
  * @param config.conversationMemory.enabled - Whether to enable conversation memory (default: false)
71
72
  * @param config.conversationMemory.maxSessions - Maximum number of concurrent sessions (default: 100)
72
73
  * @param config.conversationMemory.maxTurnsPerSession - Maximum conversation turns per session (default: 50)
74
+ * @param config.enableOrchestration - Whether to enable smart model orchestration (default: false)
73
75
  *
74
76
  * @example
75
77
  * ```typescript
@@ -84,6 +86,11 @@ export declare class NeuroLink {
84
86
  * maxTurnsPerSession: 20
85
87
  * }
86
88
  * });
89
+ *
90
+ * // With orchestration enabled
91
+ * const neurolink = new NeuroLink({
92
+ * enableOrchestration: true
93
+ * });
87
94
  * ```
88
95
  *
89
96
  * @throws {Error} When provider registry setup fails
@@ -92,6 +99,7 @@ export declare class NeuroLink {
92
99
  */
93
100
  constructor(config?: {
94
101
  conversationMemory?: Partial<ConversationMemoryConfig>;
102
+ enableOrchestration?: boolean;
95
103
  });
96
104
  /**
97
105
  * Log constructor start with comprehensive environment analysis
@@ -158,6 +166,18 @@ export declare class NeuroLink {
158
166
  * Log MCP initialization completion
159
167
  */
160
168
  private logMCPInitComplete;
169
+ /**
170
+ * Apply orchestration to determine optimal provider and model
171
+ * @param options - Original GenerateOptions
172
+ * @returns Modified options with orchestrated provider marked in context, or empty object if validation fails
173
+ */
174
+ private applyOrchestration;
175
+ /**
176
+ * Apply orchestration to determine optimal provider and model for streaming
177
+ * @param options - Original StreamOptions
178
+ * @returns Modified options with orchestrated provider marked in context, or empty object if validation fails
179
+ */
180
+ private applyStreamOrchestration;
161
181
  /**
162
182
  * MAIN ENTRY POINT: Enhanced generate method with new function signature
163
183
  * Replaces both generateText and legacy methods
@@ -35,6 +35,9 @@ import { getConversationMessages, storeConversationTurn, } from "./utils/convers
35
35
  import { ExternalServerManager } from "./mcp/externalServerManager.js";
36
36
  // Import direct tools server for automatic registration
37
37
  import { directToolsServer } from "./mcp/servers/agent/directToolsServer.js";
38
+ // Import orchestration components
39
+ import { ModelRouter } from "./utils/modelRouter.js";
40
+ import { BinaryTaskClassifier } from "./utils/taskClassifier.js";
38
41
  import { isNonNullObject } from "./utils/typeUtils.js";
39
42
  // Core types imported from "./types/index.js"
40
43
  export class NeuroLink {
@@ -75,6 +78,8 @@ export class NeuroLink {
75
78
  conversationMemory;
76
79
  conversationMemoryNeedsInit = false;
77
80
  conversationMemoryConfig;
81
+ // Add orchestration property
82
+ enableOrchestration;
78
83
  /**
79
84
  * Creates a new NeuroLink instance for AI text generation with MCP tool integration.
80
85
  *
@@ -83,6 +88,7 @@ export class NeuroLink {
83
88
  * @param config.conversationMemory.enabled - Whether to enable conversation memory (default: false)
84
89
  * @param config.conversationMemory.maxSessions - Maximum number of concurrent sessions (default: 100)
85
90
  * @param config.conversationMemory.maxTurnsPerSession - Maximum conversation turns per session (default: 50)
91
+ * @param config.enableOrchestration - Whether to enable smart model orchestration (default: false)
86
92
  *
87
93
  * @example
88
94
  * ```typescript
@@ -97,6 +103,11 @@ export class NeuroLink {
97
103
  * maxTurnsPerSession: 20
98
104
  * }
99
105
  * });
106
+ *
107
+ * // With orchestration enabled
108
+ * const neurolink = new NeuroLink({
109
+ * enableOrchestration: true
110
+ * });
100
111
  * ```
101
112
  *
102
113
  * @throws {Error} When provider registry setup fails
@@ -112,6 +123,8 @@ export class NeuroLink {
112
123
  const constructorStartTime = Date.now();
113
124
  const constructorHrTimeStart = process.hrtime.bigint();
114
125
  const constructorId = `neurolink-constructor-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
126
+ // Initialize orchestration setting
127
+ this.enableOrchestration = config?.enableOrchestration ?? false;
115
128
  this.logConstructorStart(constructorId, constructorStartTime, constructorHrTimeStart, config);
116
129
  this.initializeProviderRegistry(constructorId, constructorStartTime, constructorHrTimeStart);
117
130
  this.initializeConversationMemory(config, constructorId, constructorStartTime, constructorHrTimeStart);
@@ -758,6 +771,206 @@ export class NeuroLink {
758
771
  mcpLogger.debug("💡 Memory cleanup suggestion: MCP initialization used significant memory. Consider calling MemoryManager.forceGC() after heavy operations.");
759
772
  }
760
773
  }
774
+ /**
775
+ * Apply orchestration to determine optimal provider and model
776
+ * @param options - Original GenerateOptions
777
+ * @returns Modified options with orchestrated provider marked in context, or empty object if validation fails
778
+ */
779
+ async applyOrchestration(options) {
780
+ const startTime = Date.now();
781
+ try {
782
+ // Ensure input.text exists before proceeding
783
+ if (!options.input?.text || typeof options.input.text !== "string") {
784
+ logger.debug("Orchestration skipped - no valid input text", {
785
+ hasInput: !!options.input,
786
+ hasText: !!options.input?.text,
787
+ textType: typeof options.input?.text,
788
+ });
789
+ return {}; // Return empty object to preserve existing fallback behavior
790
+ }
791
+ // Compute classification once to avoid duplicate calls
792
+ const classification = BinaryTaskClassifier.classify(options.input.text);
793
+ // Use the model router to get the optimal route
794
+ const route = ModelRouter.route(options.input.text);
795
+ // Validate that the routed provider is available and configured
796
+ const isProviderAvailable = await this.hasProviderEnvVars(route.provider);
797
+ if (!isProviderAvailable && route.provider !== "ollama") {
798
+ logger.debug("Orchestration provider validation failed", {
799
+ taskType: classification.type,
800
+ routedProvider: route.provider,
801
+ routedModel: route.model,
802
+ reason: "Provider not configured or missing environment variables",
803
+ orchestrationTime: `${Date.now() - startTime}ms`,
804
+ });
805
+ return {}; // Return empty object to preserve existing fallback behavior
806
+ }
807
+ // For Ollama, check if service is running and model is available
808
+ if (route.provider === "ollama") {
809
+ try {
810
+ const response = await fetch("http://localhost:11434/api/tags", {
811
+ method: "GET",
812
+ signal: AbortSignal.timeout(2000),
813
+ });
814
+ if (!response.ok) {
815
+ logger.debug("Orchestration provider validation failed", {
816
+ taskType: classification.type,
817
+ routedProvider: route.provider,
818
+ routedModel: route.model,
819
+ reason: "Ollama service not responding",
820
+ orchestrationTime: `${Date.now() - startTime}ms`,
821
+ });
822
+ return {}; // Return empty object to preserve existing fallback behavior
823
+ }
824
+ const { models } = await response.json();
825
+ const modelIsAvailable = models.some((m) => m.name === (route.model || "llama3.2:latest"));
826
+ if (!modelIsAvailable) {
827
+ logger.debug("Orchestration provider validation failed", {
828
+ taskType: classification.type,
829
+ routedProvider: route.provider,
830
+ routedModel: route.model,
831
+ reason: `Ollama model '${route.model || "llama3.2:latest"}' not found`,
832
+ orchestrationTime: `${Date.now() - startTime}ms`,
833
+ });
834
+ return {}; // Return empty object to preserve existing fallback behavior
835
+ }
836
+ }
837
+ catch (error) {
838
+ logger.debug("Orchestration provider validation failed", {
839
+ taskType: classification.type,
840
+ routedProvider: route.provider,
841
+ routedModel: route.model,
842
+ reason: error instanceof Error ? error.message : "Ollama service check failed",
843
+ orchestrationTime: `${Date.now() - startTime}ms`,
844
+ });
845
+ return {}; // Return empty object to preserve existing fallback behavior
846
+ }
847
+ }
848
+ logger.debug("Orchestration route determined", {
849
+ taskType: classification.type,
850
+ selectedProvider: route.provider,
851
+ selectedModel: route.model,
852
+ confidence: route.confidence,
853
+ reasoning: route.reasoning,
854
+ orchestrationTime: `${Date.now() - startTime}ms`,
855
+ });
856
+ // Mark preferred provider in context instead of directly setting provider
857
+ // This preserves global fallback behavior while indicating orchestration preference
858
+ return {
859
+ model: route.model,
860
+ context: {
861
+ ...(options.context || {}),
862
+ __orchestratedPreferredProvider: route.provider,
863
+ },
864
+ };
865
+ }
866
+ catch (error) {
867
+ logger.error("Orchestration failed", {
868
+ error: error instanceof Error ? error.message : String(error),
869
+ orchestrationTime: `${Date.now() - startTime}ms`,
870
+ });
871
+ throw error;
872
+ }
873
+ }
874
+ /**
875
+ * Apply orchestration to determine optimal provider and model for streaming
876
+ * @param options - Original StreamOptions
877
+ * @returns Modified options with orchestrated provider marked in context, or empty object if validation fails
878
+ */
879
+ async applyStreamOrchestration(options) {
880
+ const startTime = Date.now();
881
+ try {
882
+ // Ensure input.text exists before proceeding
883
+ if (!options.input?.text || typeof options.input.text !== "string") {
884
+ logger.debug("Stream orchestration skipped - no valid input text", {
885
+ hasInput: !!options.input,
886
+ hasText: !!options.input?.text,
887
+ textType: typeof options.input?.text,
888
+ });
889
+ return {}; // Return empty object to preserve existing fallback behavior
890
+ }
891
+ // Compute classification once to avoid duplicate calls
892
+ const classification = BinaryTaskClassifier.classify(options.input.text);
893
+ // Use the model router to get the optimal route
894
+ const route = ModelRouter.route(options.input.text);
895
+ // Validate that the routed provider is available and configured
896
+ const isProviderAvailable = await this.hasProviderEnvVars(route.provider);
897
+ if (!isProviderAvailable && route.provider !== "ollama") {
898
+ logger.debug("Stream orchestration provider validation failed", {
899
+ taskType: classification.type,
900
+ routedProvider: route.provider,
901
+ routedModel: route.model,
902
+ reason: "Provider not configured or missing environment variables",
903
+ orchestrationTime: `${Date.now() - startTime}ms`,
904
+ });
905
+ return {}; // Return empty object to preserve existing fallback behavior
906
+ }
907
+ // For Ollama, check if service is running and model is available
908
+ if (route.provider === "ollama") {
909
+ try {
910
+ const response = await fetch("http://localhost:11434/api/tags", {
911
+ method: "GET",
912
+ signal: AbortSignal.timeout(2000),
913
+ });
914
+ if (!response.ok) {
915
+ logger.debug("Stream orchestration provider validation failed", {
916
+ taskType: classification.type,
917
+ routedProvider: route.provider,
918
+ routedModel: route.model,
919
+ reason: "Ollama service not responding",
920
+ orchestrationTime: `${Date.now() - startTime}ms`,
921
+ });
922
+ return {}; // Return empty object to preserve existing fallback behavior
923
+ }
924
+ const { models } = await response.json();
925
+ const modelIsAvailable = models.some((m) => m.name === (route.model || "llama3.2:latest"));
926
+ if (!modelIsAvailable) {
927
+ logger.debug("Stream orchestration provider validation failed", {
928
+ taskType: classification.type,
929
+ routedProvider: route.provider,
930
+ routedModel: route.model,
931
+ reason: `Ollama model '${route.model || "llama3.2:latest"}' not found`,
932
+ orchestrationTime: `${Date.now() - startTime}ms`,
933
+ });
934
+ return {}; // Return empty object to preserve existing fallback behavior
935
+ }
936
+ }
937
+ catch (error) {
938
+ logger.debug("Stream orchestration provider validation failed", {
939
+ taskType: classification.type,
940
+ routedProvider: route.provider,
941
+ routedModel: route.model,
942
+ reason: error instanceof Error ? error.message : "Ollama service check failed",
943
+ orchestrationTime: `${Date.now() - startTime}ms`,
944
+ });
945
+ return {}; // Return empty object to preserve existing fallback behavior
946
+ }
947
+ }
948
+ logger.debug("Stream orchestration route determined", {
949
+ taskType: classification.type,
950
+ selectedProvider: route.provider,
951
+ selectedModel: route.model,
952
+ confidence: route.confidence,
953
+ reasoning: route.reasoning,
954
+ orchestrationTime: `${Date.now() - startTime}ms`,
955
+ });
956
+ // Mark preferred provider in context instead of directly setting provider
957
+ // This preserves global fallback behavior while indicating orchestration preference
958
+ return {
959
+ model: route.model,
960
+ context: {
961
+ ...(options.context || {}),
962
+ __orchestratedPreferredProvider: route.provider,
963
+ },
964
+ };
965
+ }
966
+ catch (error) {
967
+ logger.error("Stream orchestration failed", {
968
+ error: error instanceof Error ? error.message : String(error),
969
+ orchestrationTime: `${Date.now() - startTime}ms`,
970
+ });
971
+ throw error;
972
+ }
973
+ }
761
974
  /**
762
975
  * MAIN ENTRY POINT: Enhanced generate method with new function signature
763
976
  * Replaces both generateText and legacy methods
@@ -833,6 +1046,27 @@ export class NeuroLink {
833
1046
  throw new Error("Input text is required and must be a non-empty string");
834
1047
  }
835
1048
  const startTime = Date.now();
1049
+ // Apply orchestration if enabled and no specific provider/model requested
1050
+ if (this.enableOrchestration && !options.provider && !options.model) {
1051
+ try {
1052
+ const orchestratedOptions = await this.applyOrchestration(options);
1053
+ logger.debug("Orchestration applied", {
1054
+ originalProvider: options.provider || "auto",
1055
+ orchestratedProvider: orchestratedOptions.provider,
1056
+ orchestratedModel: orchestratedOptions.model,
1057
+ prompt: options.input.text.substring(0, 100),
1058
+ });
1059
+ // Use orchestrated options
1060
+ Object.assign(options, orchestratedOptions);
1061
+ }
1062
+ catch (error) {
1063
+ logger.warn("Orchestration failed, continuing with original options", {
1064
+ error: error instanceof Error ? error.message : String(error),
1065
+ originalProvider: options.provider || "auto",
1066
+ });
1067
+ // Continue with original options if orchestration fails
1068
+ }
1069
+ }
836
1070
  // Emit generation start event (NeuroLink format - keep existing)
837
1071
  this.emitter.emit("generation:start", {
838
1072
  provider: options.provider || "auto",
@@ -854,7 +1088,7 @@ export class NeuroLink {
854
1088
  // Continue with warning rather than throwing - graceful degradation
855
1089
  }
856
1090
  }
857
- // Convert to TextGenerationOptions using factory utilities
1091
+ // 🔧 CRITICAL FIX: Convert to TextGenerationOptions while preserving the input object for multimodal support
858
1092
  const baseOptions = {
859
1093
  prompt: options.input.text,
860
1094
  provider: options.provider,
@@ -868,6 +1102,7 @@ export class NeuroLink {
868
1102
  context: options.context,
869
1103
  evaluationDomain: options.evaluationDomain,
870
1104
  toolUsageContext: options.toolUsageContext,
1105
+ input: options.input, // This includes text, images, and content arrays
871
1106
  };
872
1107
  // Apply factory enhancement using centralized utilities
873
1108
  const textOptions = enhanceTextGenerationOptions(baseOptions, factoryResult);
@@ -1428,14 +1663,21 @@ export class NeuroLink {
1428
1663
  "ollama",
1429
1664
  ];
1430
1665
  const requestedProvider = options.provider === "auto" ? undefined : options.provider;
1431
- // If specific provider requested, only use that provider (no fallback)
1432
- const tryProviders = requestedProvider
1433
- ? [requestedProvider]
1434
- : providerPriority;
1666
+ // Check for orchestrated preferred provider in context
1667
+ const preferredOrchestrated = options.context && typeof options.context === 'object' && '__orchestratedPreferredProvider' in options.context
1668
+ ? options.context.__orchestratedPreferredProvider
1669
+ : undefined;
1670
+ // Build provider list with orchestrated preference first, then fallback to full list
1671
+ const tryProviders = preferredOrchestrated
1672
+ ? [preferredOrchestrated, ...providerPriority.filter((p) => p !== preferredOrchestrated)]
1673
+ : requestedProvider
1674
+ ? [requestedProvider]
1675
+ : providerPriority;
1435
1676
  logger.debug(`[${functionTag}] Starting direct generation`, {
1436
1677
  requestedProvider: requestedProvider || "auto",
1678
+ preferredOrchestrated: preferredOrchestrated || "none",
1437
1679
  tryProviders,
1438
- allowFallback: !requestedProvider,
1680
+ allowFallback: !requestedProvider || !!preferredOrchestrated,
1439
1681
  });
1440
1682
  let lastError = null;
1441
1683
  // Try each provider in order
@@ -1650,6 +1892,28 @@ export class NeuroLink {
1650
1892
  await this.initializeConversationMemoryForGeneration(streamId, startTime, hrTimeStart);
1651
1893
  // Initialize MCP
1652
1894
  await this.initializeMCP();
1895
+ const _originalPrompt = options.input.text;
1896
+ // Apply orchestration if enabled and no specific provider/model requested
1897
+ if (this.enableOrchestration && !options.provider && !options.model) {
1898
+ try {
1899
+ const orchestratedOptions = await this.applyStreamOrchestration(options);
1900
+ logger.debug("Stream orchestration applied", {
1901
+ originalProvider: options.provider || "auto",
1902
+ orchestratedProvider: orchestratedOptions.provider,
1903
+ orchestratedModel: orchestratedOptions.model,
1904
+ prompt: options.input.text?.substring(0, 100),
1905
+ });
1906
+ // Use orchestrated options
1907
+ Object.assign(options, orchestratedOptions);
1908
+ }
1909
+ catch (error) {
1910
+ logger.warn("Stream orchestration failed, continuing with original options", {
1911
+ error: error instanceof Error ? error.message : String(error),
1912
+ originalProvider: options.provider || "auto",
1913
+ });
1914
+ // Continue with original options if orchestration fails
1915
+ }
1916
+ }
1653
1917
  factoryResult = processStreamingFactoryOptions(options);
1654
1918
  enhancedOptions = createCleanStreamOptions(options);
1655
1919
  if (options.input?.text) {
@@ -1664,7 +1928,9 @@ export class NeuroLink {
1664
1928
  const processedStream = (async function* (self) {
1665
1929
  try {
1666
1930
  for await (const chunk of mcpStream) {
1667
- if (chunk && "content" in chunk && typeof chunk.content === "string") {
1931
+ if (chunk &&
1932
+ "content" in chunk &&
1933
+ typeof chunk.content === "string") {
1668
1934
  accumulatedContent += chunk.content;
1669
1935
  // Emit chunk event for compatibility
1670
1936
  self.emitter.emit("response:chunk", chunk.content);
@@ -1941,7 +2207,9 @@ export class NeuroLink {
1941
2207
  const fallbackProcessedStream = (async function* (self) {
1942
2208
  try {
1943
2209
  for await (const chunk of fallbackStreamResult.stream) {
1944
- if (chunk && "content" in chunk && typeof chunk.content === "string") {
2210
+ if (chunk &&
2211
+ "content" in chunk &&
2212
+ typeof chunk.content === "string") {
1945
2213
  fallbackAccumulatedContent += chunk.content;
1946
2214
  // Emit chunk event
1947
2215
  self.emitter.emit("response:chunk", chunk.content);
@@ -0,0 +1,78 @@
1
+ /**
2
+ * Content type definitions for multimodal support
3
+ * Supports text and image content with provider-specific formatting
4
+ */
5
+ /**
6
+ * Text content type for multimodal messages
7
+ */
8
+ export interface TextContent {
9
+ type: "text";
10
+ text: string;
11
+ }
12
+ /**
13
+ * Image content type for multimodal messages
14
+ */
15
+ export interface ImageContent {
16
+ type: "image";
17
+ data: Buffer | string;
18
+ mediaType?: "image/jpeg" | "image/png" | "image/gif" | "image/webp" | "image/bmp" | "image/tiff";
19
+ metadata?: {
20
+ description?: string;
21
+ quality?: "low" | "high" | "auto";
22
+ dimensions?: {
23
+ width: number;
24
+ height: number;
25
+ };
26
+ filename?: string;
27
+ };
28
+ }
29
+ /**
30
+ * Union type for all content types
31
+ */
32
+ export type Content = TextContent | ImageContent;
33
+ /**
34
+ * Vision capability information for providers
35
+ */
36
+ export interface VisionCapability {
37
+ provider: string;
38
+ supportedModels: string[];
39
+ maxImageSize?: number;
40
+ supportedFormats: string[];
41
+ maxImagesPerRequest?: number;
42
+ }
43
+ /**
44
+ * Provider-specific image format requirements
45
+ */
46
+ export interface ProviderImageFormat {
47
+ provider: string;
48
+ format: "data_uri" | "base64" | "inline_data" | "source";
49
+ requiresPrefix?: boolean;
50
+ mimeTypeField?: string;
51
+ dataField?: string;
52
+ }
53
+ /**
54
+ * Image processing result
55
+ */
56
+ export interface ProcessedImage {
57
+ data: string;
58
+ mediaType: string;
59
+ size: number;
60
+ format: "data_uri" | "base64" | "inline_data" | "source";
61
+ }
62
+ /**
63
+ * Multimodal message structure for provider adapters
64
+ */
65
+ export interface MultimodalMessage {
66
+ role: "user" | "assistant" | "system";
67
+ content: Content[];
68
+ }
69
+ /**
70
+ * Provider-specific multimodal payload
71
+ */
72
+ export interface ProviderMultimodalPayload {
73
+ provider: string;
74
+ model: string;
75
+ messages?: MultimodalMessage[];
76
+ contents?: unknown[];
77
+ [key: string]: unknown;
78
+ }
@@ -0,0 +1,5 @@
1
+ /**
2
+ * Content type definitions for multimodal support
3
+ * Supports text and image content with provider-specific formatting
4
+ */
5
+ export {};
@@ -66,6 +66,25 @@ export interface ChatMessage {
66
66
  /** Content of the message */
67
67
  content: string;
68
68
  }
69
+ /**
70
+ * Content format for multimodal messages (used internally)
71
+ */
72
+ export interface MessageContent {
73
+ type: string;
74
+ text?: string;
75
+ image?: string;
76
+ mimeType?: string;
77
+ [key: string]: unknown;
78
+ }
79
+ /**
80
+ * Extended chat message for multimodal support (internal use)
81
+ */
82
+ export interface MultimodalChatMessage {
83
+ /** Role of the message sender */
84
+ role: "user" | "assistant" | "system";
85
+ /** Content of the message - can be text or multimodal content array */
86
+ content: string | MessageContent[];
87
+ }
69
88
  /**
70
89
  * Events emitted by conversation memory system
71
90
  */
@@ -6,13 +6,16 @@ import type { EvaluationData } from "./evaluation.js";
6
6
  import type { ChatMessage, ConversationMemoryConfig } from "./conversation.js";
7
7
  import type { MiddlewareFactoryOptions } from "./middlewareTypes.js";
8
8
  import type { JsonValue } from "./common.js";
9
+ import type { TextContent, ImageContent } from "./content.js";
9
10
  /**
10
11
  * Generate function options type - Primary method for content generation
11
- * Future-ready for multi-modal capabilities while maintaining text focus
12
+ * Supports multimodal content while maintaining backward compatibility
12
13
  */
13
14
  export type GenerateOptions = {
14
15
  input: {
15
16
  text: string;
17
+ images?: Array<Buffer | string>;
18
+ content?: Array<TextContent | ImageContent>;
16
19
  };
17
20
  output?: {
18
21
  format?: "text" | "structured" | "json";
@@ -5,10 +5,12 @@ export * from "./common.js";
5
5
  export * from "./tools.js";
6
6
  export * from "./providers.js";
7
7
  export * from "./cli.js";
8
+ export * from "./taskClassificationTypes.js";
8
9
  export type { Unknown, UnknownRecord, UnknownArray, JsonValue, JsonObject, JsonArray, ErrorInfo, Result, FunctionParameters, } from "./common.js";
9
10
  export type { ToolArgs, ToolContext, ToolResult, ToolDefinition, SimpleTool, AvailableTool, ToolExecution, } from "./tools.js";
10
11
  export type { AISDKModel, ProviderError, ProviderConfig } from "./providers.js";
11
12
  export type { BaseCommandArgs, GenerateCommandArgs, MCPCommandArgs, ModelsCommandArgs, CommandResult, GenerateResult, StreamChunk, } from "./cli.js";
13
+ export type { TaskType, TaskClassification, ClassificationScores, ClassificationStats, ClassificationValidation, } from "./taskClassificationTypes.js";
12
14
  export type { MCPTransportType, MCPServerConnectionStatus, MCPServerCategory, MCPServerStatus, MCPDiscoveredServer, MCPConnectedServer, MCPToolInfo, MCPExecutableTool, MCPServerMetadata, MCPToolMetadata, MCPServerRegistryEntry, } from "./mcpTypes.js";
13
15
  export type { ExternalMCPServerInstance, ExternalMCPServerStatus, ExternalMCPToolInfo, ExternalMCPServerHealth, ExternalMCPConfigValidation, ExternalMCPOperationResult, ExternalMCPToolContext, ExternalMCPToolResult, ExternalMCPServerEvents, ExternalMCPManagerConfig, } from "./externalMcp.js";
14
16
  export type { ModelCapability, ModelUseCase, ModelFilter, ModelResolutionContext, ModelStats, ModelPricing, } from "./providers.js";
@@ -9,6 +9,8 @@ export * from "./tools.js";
9
9
  export * from "./providers.js";
10
10
  // CLI types
11
11
  export * from "./cli.js";
12
+ // Task classification types
13
+ export * from "./taskClassificationTypes.js";
12
14
  // Generate types - NEW
13
15
  export * from "./generateTypes.js";
14
16
  // Analytics types - NEW
@@ -5,6 +5,7 @@ import type { AnalyticsData, TokenUsage } from "./analytics.js";
5
5
  import type { EvaluationData } from "./evaluation.js";
6
6
  import type { UnknownRecord, JsonValue } from "./common.js";
7
7
  import type { ChatMessage } from "./conversation.js";
8
+ import type { TextContent, ImageContent } from "./content.js";
8
9
  import type { MiddlewareFactoryOptions } from "./middlewareTypes.js";
9
10
  /**
10
11
  * Progress tracking and metadata for streaming operations
@@ -118,10 +119,12 @@ export interface AudioChunk {
118
119
  channels: number;
119
120
  encoding: PCMEncoding;
120
121
  }
121
- export type StreamOptions = {
122
+ export interface StreamOptions {
122
123
  input: {
123
- text?: string;
124
+ text: string;
124
125
  audio?: AudioInputSpec;
126
+ images?: Array<Buffer | string>;
127
+ content?: Array<TextContent | ImageContent>;
125
128
  };
126
129
  output?: {
127
130
  format?: "text" | "structured" | "json";
@@ -166,7 +169,7 @@ export type StreamOptions = {
166
169
  };
167
170
  conversationMessages?: ChatMessage[];
168
171
  middleware?: MiddlewareFactoryOptions;
169
- };
172
+ }
170
173
  /**
171
174
  * Stream function result type - Primary output format for streaming
172
175
  * Future-ready for multi-modal outputs while maintaining text focus
@@ -0,0 +1,52 @@
1
+ /**
2
+ * Task Classification Types
3
+ * Type definitions for the task classification system
4
+ */
5
+ /**
6
+ * Supported task types for classification
7
+ */
8
+ export type TaskType = "fast" | "reasoning";
9
+ /**
10
+ * Result of task classification analysis
11
+ */
12
+ export interface TaskClassification {
13
+ /** The classified task type */
14
+ type: TaskType;
15
+ /** Confidence score (0-1) in the classification */
16
+ confidence: number;
17
+ /** Human-readable explanation of the classification decision */
18
+ reasoning: string;
19
+ }
20
+ /**
21
+ * Internal scoring data used during classification analysis
22
+ */
23
+ export interface ClassificationScores {
24
+ /** Score indicating likelihood of fast task */
25
+ fastScore: number;
26
+ /** Score indicating likelihood of reasoning task */
27
+ reasoningScore: number;
28
+ /** Array of reasons contributing to the scores */
29
+ reasons: string[];
30
+ }
31
+ /**
32
+ * Statistics for batch classification analysis
33
+ */
34
+ export interface ClassificationStats {
35
+ /** Total number of prompts analyzed */
36
+ total: number;
37
+ /** Number of prompts classified as fast */
38
+ fast: number;
39
+ /** Number of prompts classified as reasoning */
40
+ reasoning: number;
41
+ /** Average confidence across all classifications */
42
+ averageConfidence: number;
43
+ }
44
+ /**
45
+ * Validation result for testing classification accuracy
46
+ */
47
+ export interface ClassificationValidation {
48
+ /** Whether the classification matched the expected result */
49
+ correct: boolean;
50
+ /** The actual classification result */
51
+ classification: TaskClassification;
52
+ }